You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/14 01:53:27 UTC
[05/20] hive git commit: HIVE-19820 : add ACID stats support to
background stats updater and fix bunch of edge cases found in SU tests
(Sergey Shelukhin)
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 86b469c..7573eeb 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -603,6 +603,49 @@ class GrantRevokePrivilegeResponse
::Thrift::Struct.generate_accessors self
end
+class TruncateTableRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TABLENAME = 2
+ PARTNAMES = 3
+ TXNID = 4
+ WRITEID = 5
+ VALIDWRITEIDLIST = 6
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+ PARTNAMES => {:type => ::Thrift::Types::LIST, :name => 'partNames', :element => {:type => ::Thrift::Types::STRING}, :optional => true},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId', :default => -1, :optional => true},
+ WRITEID => {:type => ::Thrift::Types::I64, :name => 'writeId', :default => -1, :optional => true},
+ VALIDWRITEIDLIST => {:type => ::Thrift::Types::STRING, :name => 'validWriteIdList', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableName is unset!') unless @tableName
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
+class TruncateTableResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+
+ FIELDS = {
+
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
class Role
include ::Thrift::Struct, ::Thrift::Struct_Union
ROLENAME = 1
@@ -1559,15 +1602,11 @@ class ColumnStatistics
include ::Thrift::Struct, ::Thrift::Struct_Union
STATSDESC = 1
STATSOBJ = 2
- TXNID = 3
- VALIDWRITEIDLIST = 4
- ISSTATSCOMPLIANT = 5
+ ISSTATSCOMPLIANT = 3
FIELDS = {
STATSDESC => {:type => ::Thrift::Types::STRUCT, :name => 'statsDesc', :class => ::ColumnStatisticsDesc},
STATSOBJ => {:type => ::Thrift::Types::LIST, :name => 'statsObj', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}},
- TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId', :default => -1, :optional => true},
- VALIDWRITEIDLIST => {:type => ::Thrift::Types::STRING, :name => 'validWriteIdList', :optional => true},
ISSTATSCOMPLIANT => {:type => ::Thrift::Types::BOOL, :name => 'isStatsCompliant', :optional => true}
}
@@ -1628,6 +1667,23 @@ class SetPartitionsStatsRequest
::Thrift::Struct.generate_accessors self
end
+class SetPartitionsStatsResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ RESULT = 1
+
+ FIELDS = {
+ RESULT => {:type => ::Thrift::Types::BOOL, :name => 'result'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field result is unset!') if @result.nil?
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
class Schema
include ::Thrift::Struct, ::Thrift::Struct_Union
FIELDSCHEMAS = 1
@@ -5057,19 +5113,21 @@ end
class AlterPartitionsRequest
include ::Thrift::Struct, ::Thrift::Struct_Union
- DBNAME = 1
- TABLENAME = 2
- PARTITIONS = 3
- ENVIRONMENTCONTEXT = 4
- TXNID = 5
- WRITEID = 6
- VALIDWRITEIDLIST = 7
+ CATNAME = 1
+ DBNAME = 2
+ TABLENAME = 3
+ PARTITIONS = 4
+ ENVIRONMENTCONTEXT = 5
+ TXNID = 6
+ WRITEID = 7
+ VALIDWRITEIDLIST = 8
FIELDS = {
+ CATNAME => {:type => ::Thrift::Types::STRING, :name => 'catName', :optional => true},
DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}},
- ENVIRONMENTCONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environmentContext', :class => ::EnvironmentContext},
+ ENVIRONMENTCONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environmentContext', :class => ::EnvironmentContext, :optional => true},
TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId', :default => -1, :optional => true},
WRITEID => {:type => ::Thrift::Types::I64, :name => 'writeId', :default => -1, :optional => true},
VALIDWRITEIDLIST => {:type => ::Thrift::Types::STRING, :name => 'validWriteIdList', :optional => true}
@@ -5081,7 +5139,6 @@ class AlterPartitionsRequest
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableName is unset!') unless @tableName
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partitions is unset!') unless @partitions
- raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field environmentContext is unset!') unless @environmentContext
end
::Thrift::Struct.generate_accessors self
@@ -5102,6 +5159,54 @@ class AlterPartitionsResponse
::Thrift::Struct.generate_accessors self
end
+class AlterTableRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ CATNAME = 1
+ DBNAME = 2
+ TABLENAME = 3
+ TABLE = 4
+ ENVIRONMENTCONTEXT = 5
+ TXNID = 6
+ WRITEID = 7
+ VALIDWRITEIDLIST = 8
+
+ FIELDS = {
+ CATNAME => {:type => ::Thrift::Types::STRING, :name => 'catName', :optional => true},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+ TABLE => {:type => ::Thrift::Types::STRUCT, :name => 'table', :class => ::Table},
+ ENVIRONMENTCONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environmentContext', :class => ::EnvironmentContext, :optional => true},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId', :default => -1, :optional => true},
+ WRITEID => {:type => ::Thrift::Types::I64, :name => 'writeId', :default => -1, :optional => true},
+ VALIDWRITEIDLIST => {:type => ::Thrift::Types::STRING, :name => 'validWriteIdList', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableName is unset!') unless @tableName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field table is unset!') unless @table
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
+class AlterTableResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+
+ FIELDS = {
+
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
class MetaException < ::Thrift::Exception
include ::Thrift::Struct, ::Thrift::Struct_Union
def initialize(message=nil)
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
index fdcd3de..0a68424 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
@@ -579,6 +579,22 @@ module ThriftHiveMetastore
return
end
+ def truncate_table_req(req)
+ send_truncate_table_req(req)
+ return recv_truncate_table_req()
+ end
+
+ def send_truncate_table_req(req)
+ send_message('truncate_table_req', Truncate_table_req_args, :req => req)
+ end
+
+ def recv_truncate_table_req()
+ result = receive_message(Truncate_table_req_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'truncate_table_req failed: unknown result')
+ end
+
def get_tables(db_name, pattern)
send_get_tables(db_name, pattern)
return recv_get_tables()
@@ -827,6 +843,23 @@ module ThriftHiveMetastore
return
end
+ def alter_table_req(req)
+ send_alter_table_req(req)
+ return recv_alter_table_req()
+ end
+
+ def send_alter_table_req(req)
+ send_message('alter_table_req', Alter_table_req_args, :req => req)
+ end
+
+ def recv_alter_table_req()
+ result = receive_message(Alter_table_req_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'alter_table_req failed: unknown result')
+ end
+
def add_partition(new_part)
send_add_partition(new_part)
return recv_add_partition()
@@ -1432,21 +1465,21 @@ module ThriftHiveMetastore
return
end
- def alter_partitions_with_environment_context_req(req)
- send_alter_partitions_with_environment_context_req(req)
- return recv_alter_partitions_with_environment_context_req()
+ def alter_partitions_req(req)
+ send_alter_partitions_req(req)
+ return recv_alter_partitions_req()
end
- def send_alter_partitions_with_environment_context_req(req)
- send_message('alter_partitions_with_environment_context_req', Alter_partitions_with_environment_context_req_args, :req => req)
+ def send_alter_partitions_req(req)
+ send_message('alter_partitions_req', Alter_partitions_req_args, :req => req)
end
- def recv_alter_partitions_with_environment_context_req()
- result = receive_message(Alter_partitions_with_environment_context_req_result)
+ def recv_alter_partitions_req()
+ result = receive_message(Alter_partitions_req_result)
return result.success unless result.success.nil?
raise result.o1 unless result.o1.nil?
raise result.o2 unless result.o2.nil?
- raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'alter_partitions_with_environment_context_req failed: unknown result')
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'alter_partitions_req failed: unknown result')
end
def alter_partition_with_environment_context(db_name, tbl_name, new_part, environment_context)
@@ -1726,6 +1759,44 @@ module ThriftHiveMetastore
raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'update_partition_column_statistics failed: unknown result')
end
+ def update_table_column_statistics_req(req)
+ send_update_table_column_statistics_req(req)
+ return recv_update_table_column_statistics_req()
+ end
+
+ def send_update_table_column_statistics_req(req)
+ send_message('update_table_column_statistics_req', Update_table_column_statistics_req_args, :req => req)
+ end
+
+ def recv_update_table_column_statistics_req()
+ result = receive_message(Update_table_column_statistics_req_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ raise result.o3 unless result.o3.nil?
+ raise result.o4 unless result.o4.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'update_table_column_statistics_req failed: unknown result')
+ end
+
+ def update_partition_column_statistics_req(req)
+ send_update_partition_column_statistics_req(req)
+ return recv_update_partition_column_statistics_req()
+ end
+
+ def send_update_partition_column_statistics_req(req)
+ send_message('update_partition_column_statistics_req', Update_partition_column_statistics_req_args, :req => req)
+ end
+
+ def recv_update_partition_column_statistics_req()
+ result = receive_message(Update_partition_column_statistics_req_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ raise result.o3 unless result.o3.nil?
+ raise result.o4 unless result.o4.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'update_partition_column_statistics_req failed: unknown result')
+ end
+
def get_table_column_statistics(db_name, tbl_name, col_name)
send_get_table_column_statistics(db_name, tbl_name, col_name)
return recv_get_table_column_statistics()
@@ -3953,6 +4024,17 @@ module ThriftHiveMetastore
write_result(result, oprot, 'truncate_table', seqid)
end
+ def process_truncate_table_req(seqid, iprot, oprot)
+ args = read_args(iprot, Truncate_table_req_args)
+ result = Truncate_table_req_result.new()
+ begin
+ result.success = @handler.truncate_table_req(args.req)
+ rescue ::MetaException => o1
+ result.o1 = o1
+ end
+ write_result(result, oprot, 'truncate_table_req', seqid)
+ end
+
def process_get_tables(seqid, iprot, oprot)
args = read_args(iprot, Get_tables_args)
result = Get_tables_result.new()
@@ -4140,6 +4222,19 @@ module ThriftHiveMetastore
write_result(result, oprot, 'alter_table_with_cascade', seqid)
end
+ def process_alter_table_req(seqid, iprot, oprot)
+ args = read_args(iprot, Alter_table_req_args)
+ result = Alter_table_req_result.new()
+ begin
+ result.success = @handler.alter_table_req(args.req)
+ rescue ::InvalidOperationException => o1
+ result.o1 = o1
+ rescue ::MetaException => o2
+ result.o2 = o2
+ end
+ write_result(result, oprot, 'alter_table_req', seqid)
+ end
+
def process_add_partition(seqid, iprot, oprot)
args = read_args(iprot, Add_partition_args)
result = Add_partition_result.new()
@@ -4621,17 +4716,17 @@ module ThriftHiveMetastore
write_result(result, oprot, 'alter_partitions_with_environment_context', seqid)
end
- def process_alter_partitions_with_environment_context_req(seqid, iprot, oprot)
- args = read_args(iprot, Alter_partitions_with_environment_context_req_args)
- result = Alter_partitions_with_environment_context_req_result.new()
+ def process_alter_partitions_req(seqid, iprot, oprot)
+ args = read_args(iprot, Alter_partitions_req_args)
+ result = Alter_partitions_req_result.new()
begin
- result.success = @handler.alter_partitions_with_environment_context_req(args.req)
+ result.success = @handler.alter_partitions_req(args.req)
rescue ::InvalidOperationException => o1
result.o1 = o1
rescue ::MetaException => o2
result.o2 = o2
end
- write_result(result, oprot, 'alter_partitions_with_environment_context_req', seqid)
+ write_result(result, oprot, 'alter_partitions_req', seqid)
end
def process_alter_partition_with_environment_context(seqid, iprot, oprot)
@@ -4858,6 +4953,40 @@ module ThriftHiveMetastore
write_result(result, oprot, 'update_partition_column_statistics', seqid)
end
+ def process_update_table_column_statistics_req(seqid, iprot, oprot)
+ args = read_args(iprot, Update_table_column_statistics_req_args)
+ result = Update_table_column_statistics_req_result.new()
+ begin
+ result.success = @handler.update_table_column_statistics_req(args.req)
+ rescue ::NoSuchObjectException => o1
+ result.o1 = o1
+ rescue ::InvalidObjectException => o2
+ result.o2 = o2
+ rescue ::MetaException => o3
+ result.o3 = o3
+ rescue ::InvalidInputException => o4
+ result.o4 = o4
+ end
+ write_result(result, oprot, 'update_table_column_statistics_req', seqid)
+ end
+
+ def process_update_partition_column_statistics_req(seqid, iprot, oprot)
+ args = read_args(iprot, Update_partition_column_statistics_req_args)
+ result = Update_partition_column_statistics_req_result.new()
+ begin
+ result.success = @handler.update_partition_column_statistics_req(args.req)
+ rescue ::NoSuchObjectException => o1
+ result.o1 = o1
+ rescue ::InvalidObjectException => o2
+ result.o2 = o2
+ rescue ::MetaException => o3
+ result.o3 = o3
+ rescue ::InvalidInputException => o4
+ result.o4 = o4
+ end
+ write_result(result, oprot, 'update_partition_column_statistics_req', seqid)
+ end
+
def process_get_table_column_statistics(seqid, iprot, oprot)
args = read_args(iprot, Get_table_column_statistics_args)
result = Get_table_column_statistics_result.new()
@@ -7356,6 +7485,40 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Truncate_table_req_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ REQ = 1
+
+ FIELDS = {
+ REQ => {:type => ::Thrift::Types::STRUCT, :name => 'req', :class => ::TruncateTableRequest}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Truncate_table_req_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::TruncateTableResponse},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Get_tables_args
include ::Thrift::Struct, ::Thrift::Struct_Union
DB_NAME = 1
@@ -7924,6 +8087,42 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Alter_table_req_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ REQ = 1
+
+ FIELDS = {
+ REQ => {:type => ::Thrift::Types::STRUCT, :name => 'req', :class => ::AlterTableRequest}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Alter_table_req_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+ O2 = 2
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::AlterTableResponse},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::InvalidOperationException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::MetaException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Add_partition_args
include ::Thrift::Struct, ::Thrift::Struct_Union
NEW_PART = 1
@@ -9362,7 +9561,7 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
- class Alter_partitions_with_environment_context_req_args
+ class Alter_partitions_req_args
include ::Thrift::Struct, ::Thrift::Struct_Union
REQ = 1
@@ -9378,7 +9577,7 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
- class Alter_partitions_with_environment_context_req_result
+ class Alter_partitions_req_result
include ::Thrift::Struct, ::Thrift::Struct_Union
SUCCESS = 0
O1 = 1
@@ -10018,6 +10217,86 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Update_table_column_statistics_req_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ REQ = 1
+
+ FIELDS = {
+ REQ => {:type => ::Thrift::Types::STRUCT, :name => 'req', :class => ::SetPartitionsStatsRequest}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Update_table_column_statistics_req_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+ O2 = 2
+ O3 = 3
+ O4 = 4
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::SetPartitionsStatsResponse},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::NoSuchObjectException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::InvalidObjectException},
+ O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::MetaException},
+ O4 => {:type => ::Thrift::Types::STRUCT, :name => 'o4', :class => ::InvalidInputException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Update_partition_column_statistics_req_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ REQ = 1
+
+ FIELDS = {
+ REQ => {:type => ::Thrift::Types::STRUCT, :name => 'req', :class => ::SetPartitionsStatsRequest}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Update_partition_column_statistics_req_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+ O2 = 2
+ O3 = 3
+ O4 = 4
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::SetPartitionsStatsResponse},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::NoSuchObjectException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::InvalidObjectException},
+ O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::MetaException},
+ O4 => {:type => ::Thrift::Types::STRUCT, :name => 'o4', :class => ::InvalidInputException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Get_table_column_statistics_args
include ::Thrift::Struct, ::Thrift::Struct_Union
DB_NAME = 1
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index af9b0b1..35be3c4 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -157,10 +157,6 @@ public class StatsSetupConst {
public static final String CASCADE = "CASCADE";
- // TODO: when alter calls are switched to req/resp models, replace these and the above with fields.
- public static final String TXN_ID = "WRITER_TXN_ID";
- public static final String VALID_WRITE_IDS = "WRITER_WRITE_ID";
-
public static final String TRUE = "true";
public static final String FALSE = "false";
@@ -275,10 +271,11 @@ public class StatsSetupConst {
if (params == null) {
return false;
}
+ // TODO: should this also check that the basic flag is valid?
ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
return stats.columnStats.containsKey(colName);
}
-
+
public static void clearColumnStatsState(Map<String, String> params) {
if (params == null) {
return;
@@ -321,7 +318,7 @@ public class StatsSetupConst {
setColumnStatsState(params, cols);
}
}
-
+
private static ColumnStatsAccurate parseStatsAcc(String statsAcc) {
if (statsAcc == null) {
return new ColumnStatsAccurate();
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
index e7cf07f..f3dc264 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
@@ -61,7 +61,7 @@ public interface AlterHandler extends Configurable {
default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname,
String name, Table newTable, EnvironmentContext envContext)
throws InvalidOperationException, MetaException {
- alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null);
+ alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null, -1, null);
}
/**
@@ -88,7 +88,8 @@ public interface AlterHandler extends Configurable {
*/
void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname,
String name, Table newTable, EnvironmentContext envContext,
- IHMSHandler handler) throws InvalidOperationException, MetaException;
+ IHMSHandler handler, long txnId, String writeIdList)
+ throws InvalidOperationException, MetaException;
/**
* @deprecated As of release 2.2.0. Replaced by {@link #alterPartition(RawStore, Warehouse, String,
@@ -145,7 +146,7 @@ public interface AlterHandler extends Configurable {
Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName,
final String dbname, final String name, final List<String> part_vals,
final Partition new_part, EnvironmentContext environmentContext,
- IHMSHandler handler)
+ IHMSHandler handler, long txnId, String validWriteIds)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 8b2a6ba..e8226f8 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -90,7 +90,8 @@ public class HiveAlterHandler implements AlterHandler {
@Override
public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname,
String name, Table newt, EnvironmentContext environmentContext,
- IHMSHandler handler) throws InvalidOperationException, MetaException {
+ IHMSHandler handler, long txnId, String writeIdList)
+ throws InvalidOperationException, MetaException {
catName = normalizeIdentifier(catName);
name = name.toLowerCase();
dbname = dbname.toLowerCase();
@@ -296,7 +297,7 @@ public class HiveAlterHandler implements AlterHandler {
partValues.add(part.getValues());
}
msdb.alterPartitions(catName, newDbName, newTblName, partValues,
- partBatch, -1, -1, null);
+ partBatch, newt.getWriteId(), txnId, writeIdList);
}
}
@@ -304,14 +305,15 @@ public class HiveAlterHandler implements AlterHandler {
ColumnStatistics newPartColStats = partColStats.getValue();
newPartColStats.getStatsDesc().setDbName(newDbName);
newPartColStats.getStatsDesc().setTableName(newTblName);
- msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues());
+ msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues(),
+ txnId, writeIdList, newt.getWriteId());
}
} else {
- alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext);
+ alterTableUpdateTableColumnStats(
+ msdb, oldt, newt, environmentContext, txnId, writeIdList);
}
} else {
// operations other than table rename
-
if (MetaStoreUtils.requireCalStats(null, null, newt, environmentContext) &&
!isPartitionedTable) {
Database db = msdb.getDatabase(catName, newDbName);
@@ -330,23 +332,26 @@ public class HiveAlterHandler implements AlterHandler {
ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name,
part.getValues(), oldCols, oldt, part, null);
assert(colStats == null);
- // Note: we don't do txn stats validation here; this can only delete stats?
if (cascade) {
- msdb.alterPartition(catName, dbname, name, part.getValues(), part, -1, null);
+ msdb.alterPartition(
+ catName, dbname, name, part.getValues(), part, txnId, writeIdList);
} else {
// update changed properties (stats)
oldPart.setParameters(part.getParameters());
- msdb.alterPartition(catName, dbname, name, part.getValues(), oldPart, -1, null);
+ msdb.alterPartition(
+ catName, dbname, name, part.getValues(), oldPart, txnId, writeIdList);
}
}
// Don't validate table-level stats for a partitoned table.
msdb.alterTable(catName, dbname, name, newt, -1, null);
} else {
LOG.warn("Alter table not cascaded to partitions.");
- alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext);
+ alterTableUpdateTableColumnStats(
+ msdb, oldt, newt, environmentContext, txnId, writeIdList);
}
} else {
- alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext);
+ alterTableUpdateTableColumnStats(
+ msdb, oldt, newt, environmentContext, txnId, writeIdList);
}
}
@@ -426,14 +431,13 @@ public class HiveAlterHandler implements AlterHandler {
EnvironmentContext environmentContext)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
return alterPartition(msdb, wh, DEFAULT_CATALOG_NAME, dbname, name, part_vals, new_part,
- environmentContext, null);
+ environmentContext, null, -1, null);
}
@Override
- public Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName,
- final String dbname, final String name,
- final List<String> part_vals, final Partition new_part,
- EnvironmentContext environmentContext, IHMSHandler handler)
+ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, String dbname,
+ String name, List<String> part_vals, final Partition new_part,
+ EnvironmentContext environmentContext, IHMSHandler handler, long txnId, String validWriteIds)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
boolean success = false;
Partition oldPart;
@@ -449,13 +453,6 @@ public class HiveAlterHandler implements AlterHandler {
new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
- long txnId = -1;
- String validWriteIds = null;
- if (environmentContext != null && environmentContext.isSetProperties()
- && environmentContext.getProperties().containsKey(StatsSetupConst.VALID_WRITE_IDS)) {
- txnId = Long.parseLong(environmentContext.getProperties().get(StatsSetupConst.TXN_ID));
- validWriteIds = environmentContext.getProperties().get(StatsSetupConst.VALID_WRITE_IDS);
- }
//alter partition
if (part_vals == null || part_vals.size() == 0) {
@@ -623,7 +620,10 @@ public class HiveAlterHandler implements AlterHandler {
if (cs != null) {
cs.getStatsDesc().setPartName(newPartName);
try {
- msdb.updatePartitionColumnStatistics(cs, new_part.getValues());
+ // Verifying ACID state again is not strictly needed here (alterPartition above does it),
+ // but we are going to use the uniform approach for simplicity.
+ msdb.updatePartitionColumnStatistics(cs, new_part.getValues(),
+ txnId, validWriteIds, new_part.getWriteId());
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
} catch (NoSuchObjectException nsoe) {
@@ -796,7 +796,7 @@ public class HiveAlterHandler implements AlterHandler {
@VisibleForTesting
void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable,
- EnvironmentContext ec)
+ EnvironmentContext ec, long txnId, String validWriteIds)
throws MetaException, InvalidObjectException {
String catName = normalizeIdentifier(oldTable.isSetCatName() ? oldTable.getCatName() :
getDefaultCatalog(conf));
@@ -804,77 +804,65 @@ public class HiveAlterHandler implements AlterHandler {
String tableName = normalizeIdentifier(oldTable.getTableName());
String newDbName = newTable.getDbName().toLowerCase();
String newTableName = normalizeIdentifier(newTable.getTableName());
- long txnId = -1;
- String validWriteIds = null;
- if (ec != null && ec.isSetProperties() && ec.getProperties().containsKey(
- StatsSetupConst.VALID_WRITE_IDS)) {
- txnId = Long.parseLong(ec.getProperties().get(StatsSetupConst.TXN_ID));
- validWriteIds = ec.getProperties().get(StatsSetupConst.VALID_WRITE_IDS);
- }
try {
List<FieldSchema> oldCols = oldTable.getSd().getCols();
List<FieldSchema> newCols = newTable.getSd().getCols();
List<ColumnStatisticsObj> newStatsObjs = new ArrayList<>();
ColumnStatistics colStats = null;
- boolean updateColumnStats = true;
-
- // Nothing to update if everything is the same
- if (newDbName.equals(dbName) &&
- newTableName.equals(tableName) &&
- MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) {
- updateColumnStats = false;
+ boolean updateColumnStats = !newDbName.equals(dbName) || !newTableName.equals(tableName)
+ || !MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols);
+ if (updateColumnStats) {
+ List<String> oldColNames = new ArrayList<>(oldCols.size());
+ for (FieldSchema oldCol : oldCols) {
+ oldColNames.add(oldCol.getName());
}
- if (updateColumnStats) {
- List<String> oldColNames = new ArrayList<>(oldCols.size());
- for (FieldSchema oldCol : oldCols) {
- oldColNames.add(oldCol.getName());
- }
-
- // Collect column stats which need to be rewritten and remove old stats.
- colStats = msdb.getTableColumnStatistics(catName, dbName, tableName, oldColNames);
- if (colStats == null) {
- updateColumnStats = false;
- } else {
- List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
- if (statsObjs != null) {
- List<String> deletedCols = new ArrayList<>();
- for (ColumnStatisticsObj statsObj : statsObjs) {
- boolean found = false;
- for (FieldSchema newCol : newCols) {
- if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
- && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
- found = true;
- break;
- }
+ // NOTE: this doesn't check stats being compliant, but the alterTable call below does.
+ // The worst we can do is delete the stats.
+ // Collect column stats which need to be rewritten and remove old stats.
+ colStats = msdb.getTableColumnStatistics(catName, dbName, tableName, oldColNames);
+ if (colStats == null) {
+ updateColumnStats = false;
+ } else {
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ if (statsObjs != null) {
+ List<String> deletedCols = new ArrayList<>();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ boolean found = false;
+ for (FieldSchema newCol : newCols) {
+ if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
+ && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
+ found = true;
+ break;
}
+ }
- if (found) {
- if (!newDbName.equals(dbName) || !newTableName.equals(tableName)) {
- msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName());
- newStatsObjs.add(statsObj);
- deletedCols.add(statsObj.getColName());
- }
- } else {
+ if (found) {
+ if (!newDbName.equals(dbName) || !newTableName.equals(tableName)) {
msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName());
+ newStatsObjs.add(statsObj);
deletedCols.add(statsObj.getColName());
}
+ } else {
+ msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName());
+ deletedCols.add(statsObj.getColName());
}
- StatsSetupConst.removeColumnStatsState(newTable.getParameters(), deletedCols);
}
+ StatsSetupConst.removeColumnStatsState(newTable.getParameters(), deletedCols);
}
}
+ }
- // Change to new table and append stats for the new table
- msdb.alterTable(catName, dbName, tableName, newTable, txnId, validWriteIds);
- if (updateColumnStats && !newStatsObjs.isEmpty()) {
- ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
- statsDesc.setDbName(newDbName);
- statsDesc.setTableName(newTableName);
- colStats.setStatsObj(newStatsObjs);
- msdb.updateTableColumnStatistics(colStats);
- }
+ // Change to new table and append stats for the new table
+ msdb.alterTable(catName, dbName, tableName, newTable, txnId, validWriteIds);
+ if (updateColumnStats && !newStatsObjs.isEmpty()) {
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ statsDesc.setDbName(newDbName);
+ statsDesc.setTableName(newTableName);
+ colStats.setStatsObj(newStatsObjs);
+ msdb.updateTableColumnStatistics(colStats, txnId, validWriteIds, newTable.getWriteId());
+ }
} catch (NoSuchObjectException nsoe) {
LOG.debug("Could not find db entry." + nsoe);
} catch (InvalidInputException e) {
@@ -907,7 +895,7 @@ public class HiveAlterHandler implements AlterHandler {
oldColNames.add(oldCol.getName());
}
List<String> oldPartNames = Lists.newArrayList(oldPartName);
- // Note: doesn't take txn stats into account. This method can only remove stats.
+ // TODO: doesn't take txn stats into account. This method can only remove stats.
List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(catName, dbname, tblname,
oldPartNames, oldColNames);
assert (partsColStats.size() <= 1);
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index a46d2f9..091e5de 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2692,12 +2692,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
return;
}
- private void alterPartitionForTruncate(final RawStore ms,
- final String catName,
- final String dbName,
- final String tableName,
- final Table table,
- final Partition partition) throws Exception {
+ private void alterPartitionForTruncate(RawStore ms, String catName, String dbName, String tableName,
+ Table table, Partition partition, long txnId, String validWriteIds, long writeId) throws Exception {
EnvironmentContext environmentContext = new EnvironmentContext();
updateStatsForTruncate(partition.getParameters(), environmentContext);
@@ -2713,20 +2709,21 @@ public class HiveMetaStore extends ThriftHiveMetastore {
new AlterPartitionEvent(partition, partition, table, true, true, this));
}
+ if (writeId > 0) {
+ partition.setWriteId(writeId);
+ }
alterHandler.alterPartition(ms, wh, catName, dbName, tableName, null, partition,
- environmentContext, this);
+ environmentContext, this, txnId, validWriteIds);
}
- private void alterTableStatsForTruncate(final RawStore ms,
- final String catName,
- final String dbName,
- final String tableName,
- final Table table,
- final List<String> partNames) throws Exception {
+ private void alterTableStatsForTruncate(RawStore ms, String catName, String dbName,
+ String tableName, Table table, List<String> partNames,
+ long txnId, String validWriteIds, long writeId) throws Exception {
if (partNames == null) {
if (0 != table.getPartitionKeysSize()) {
for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) {
- alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition);
+ alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition,
+ txnId, validWriteIds, writeId);
}
} else {
EnvironmentContext environmentContext = new EnvironmentContext();
@@ -2744,11 +2741,17 @@ public class HiveMetaStore extends ThriftHiveMetastore {
new AlterTableEvent(table, table, true, true, this));
}
- alterHandler.alterTable(ms, wh, catName, dbName, tableName, table, environmentContext, this);
+ // TODO: this should actually pass thru and set writeId for txn stats.
+ if (writeId > 0) {
+ table.setWriteId(writeId);
+ }
+ alterHandler.alterTable(ms, wh, catName, dbName, tableName, table,
+ environmentContext, this, txnId, validWriteIds);
}
} else {
for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) {
- alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition);
+ alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition,
+ txnId, validWriteIds, writeId);
}
}
return;
@@ -2786,6 +2789,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@Override
public void truncate_table(final String dbName, final String tableName, List<String> partNames)
throws NoSuchObjectException, MetaException {
+ // Deprecated path, won't work for txn tables.
+ truncateTableInternal(dbName, tableName, partNames, -1, null, -1);
+ }
+
+ @Override
+ public TruncateTableResponse truncate_table_req(TruncateTableRequest req)
+ throws MetaException, TException {
+ truncateTableInternal(req.getDbName(), req.getTableName(), req.getPartNames(),
+ req.getTxnId(), req.getValidWriteIdList(), req.getWriteId());
+ return new TruncateTableResponse();
+ }
+
+ private void truncateTableInternal(String dbName, String tableName, List<String> partNames,
+ long txnId, String validWriteIds, long writeId) throws MetaException, NoSuchObjectException {
try {
String[] parsedDbName = parseDbName(dbName, conf);
Table tbl = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
@@ -2817,7 +2834,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// Alter the table/partition stats and also notify truncate table event
alterTableStatsForTruncate(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tableName, tbl, partNames);
+ tableName, tbl, partNames, txnId, validWriteIds, writeId);
} catch (IOException e) {
throw new MetaException(e.getMessage());
} catch (MetaException | NoSuchObjectException e) {
@@ -4804,6 +4821,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
final EnvironmentContext envContext)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
+ // TODO: this method name is confusing, it actually does full alter (sortof)
rename_partition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, null, newPartition,
envContext);
}
@@ -4855,7 +4873,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
oldPart = alterHandler.alterPartition(getMS(), wh, catName, db_name, tbl_name,
- part_vals, new_part, envContext, this);
+ part_vals, new_part, envContext, this, -1, null);
// Only fetch the table if we actually have a listener
Table table = null;
@@ -4890,15 +4908,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
public void alter_partitions(final String db_name, final String tbl_name,
final List<Partition> new_parts)
throws TException {
- alter_partitions_with_environment_context(
- db_name, tbl_name, new_parts, null, -1, null, -1);
+ String[] o = parseDbName(db_name, conf);
+ alter_partitions_with_environment_context(o[0], o[1],
+ tbl_name, new_parts, null, -1, null, -1);
}
@Override
- public AlterPartitionsResponse alter_partitions_with_environment_context_req(
- AlterPartitionsRequest req)
- throws TException {
- alter_partitions_with_environment_context(
+ public AlterPartitionsResponse alter_partitions_req(AlterPartitionsRequest req) throws TException {
+ alter_partitions_with_environment_context(req.getCatName(),
req.getDbName(), req.getTableName(), req.getPartitions(), req.getEnvironmentContext(),
req.isSetTxnId() ? req.getTxnId() : -1,
req.isSetValidWriteIdList() ? req.getValidWriteIdList() : null,
@@ -4912,17 +4929,23 @@ public class HiveMetaStore extends ThriftHiveMetastore {
public void alter_partitions_with_environment_context(final String db_name, final String tbl_name,
final List<Partition> new_parts, EnvironmentContext environmentContext)
throws TException {
- alter_partitions_with_environment_context(db_name, tbl_name, new_parts, environmentContext,
+ String[] o = parseDbName(db_name, conf);
+ alter_partitions_with_environment_context(o[0], o[1], tbl_name, new_parts, environmentContext,
-1, null, -1);
}
- private void alter_partitions_with_environment_context(final String db_name, final String tbl_name,
+ private void alter_partitions_with_environment_context(String catName, String db_name, final String tbl_name,
final List<Partition> new_parts, EnvironmentContext environmentContext,
long txnId, String writeIdList, long writeId)
throws TException {
+ if (environmentContext == null) {
+ environmentContext = new EnvironmentContext();
+ }
+ if (catName == null) {
+ catName = MetaStoreUtils.getDefaultCatalog(conf);
+ }
- String[] parsedDbName = parseDbName(db_name, conf);
- startTableFunction("alter_partitions", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
+ startTableFunction("alter_partitions", catName, db_name, tbl_name);
if (LOG.isInfoEnabled()) {
for (Partition tmpPart : new_parts) {
@@ -4939,10 +4962,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (!tmpPart.isSetCatName()) {
tmpPart.setCatName(getDefaultCatalog(conf));
}
- firePreEvent(new PreAlterPartitionEvent(parsedDbName[DB_NAME], tbl_name, null, tmpPart, this));
+ firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, null, tmpPart, this));
}
- oldParts = alterHandler.alterPartitions(getMS(), wh, parsedDbName[CAT_NAME],
- parsedDbName[DB_NAME], tbl_name, new_parts, environmentContext, txnId, writeIdList, writeId, this);
+ oldParts = alterHandler.alterPartitions(getMS(), wh,
+ catName, db_name, tbl_name, new_parts, environmentContext, txnId, writeIdList, writeId, this);
Iterator<Partition> olditr = oldParts.iterator();
// Only fetch the table if we have a listener that needs it.
Table table = null;
@@ -4956,8 +4979,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
if (table == null) {
- table = getMS().getTable(
- parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, -1, null);
+ table = getMS().getTable(catName, db_name, tbl_name, -1, null);
}
if (!listeners.isEmpty()) {
@@ -4995,7 +5017,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throws InvalidOperationException, MetaException {
// Do not set an environment context.
String[] parsedDbName = parseDbName(dbname, conf);
- alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable, null);
+ alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable,
+ null, -1, null);
}
@Override
@@ -5008,7 +5031,16 @@ public class HiveMetaStore extends ThriftHiveMetastore {
envContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);
}
String[] parsedDbName = parseDbName(dbname, conf);
- alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable, envContext);
+ alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable,
+ envContext, -1, null);
+ }
+
+ @Override
+ public AlterTableResponse alter_table_req(AlterTableRequest req)
+ throws InvalidOperationException, MetaException, TException {
+ alter_table_core(req.getCatName(), req.getDbName(), req.getTableName(),
+ req.getTable(), req.getEnvironmentContext(), req.getTxnId(), req.getValidWriteIdList());
+ return new AlterTableResponse();
}
@Override
@@ -5017,14 +5049,21 @@ public class HiveMetaStore extends ThriftHiveMetastore {
final EnvironmentContext envContext)
throws InvalidOperationException, MetaException {
String[] parsedDbName = parseDbName(dbname, conf);
- alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable, envContext);
+ alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ name, newTable, envContext, -1, null);
}
- private void alter_table_core(final String catName, final String dbname, final String name,
- final Table newTable, final EnvironmentContext envContext)
+ private void alter_table_core(String catName, String dbname, String name, Table newTable,
+ EnvironmentContext envContext, long txnId, String validWriteIdList)
throws InvalidOperationException, MetaException {
startFunction("alter_table", ": " + TableName.getQualified(catName, dbname, name)
+ " newtbl=" + newTable.getTableName());
+ if (envContext == null) {
+ envContext = new EnvironmentContext();
+ }
+ if (catName == null) {
+ catName = MetaStoreUtils.getDefaultCatalog(conf);
+ }
// Update the time if it hasn't been specified.
if (newTable.getParameters() == null ||
@@ -5052,7 +5091,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
Table oldt = get_table_core(catName, dbname, name);
firePreEvent(new PreAlterTableEvent(oldt, newTable, this));
alterHandler.alterTable(getMS(), wh, catName, dbname, name, newTable,
- envContext, this);
+ envContext, this, txnId, validWriteIdList);
success = true;
} catch (NoSuchObjectException e) {
// thrown when the table to be altered does not exist
@@ -5600,6 +5639,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
private String lowerCaseConvertPartName(String partName) throws MetaException {
+ if (partName == null) return partName;
boolean isFirst = true;
Map<String, String> partSpec = Warehouse.makeEscSpecFromName(partName);
String convertedPartName = new String();
@@ -5618,6 +5658,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
return convertedPartName;
}
+ @Deprecated
@Override
public ColumnStatistics get_table_column_statistics(String dbName, String tableName,
String colName) throws TException {
@@ -5660,7 +5701,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
ColumnStatistics cs = getMS().getTableColumnStatistics(
catName, dbName, tblName, lowerCaseColNames,
request.getTxnId(), request.getValidWriteIdList());
- result = new TableStatsResult((cs == null || cs.getStatsObj() == null)
+ // Note: stats compliance is not propagated to the client; instead, we just return nothing
+ // if stats are not compliant for now. This won't work for stats merging, but that
+ // is currently only done on metastore size (see set_aggr...).
+ // For some optimizations we might make use of incorrect stats that are "better than
+ // nothing", so this may change in future.
+ result = new TableStatsResult((cs == null || cs.getStatsObj() == null
+ || (cs.isSetIsStatsCompliant() && !cs.isIsStatsCompliant()))
? Lists.newArrayList() : cs.getStatsObj());
} finally {
endFunction("get_table_statistics_req", result == null, null, tblName);
@@ -5725,8 +5772,16 @@ public class HiveMetaStore extends ThriftHiveMetastore {
request.isSetTxnId() ? request.getTxnId() : -1,
request.isSetValidWriteIdList() ? request.getValidWriteIdList() : null);
Map<String, List<ColumnStatisticsObj>> map = new HashMap<>();
- for (ColumnStatistics stat : stats) {
- map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj());
+ if (stats != null) {
+ for (ColumnStatistics stat : stats) {
+ // Note: stats compliance is not propagated to the client; instead, we just return nothing
+ // if stats are not compliant for now. This won't work for stats merging, but that
+ // is currently only done on metastore size (see set_aggr...).
+ // For some optimizations we might make use of incorrect stats that are "better than
+ // nothing", so this may change in future.
+ if (stat.isSetIsStatsCompliant() && !stat.isIsStatsCompliant()) continue;
+ map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj());
+ }
}
result = new PartitionsStatsResult(map);
} finally {
@@ -5737,79 +5792,73 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@Override
public boolean update_table_column_statistics(ColumnStatistics colStats) throws TException {
- String catName;
- String dbName;
- String tableName;
- String colName;
- ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
- catName = statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf);
- dbName = statsDesc.getDbName().toLowerCase();
- tableName = statsDesc.getTableName().toLowerCase();
-
- statsDesc.setCatName(catName);
- statsDesc.setDbName(dbName);
- statsDesc.setTableName(tableName);
- long time = System.currentTimeMillis() / 1000;
- statsDesc.setLastAnalyzed(time);
-
- List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ // Deprecated API, won't work for transactional tables
+ return updateTableColumnStatsInternal(colStats, -1, null, -1);
+ }
- startFunction("write_column_statistics", ": table=" +
- TableName.getQualified(catName, dbName, tableName));
- for (ColumnStatisticsObj statsObj:statsObjs) {
- colName = statsObj.getColName().toLowerCase();
- statsObj.setColName(colName);
- statsObj.setColType(statsObj.getColType().toLowerCase());
+ @Override
+ public SetPartitionsStatsResponse update_table_column_statistics_req(
+ SetPartitionsStatsRequest req) throws NoSuchObjectException,
+ InvalidObjectException, MetaException, InvalidInputException,
+ TException {
+ if (req.getColStatsSize() != 1) {
+ throw new InvalidInputException("Only one stats object expected");
+ }
+ if (req.isNeedMerge()) {
+ throw new InvalidInputException("Merge is not supported for non-aggregate stats");
}
+ ColumnStatistics colStats = req.getColStatsIterator().next();
+ boolean ret = updateTableColumnStatsInternal(colStats,
+ req.getTxnId(), req.getValidWriteIdList(), req.getWriteId());
+ return new SetPartitionsStatsResponse(ret);
+ }
- colStats.setStatsDesc(statsDesc);
- colStats.setStatsObj(statsObjs);
+ private boolean updateTableColumnStatsInternal(ColumnStatistics colStats,
+ long txnId, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ normalizeColStatsInput(colStats);
- boolean ret = false;
+ startFunction("write_column_statistics", ": table=" + TableName.getQualified(
+ colStats.getStatsDesc().getCatName(), colStats.getStatsDesc().getDbName(),
+ colStats.getStatsDesc().getTableName()));
+ boolean ret = false;
try {
- ret = getMS().updateTableColumnStatistics(colStats);
- return ret;
+ ret = getMS().updateTableColumnStatistics(colStats, txnId, validWriteIds, writeId);
} finally {
- endFunction("write_column_statistics", ret != false, null, tableName);
+ endFunction("write_column_statistics", ret != false, null,
+ colStats.getStatsDesc().getTableName());
}
+ return ret;
}
- private boolean updatePartitonColStats(Table tbl, ColumnStatistics colStats)
- throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
- String catName;
- String dbName;
- String tableName;
- String partName;
- String colName;
-
+ private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException {
+ // TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless.
ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
- catName = statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf);
- dbName = statsDesc.getDbName().toLowerCase();
- tableName = statsDesc.getTableName().toLowerCase();
- partName = lowerCaseConvertPartName(statsDesc.getPartName());
-
- statsDesc.setCatName(catName);
- statsDesc.setDbName(dbName);
- statsDesc.setTableName(tableName);
- statsDesc.setPartName(partName);
-
+ statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf));
+ statsDesc.setDbName(statsDesc.getDbName().toLowerCase());
+ statsDesc.setTableName(statsDesc.getTableName().toLowerCase());
+ statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName()));
long time = System.currentTimeMillis() / 1000;
statsDesc.setLastAnalyzed(time);
- List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
-
- startFunction("write_partition_column_statistics",
- ": db=" + dbName + " table=" + tableName
- + " part=" + partName);
- for (ColumnStatisticsObj statsObj:statsObjs) {
- colName = statsObj.getColName().toLowerCase();
- statsObj.setColName(colName);
+ for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) {
+ statsObj.setColName(statsObj.getColName().toLowerCase());
statsObj.setColType(statsObj.getColType().toLowerCase());
}
-
colStats.setStatsDesc(statsDesc);
- colStats.setStatsObj(statsObjs);
+ colStats.setStatsObj(colStats.getStatsObj());
+ }
+
+ private boolean updatePartitonColStatsInternal(Table tbl, ColumnStatistics colStats,
+ long txnId, String validWriteIds, long writeId)
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ normalizeColStatsInput(colStats);
+
+ ColumnStatisticsDesc csd = colStats.getStatsDesc();
+ String catName = csd.getCatName(), dbName = csd.getDbName(), tableName = csd.getTableName();
+ startFunction("write_partition_column_statistics", ": db=" + dbName + " table=" + tableName
+ + " part=" + csd.getPartName());
boolean ret = false;
@@ -5817,9 +5866,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (tbl == null) {
tbl = getTable(catName, dbName, tableName);
}
- List<String> partVals = getPartValsFromName(tbl, partName);
- ret = getMS().updatePartitionColumnStatistics(colStats, partVals);
- return ret;
+ List<String> partVals = getPartValsFromName(tbl, csd.getPartName());
+ return getMS().updatePartitionColumnStatistics(
+ colStats, partVals, txnId, validWriteIds, writeId);
} finally {
endFunction("write_partition_column_statistics", ret != false, null, tableName);
}
@@ -5827,7 +5876,26 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@Override
public boolean update_partition_column_statistics(ColumnStatistics colStats) throws TException {
- return updatePartitonColStats(null, colStats);
+ // Deprecated API.
+ return updatePartitonColStatsInternal(null, colStats, -1, null, -1);
+ }
+
+
+ @Override
+ public SetPartitionsStatsResponse update_partition_column_statistics_req(
+ SetPartitionsStatsRequest req) throws NoSuchObjectException,
+ InvalidObjectException, MetaException, InvalidInputException,
+ TException {
+ if (req.getColStatsSize() != 1) {
+ throw new InvalidInputException("Only one stats object expected");
+ }
+ if (req.isNeedMerge()) {
+ throw new InvalidInputException("Merge is not supported for non-aggregate stats");
+ }
+ ColumnStatistics colStats = req.getColStatsIterator().next();
+ boolean ret = updatePartitonColStatsInternal(null, colStats,
+ req.getTxnId(), req.getValidWriteIdList(), req.getWriteId());
+ return new SetPartitionsStatsResponse(ret);
}
@Override
@@ -5843,13 +5911,25 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("delete_column_statistics_by_partition",": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) +
" partition=" + convertedPartName + " column=" + colName);
- boolean ret = false;
+ boolean ret = false, committed = false;
+ getMS().openTransaction();
try {
List<String> partVals = getPartValsFromName(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, convertedPartName);
+ Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
+ // This API looks unused; if it were used we'd need to update stats state and write ID.
+ // We cannot just randomly nuke some txn stats.
+ if (TxnUtils.isTransactionalTable(table)) {
+ throw new MetaException("Cannot delete stats via this API for a transactional table");
+ }
+
ret = getMS().deletePartitionColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
convertedPartName, partVals, colName);
+ committed = getMS().commitTransaction();
} finally {
+ if (!committed) {
+ getMS().rollbackTransaction();
+ }
endFunction("delete_column_statistics_by_partition", ret != false, null, tableName);
}
return ret;
@@ -5870,10 +5950,23 @@ public class HiveMetaStore extends ThriftHiveMetastore {
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) + " column=" +
colName);
- boolean ret = false;
+
+ boolean ret = false, committed = false;
+ getMS().openTransaction();
try {
+ Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
+ // This API looks unused; if it were used we'd need to update stats state and write ID.
+ // We cannot just randomly nuke some txn stats.
+ if (TxnUtils.isTransactionalTable(table)) {
+ throw new MetaException("Cannot delete stats via this API for a transactional table");
+ }
+
ret = getMS().deleteTableColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colName);
+ committed = getMS().commitTransaction();
} finally {
+ if (!committed) {
+ getMS().rollbackTransaction();
+ }
endFunction("delete_column_statistics_by_table", ret != false, null, tableName);
}
return ret;
@@ -7489,31 +7582,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw new MetaException(
"Expecting only 1 ColumnStatistics for table's column stats, but find "
+ request.getColStatsSize());
+ }
+ if (request.isSetNeedMerge() && request.isNeedMerge()) {
+ return updateTableColumnStatsWithMerge(catName, dbName, tableName, colNames, request);
} else {
- if (request.isSetNeedMerge() && request.isNeedMerge()) {
- // one single call to get all column stats
- ColumnStatistics csOld =
- getMS().getTableColumnStatistics(
- catName, dbName, tableName, colNames,
- request.getTxnId(), request.getValidWriteIdList());
- Table t = getTable(catName, dbName, tableName);
- // we first use t.getParameters() to prune the stats
- MetaStoreUtils.getMergableCols(firstColStats, t.getParameters());
- // we merge those that can be merged
- if (csOld != null && csOld.getStatsObjSize() != 0
- && !firstColStats.getStatsObj().isEmpty()) {
- MetaStoreUtils.mergeColStats(firstColStats, csOld);
- }
- if (!firstColStats.getStatsObj().isEmpty()) {
- return update_table_column_statistics(firstColStats);
- } else {
- LOG.debug("All the column stats are not accurate to merge.");
- return true;
- }
- } else {
- // This is the overwrite case, we do not care about the accuracy.
- return update_table_column_statistics(firstColStats);
- }
+ // This is the overwrite case, we do not care about the accuracy.
+ return updateTableColumnStatsInternal(firstColStats, request.getTxnId(),
+ request.getValidWriteIdList(), request.getWriteId());
}
} else {
// partition level column stats merging
@@ -7529,54 +7604,151 @@ public class HiveMetaStore extends ThriftHiveMetastore {
newStatsMap.put(partName, csNew);
}
- Map<String, ColumnStatistics> oldStatsMap = new HashMap<>();
- Map<String, Partition> mapToPart = new HashMap<>();
if (request.isSetNeedMerge() && request.isNeedMerge()) {
- // a single call to get all column stats for all partitions
- List<String> partitionNames = new ArrayList<>();
- partitionNames.addAll(newStatsMap.keySet());
- List<ColumnStatistics> csOlds =
- getMS().getPartitionColumnStatistics(
- catName, dbName, tableName, partitionNames, colNames,
- request.getTxnId(), request.getValidWriteIdList());
- if (newStatsMap.values().size() != csOlds.size()) {
- // some of the partitions miss stats.
- LOG.debug("Some of the partitions miss stats.");
- }
- for (ColumnStatistics csOld : csOlds) {
- oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld);
+ ret = updatePartColumnStatsWithMerge(catName, dbName, tableName,
+ colNames, newStatsMap, request);
+ } else { // No merge.
+ Table t = getTable(catName, dbName, tableName);
+ for (Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
+ // We don't short-circuit on errors here anymore. That can leave acid stats invalid.
+ ret = updatePartitonColStatsInternal(t, entry.getValue(), request.getTxnId(),
+ request.getValidWriteIdList(), request.getWriteId()) && ret;
}
+ }
+ }
+ return ret;
+ }
- // another single call to get all the partition objects
- partitions = getMS().getPartitionsByNames(catName, dbName, tableName, partitionNames);
- for (int index = 0; index < partitionNames.size(); index++) {
- mapToPart.put(partitionNames.get(index), partitions.get(index));
- }
+ private boolean updatePartColumnStatsWithMerge(String catName, String dbName, String tableName,
+ List<String> colNames, Map<String, ColumnStatistics> newStatsMap, SetPartitionsStatsRequest request)
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ RawStore ms = getMS();
+ ms.openTransaction();
+ boolean isCommitted = false, result = false;
+ try {
+ // a single call to get all column stats for all partitions
+ List<String> partitionNames = new ArrayList<>();
+ partitionNames.addAll(newStatsMap.keySet());
+ List<ColumnStatistics> csOlds = ms.getPartitionColumnStatistics(catName, dbName, tableName,
+ partitionNames, colNames, request.getTxnId(), request.getValidWriteIdList());
+ if (newStatsMap.values().size() != csOlds.size()) {
+ // some of the partitions miss stats.
+ LOG.debug("Some of the partitions miss stats.");
}
- Table t = getTable(catName, dbName, tableName,
- request.getTxnId(), request.getValidWriteIdList());
+ Map<String, ColumnStatistics> oldStatsMap = new HashMap<>();
+ for (ColumnStatistics csOld : csOlds) {
+ oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld);
+ }
+
+ // another single call to get all the partition objects
+ List<Partition> partitions = ms.getPartitionsByNames(catName, dbName, tableName, partitionNames);
+ Map<String, Partition> mapToPart = new HashMap<>();
+ for (int index = 0; index < partitionNames.size(); index++) {
+ mapToPart.put(partitionNames.get(index), partitions.get(index));
+ }
+
+ Table t = getTable(catName, dbName, tableName);
for (Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
ColumnStatistics csNew = entry.getValue();
ColumnStatistics csOld = oldStatsMap.get(entry.getKey());
- if (request.isSetNeedMerge() && request.isNeedMerge()) {
+ boolean isInvalidTxnStats = csOld != null
+ && csOld.isSetIsStatsCompliant() && !csOld.isIsStatsCompliant();
+ Partition part = mapToPart.get(entry.getKey());
+ if (isInvalidTxnStats) {
+ // No columns can be merged; a shortcut for getMergableCols.
+ csNew.setStatsObj(Lists.newArrayList());
+ } else {
// we first use getParameters() to prune the stats
- MetaStoreUtils.getMergableCols(csNew, mapToPart.get(entry.getKey()).getParameters());
+ MetaStoreUtils.getMergableCols(csNew, part.getParameters());
// we merge those that can be merged
if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) {
MetaStoreUtils.mergeColStats(csNew, csOld);
}
- if (!csNew.getStatsObj().isEmpty()) {
- ret = ret && updatePartitonColStats(t, csNew);
- } else {
- LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName()
- + " are not accurate to merge.");
- }
+ }
+
+ if (!csNew.getStatsObj().isEmpty()) {
+ // We don't short-circuit on errors here anymore. That can leave acid stats invalid.
+ result = updatePartitonColStatsInternal(t, csNew, request.getTxnId(),
+ request.getValidWriteIdList(), request.getWriteId()) && result;
+ } else if (isInvalidTxnStats) {
+ // For now because the stats state is such as it is, we will invalidate everything.
+ // Overall the sematics here are not clear - we could invalide only some columns, but does
+ // that make any physical sense? Could query affect some columns but not others?
+ part.setWriteId(request.getWriteId());
+ StatsSetupConst.clearColumnStatsState(part.getParameters());
+ StatsSetupConst.setBasicStatsState(part.getParameters(), StatsSetupConst.FALSE);
+ ms.alterPartition(catName, dbName, tableName, part.getValues(), part,
+ request.getTxnId(), request.getValidWriteIdList());
+ result = false;
} else {
- ret = ret && updatePartitonColStats(t, csNew);
+ // TODO: why doesn't the original call for non acid tables invalidate the stats?
+ LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName()
+ + " are not accurate to merge.");
}
}
+ ms.commitTransaction();
+ isCommitted = true;
+ } finally {
+ if (!isCommitted) {
+ ms.rollbackTransaction();
+ }
}
- return ret;
+ return result;
+ }
+
+
+ private boolean updateTableColumnStatsWithMerge(String catName, String dbName, String tableName,
+ List<String> colNames, SetPartitionsStatsRequest request) throws MetaException,
+ NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ ColumnStatistics firstColStats = request.getColStats().get(0);
+ RawStore ms = getMS();
+ ms.openTransaction();
+ boolean isCommitted = false, result = false;
+ try {
+ ColumnStatistics csOld = ms.getTableColumnStatistics(catName, dbName, tableName, colNames,
+ request.getTxnId(), request.getValidWriteIdList());
+ // we first use the valid stats list to prune the stats
+ boolean isInvalidTxnStats = csOld != null
+ && csOld.isSetIsStatsCompliant() && !csOld.isIsStatsCompliant();
+ if (isInvalidTxnStats) {
+ // No columns can be merged; a shortcut for getMergableCols.
+ firstColStats.setStatsObj(Lists.newArrayList());
+ } else {
+ Table t = getTable(catName, dbName, tableName);
+ MetaStoreUtils.getMergableCols(firstColStats, t.getParameters());
+
+ // we merge those that can be merged
+ if (csOld != null && csOld.getStatsObjSize() != 0 && !firstColStats.getStatsObj().isEmpty()) {
+ MetaStoreUtils.mergeColStats(firstColStats, csOld);
+ }
+ }
+
+ if (!firstColStats.getStatsObj().isEmpty()) {
+ result = updateTableColumnStatsInternal(firstColStats, request.getTxnId(),
+ request.getValidWriteIdList(), request.getWriteId());
+ } else if (isInvalidTxnStats) {
+ // For now because the stats state is such as it is, we will invalidate everything.
+ // Overall the sematics here are not clear - we could invalide only some columns, but does
+ // that make any physical sense? Could query affect some columns but not others?
+ Table t = getTable(catName, dbName, tableName);
+ t.setWriteId(request.getWriteId());
+ StatsSetupConst.clearColumnStatsState(t.getParameters());
+ StatsSetupConst.setBasicStatsState(t.getParameters(), StatsSetupConst.FALSE);
+ ms.alterTable(catName, dbName, tableName, t, request.getTxnId(), request.getValidWriteIdList());
+ } else {
+ // TODO: why doesn't the original call for non acid tables invalidate the stats?
+ LOG.debug("All the column stats are not accurate to merge.");
+ result = true;
+ }
+
+ ms.commitTransaction();
+ isCommitted = true;
+ } finally {
+ if (!isCommitted) {
+ ms.rollbackTransaction();
+ }
+ }
+ return result;
}
private Table getTable(String catName, String dbName, String tableName)
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index cc417ea..38327d7 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -130,7 +130,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
//copied from ErrorMsg.java
private static final String REPL_EVENTS_MISSING_IN_METASTORE = "Notification events are missing in the meta store.";
-
+
static final protected Logger LOG = LoggerFactory.getLogger(HiveMetaStoreClient.class);
public HiveMetaStoreClient(Configuration conf) throws MetaException {
@@ -404,15 +404,36 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
if (hook != null) {
hook.preAlterTable(new_tbl, envContext);
}
- client.alter_table_with_environment_context(prependCatalogToDbName(dbname, conf),
- tbl_name, new_tbl, envContext);
+ AlterTableRequest req = new AlterTableRequest(dbname, tbl_name, new_tbl);
+ req.setCatName(MetaStoreUtils.getDefaultCatalog(conf));
+ req.setEnvironmentContext(envContext);
+ client.alter_table_req(req);
}
@Override
public void alter_table(String catName, String dbName, String tblName, Table newTable,
EnvironmentContext envContext) throws TException {
- client.alter_table_with_environment_context(prependCatalogToDbName(catName,
- dbName, conf), tblName, newTable, envContext);
+ // This never used to call the hook. Why? There's overload madness in metastore...
+ AlterTableRequest req = new AlterTableRequest(dbName, tblName, newTable);
+ req.setCatName(catName);
+ req.setEnvironmentContext(envContext);
+ client.alter_table_req(req);
+ }
+
+ @Override
+ public void alter_table(String catName, String dbName, String tbl_name, Table new_tbl,
+ EnvironmentContext envContext, long txnId, String validWriteIds)
+ throws InvalidOperationException, MetaException, TException {
+ HiveMetaHook hook = getHook(new_tbl);
+ if (hook != null) {
+ hook.preAlterTable(new_tbl, envContext);
+ }
+ AlterTableRequest req = new AlterTableRequest(dbName, tbl_name, new_tbl);
+ req.setCatName(catName);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(validWriteIds);
+ req.setEnvironmentContext(envContext);
+ client.alter_table_req(req);
}
@Override
@@ -1339,14 +1360,33 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
+ public void truncateTable(String dbName, String tableName, List<String> partNames,
+ long txnId, String validWriteIds, long writeId) throws TException {
+ truncateTableInternal(getDefaultCatalog(conf),
+ dbName, tableName, partNames, txnId, validWriteIds, writeId);
+ }
+
+ @Override
public void truncateTable(String dbName, String tableName, List<String> partNames) throws TException {
- truncateTable(getDefaultCatalog(conf), dbName, tableName, partNames);
+ truncateTableInternal(getDefaultCatalog(conf), dbName, tableName, partNames, -1, null, -1);
}
@Override
public void truncateTable(String catName, String dbName, String tableName, List<String> partNames)
throws TException {
- client.truncate_table(prependCatalogToDbName(catName, dbName, conf), tableName, partNames);
+ truncateTableInternal(catName, dbName, tableName, partNames, -1, null, -1);
+ }
+
+ private void truncateTableInternal(String catName, String dbName, String tableName,
+ List<String> partNames, long txnId, String validWriteIds, long writeId)
+ throws MetaException, TException {
+ TruncateTableRequest req = new TruncateTableRequest(
+ prependCatalogToDbName(catName, dbName, conf), tableName);
+ req.setPartNames(partNames);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(validWriteIds);
+ req.setWriteId(writeId);
+ client.truncate_table_req(req);
}
/**
@@ -1870,7 +1910,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
- public void alter_partition(String dbName, String tblName, Partition newPart, EnvironmentContext environmentContext)
+ public void alter_partition(String dbName, String tblName, Partition newPart,
+ EnvironmentContext environmentContext)
throws InvalidOperationException, MetaException, TException {
alter_partition(getDefaultCatalog(conf), dbName, tblName, newPart, environmentContext);
}
@@ -1878,11 +1919,26 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void alter_partition(String catName, String dbName, String tblName, Partition newPart,
EnvironmentContext environmentContext) throws TException {
- client.alter_partition_with_environment_context(prependCatalogToDbName(catName, dbName, conf), tblName,
- newPart, environmentContext);
+ AlterPartitionsRequest req = new AlterPartitionsRequest(dbName, tblName, Lists.newArrayList(newPart));
+ req.setCatName(catName);
+ req.setEnvironmentContext(environmentContext);
+ client.alter_partitions_req(req);
}
@Override
+ public void alter_partition(String dbName, String tblName, Partition newPart,
+ EnvironmentContext environmentContext, long txnId, String writeIdList)
+ throws InvalidOperationException, MetaException, TException {
+ AlterPartitionsRequest req = new AlterPartitionsRequest(
+ dbName, tblName, Lists.newArrayList(newPart));
+ req.setEnvironmentContext(environmentContext);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(writeIdList);
+ client.alter_partitions_req(req);
+ }
+
+ @Deprecated
+ @Override
public void alter_partitions(String dbName, String tblName, List<Partition> newParts)
throws TException {
alter_partitions(
@@ -1901,8 +1957,6 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
EnvironmentContext environmentContext,
long txnId, String writeIdList, long writeId)
throws InvalidOperationException, MetaException, TException {
- //client.alter_partition_with_environment_context(getDefaultCatalog(conf),
- // dbName, tblName, newParts, environmentContext);
alter_partitions(getDefaultCatalog(conf),
dbName, tblName, newParts, environmentContext, txnId, writeIdList, writeId);
@@ -1914,14 +1968,15 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
EnvironmentContext environmentContext,
long txnId, String writeIdList, long writeId) throws TException {
AlterPartitionsRequest req = new AlterPartitionsRequest();
- req.setDbName(prependCatalogToDbName(catName, dbName, conf));
+ req.setCatName(catName);
+ req.setDbName(dbName);
req.setTableName(tblName);
req.setPartitions(newParts);
req.setEnvironmentContext(environmentContext);
req.setTxnId(txnId);
req.setValidWriteIdList(writeIdList);
req.setWriteId(writeId);
- client.alter_partitions_with_environment_context_req(req);
+ client.alter_partitions_req(req);
}
@Override
@@ -2005,7 +2060,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
if (!statsObj.getStatsDesc().isSetCatName()) {
statsObj.getStatsDesc().setCatName(getDefaultCatalog(conf));
}
- return client.update_table_column_statistics(statsObj);
+ // Note: currently this method doesn't set txn properties and thus won't work on txn tables.
+ SetPartitionsStatsRequest req = new SetPartitionsStatsRequest();
+ req.addToColStats(statsObj);
+ req.setNeedMerge(false);
+ return client.update_table_column_statistics_req(req).isResult();
}
@Override
@@ -2013,7 +2072,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
if (!statsObj.getStatsDesc().isSetCatName()) {
statsObj.getStatsDesc().setCatName(getDefaultCatalog(conf));
}
- return client.update_partition_column_statistics(statsObj);
+ // Note: currently this method doesn't set txn properties and thus won't work on txn tables.
+ SetPartitionsStatsRequest req = new SetPartitionsStatsRequest();
+ req.addToColStats(statsObj);
+ req.setNeedMerge(false);
+ return client.update_partition_column_statistics_req(req).isResult();
}
@Override
@@ -3431,5 +3494,4 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
req.setMaxCreateTime(maxCreateTime);
return client.get_runtime_stats(req);
}
-
}