You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/03/28 15:03:49 UTC
svn commit: r1669775 [10/35] - in /hive/branches/spark: ./
ant/src/org/apache/hadoop/hive/ant/ beeline/src/java/org/apache/hive/beeline/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/common/type/ common/src/java/...
Modified: hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Sat Mar 28 14:03:43 2015
@@ -8160,6 +8160,118 @@ class ShowCompactResponse:
def __ne__(self, other):
return not (self == other)
+class AddDynamicPartitions:
+ """
+ Attributes:
+ - txnid
+ - dbname
+ - tablename
+ - partitionnames
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.I64, 'txnid', None, None, ), # 1
+ (2, TType.STRING, 'dbname', None, None, ), # 2
+ (3, TType.STRING, 'tablename', None, None, ), # 3
+ (4, TType.LIST, 'partitionnames', (TType.STRING,None), None, ), # 4
+ )
+
+ def __init__(self, txnid=None, dbname=None, tablename=None, partitionnames=None,):
+ self.txnid = txnid
+ self.dbname = dbname
+ self.tablename = tablename
+ self.partitionnames = partitionnames
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.I64:
+ self.txnid = iprot.readI64();
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.dbname = iprot.readString();
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.tablename = iprot.readString();
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.LIST:
+ self.partitionnames = []
+ (_etype437, _size434) = iprot.readListBegin()
+ for _i438 in xrange(_size434):
+ _elem439 = iprot.readString();
+ self.partitionnames.append(_elem439)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('AddDynamicPartitions')
+ if self.txnid is not None:
+ oprot.writeFieldBegin('txnid', TType.I64, 1)
+ oprot.writeI64(self.txnid)
+ oprot.writeFieldEnd()
+ if self.dbname is not None:
+ oprot.writeFieldBegin('dbname', TType.STRING, 2)
+ oprot.writeString(self.dbname)
+ oprot.writeFieldEnd()
+ if self.tablename is not None:
+ oprot.writeFieldBegin('tablename', TType.STRING, 3)
+ oprot.writeString(self.tablename)
+ oprot.writeFieldEnd()
+ if self.partitionnames is not None:
+ oprot.writeFieldBegin('partitionnames', TType.LIST, 4)
+ oprot.writeListBegin(TType.STRING, len(self.partitionnames))
+ for iter440 in self.partitionnames:
+ oprot.writeString(iter440)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.txnid is None:
+ raise TProtocol.TProtocolException(message='Required field txnid is unset!')
+ if self.dbname is None:
+ raise TProtocol.TProtocolException(message='Required field dbname is unset!')
+ if self.tablename is None:
+ raise TProtocol.TProtocolException(message='Required field tablename is unset!')
+ if self.partitionnames is None:
+ raise TProtocol.TProtocolException(message='Required field partitionnames is unset!')
+ return
+
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
class NotificationEventRequest:
"""
Attributes:
@@ -8388,11 +8500,11 @@ class NotificationEventResponse:
if fid == 1:
if ftype == TType.LIST:
self.events = []
- (_etype437, _size434) = iprot.readListBegin()
- for _i438 in xrange(_size434):
- _elem439 = NotificationEvent()
- _elem439.read(iprot)
- self.events.append(_elem439)
+ (_etype444, _size441) = iprot.readListBegin()
+ for _i445 in xrange(_size441):
+ _elem446 = NotificationEvent()
+ _elem446.read(iprot)
+ self.events.append(_elem446)
iprot.readListEnd()
else:
iprot.skip(ftype)
@@ -8409,8 +8521,8 @@ class NotificationEventResponse:
if self.events is not None:
oprot.writeFieldBegin('events', TType.LIST, 1)
oprot.writeListBegin(TType.STRUCT, len(self.events))
- for iter440 in self.events:
- iter440.write(oprot)
+ for iter447 in self.events:
+ iter447.write(oprot)
oprot.writeListEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
@@ -8521,10 +8633,10 @@ class InsertEventRequestData:
if fid == 1:
if ftype == TType.LIST:
self.filesAdded = []
- (_etype444, _size441) = iprot.readListBegin()
- for _i445 in xrange(_size441):
- _elem446 = iprot.readString();
- self.filesAdded.append(_elem446)
+ (_etype451, _size448) = iprot.readListBegin()
+ for _i452 in xrange(_size448):
+ _elem453 = iprot.readString();
+ self.filesAdded.append(_elem453)
iprot.readListEnd()
else:
iprot.skip(ftype)
@@ -8541,8 +8653,8 @@ class InsertEventRequestData:
if self.filesAdded is not None:
oprot.writeFieldBegin('filesAdded', TType.LIST, 1)
oprot.writeListBegin(TType.STRING, len(self.filesAdded))
- for iter447 in self.filesAdded:
- oprot.writeString(iter447)
+ for iter454 in self.filesAdded:
+ oprot.writeString(iter454)
oprot.writeListEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
@@ -8685,10 +8797,10 @@ class FireEventRequest:
elif fid == 5:
if ftype == TType.LIST:
self.partitionVals = []
- (_etype451, _size448) = iprot.readListBegin()
- for _i452 in xrange(_size448):
- _elem453 = iprot.readString();
- self.partitionVals.append(_elem453)
+ (_etype458, _size455) = iprot.readListBegin()
+ for _i459 in xrange(_size455):
+ _elem460 = iprot.readString();
+ self.partitionVals.append(_elem460)
iprot.readListEnd()
else:
iprot.skip(ftype)
@@ -8721,8 +8833,8 @@ class FireEventRequest:
if self.partitionVals is not None:
oprot.writeFieldBegin('partitionVals', TType.LIST, 5)
oprot.writeListBegin(TType.STRING, len(self.partitionVals))
- for iter454 in self.partitionVals:
- oprot.writeString(iter454)
+ for iter461 in self.partitionVals:
+ oprot.writeString(iter461)
oprot.writeListEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
Modified: hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Sat Mar 28 14:03:43 2015
@@ -1994,6 +1994,32 @@ class ShowCompactResponse
::Thrift::Struct.generate_accessors self
end
+class AddDynamicPartitions
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXNID = 1
+ DBNAME = 2
+ TABLENAME = 3
+ PARTITIONNAMES = 4
+
+ FIELDS = {
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'},
+ PARTITIONNAMES => {:type => ::Thrift::Types::LIST, :name => 'partitionnames', :element => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txnid is unset!') unless @txnid
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablename is unset!') unless @tablename
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partitionnames is unset!') unless @partitionnames
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
class NotificationEventRequest
include ::Thrift::Struct, ::Thrift::Struct_Union
LASTEVENT = 1
Modified: hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb (original)
+++ hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb Sat Mar 28 14:03:43 2015
@@ -228,6 +228,24 @@ module ThriftHiveMetastore
raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_fields failed: unknown result')
end
+ def get_fields_with_environment_context(db_name, table_name, environment_context)
+ send_get_fields_with_environment_context(db_name, table_name, environment_context)
+ return recv_get_fields_with_environment_context()
+ end
+
+ def send_get_fields_with_environment_context(db_name, table_name, environment_context)
+ send_message('get_fields_with_environment_context', Get_fields_with_environment_context_args, :db_name => db_name, :table_name => table_name, :environment_context => environment_context)
+ end
+
+ def recv_get_fields_with_environment_context()
+ result = receive_message(Get_fields_with_environment_context_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ raise result.o3 unless result.o3.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_fields_with_environment_context failed: unknown result')
+ end
+
def get_schema(db_name, table_name)
send_get_schema(db_name, table_name)
return recv_get_schema()
@@ -246,6 +264,24 @@ module ThriftHiveMetastore
raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_schema failed: unknown result')
end
+ def get_schema_with_environment_context(db_name, table_name, environment_context)
+ send_get_schema_with_environment_context(db_name, table_name, environment_context)
+ return recv_get_schema_with_environment_context()
+ end
+
+ def send_get_schema_with_environment_context(db_name, table_name, environment_context)
+ send_message('get_schema_with_environment_context', Get_schema_with_environment_context_args, :db_name => db_name, :table_name => table_name, :environment_context => environment_context)
+ end
+
+ def recv_get_schema_with_environment_context()
+ result = receive_message(Get_schema_with_environment_context_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ raise result.o3 unless result.o3.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_schema_with_environment_context failed: unknown result')
+ end
+
def create_table(tbl)
send_create_table(tbl)
recv_create_table()
@@ -1977,6 +2013,22 @@ module ThriftHiveMetastore
raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'show_compact failed: unknown result')
end
+ def add_dynamic_partitions(rqst)
+ send_add_dynamic_partitions(rqst)
+ recv_add_dynamic_partitions()
+ end
+
+ def send_add_dynamic_partitions(rqst)
+ send_message('add_dynamic_partitions', Add_dynamic_partitions_args, :rqst => rqst)
+ end
+
+ def recv_add_dynamic_partitions()
+ result = receive_message(Add_dynamic_partitions_result)
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ return
+ end
+
def get_next_notification(rqst)
send_get_next_notification(rqst)
return recv_get_next_notification()
@@ -2194,6 +2246,21 @@ module ThriftHiveMetastore
write_result(result, oprot, 'get_fields', seqid)
end
+ def process_get_fields_with_environment_context(seqid, iprot, oprot)
+ args = read_args(iprot, Get_fields_with_environment_context_args)
+ result = Get_fields_with_environment_context_result.new()
+ begin
+ result.success = @handler.get_fields_with_environment_context(args.db_name, args.table_name, args.environment_context)
+ rescue ::MetaException => o1
+ result.o1 = o1
+ rescue ::UnknownTableException => o2
+ result.o2 = o2
+ rescue ::UnknownDBException => o3
+ result.o3 = o3
+ end
+ write_result(result, oprot, 'get_fields_with_environment_context', seqid)
+ end
+
def process_get_schema(seqid, iprot, oprot)
args = read_args(iprot, Get_schema_args)
result = Get_schema_result.new()
@@ -2209,6 +2276,21 @@ module ThriftHiveMetastore
write_result(result, oprot, 'get_schema', seqid)
end
+ def process_get_schema_with_environment_context(seqid, iprot, oprot)
+ args = read_args(iprot, Get_schema_with_environment_context_args)
+ result = Get_schema_with_environment_context_result.new()
+ begin
+ result.success = @handler.get_schema_with_environment_context(args.db_name, args.table_name, args.environment_context)
+ rescue ::MetaException => o1
+ result.o1 = o1
+ rescue ::UnknownTableException => o2
+ result.o2 = o2
+ rescue ::UnknownDBException => o3
+ result.o3 = o3
+ end
+ write_result(result, oprot, 'get_schema_with_environment_context', seqid)
+ end
+
def process_create_table(seqid, iprot, oprot)
args = read_args(iprot, Create_table_args)
result = Create_table_result.new()
@@ -3538,6 +3620,19 @@ module ThriftHiveMetastore
write_result(result, oprot, 'show_compact', seqid)
end
+ def process_add_dynamic_partitions(seqid, iprot, oprot)
+ args = read_args(iprot, Add_dynamic_partitions_args)
+ result = Add_dynamic_partitions_result.new()
+ begin
+ @handler.add_dynamic_partitions(args.rqst)
+ rescue ::NoSuchTxnException => o1
+ result.o1 = o1
+ rescue ::TxnAbortedException => o2
+ result.o2 = o2
+ end
+ write_result(result, oprot, 'add_dynamic_partitions', seqid)
+ end
+
def process_get_next_notification(seqid, iprot, oprot)
args = read_args(iprot, Get_next_notification_args)
result = Get_next_notification_result.new()
@@ -4030,6 +4125,48 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Get_fields_with_environment_context_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DB_NAME = 1
+ TABLE_NAME = 2
+ ENVIRONMENT_CONTEXT = 3
+
+ FIELDS = {
+ DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+ TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+ ENVIRONMENT_CONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environment_context', :class => ::EnvironmentContext}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Get_fields_with_environment_context_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+ O2 = 2
+ O3 = 3
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::UnknownTableException},
+ O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::UnknownDBException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Get_schema_args
include ::Thrift::Struct, ::Thrift::Struct_Union
DB_NAME = 1
@@ -4070,6 +4207,48 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Get_schema_with_environment_context_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DB_NAME = 1
+ TABLE_NAME = 2
+ ENVIRONMENT_CONTEXT = 3
+
+ FIELDS = {
+ DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+ TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+ ENVIRONMENT_CONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environment_context', :class => ::EnvironmentContext}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Get_schema_with_environment_context_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+ O2 = 2
+ O3 = 3
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::UnknownTableException},
+ O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::UnknownDBException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Create_table_args
include ::Thrift::Struct, ::Thrift::Struct_Union
TBL = 1
@@ -8040,6 +8219,40 @@ module ThriftHiveMetastore
}
def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Add_dynamic_partitions_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ RQST = 1
+
+ FIELDS = {
+ RQST => {:type => ::Thrift::Types::STRUCT, :name => 'rqst', :class => ::AddDynamicPartitions}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Add_dynamic_partitions_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ O1 = 1
+ O2 = 2
+
+ FIELDS = {
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::NoSuchTxnException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::TxnAbortedException}
+ }
+
+ def struct_fields; FIELDS; end
def validate
end
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java Sat Mar 28 14:03:43 2015
@@ -100,6 +100,21 @@ public class Deadline {
}
/**
+ * Check whether the timer is started.
+ * @return
+ * @throws MetaException
+ */
+ public static boolean isStarted() throws MetaException {
+ Deadline deadline = getCurrentDeadline();
+ if (deadline != null) {
+ return deadline.startTime >= 0;
+ } else {
+ throw newMetaException(new DeadlineException("The threadlocal Deadline is null," +
+ " please register it firstly."));
+ }
+ }
+
+ /**
* start the timer before a method is invoked.
* @param method
*/
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Sat Mar 28 14:03:43 2015
@@ -33,8 +33,11 @@ import org.apache.hadoop.hive.common.Fil
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -42,6 +45,9 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hive.common.util.HiveStringUtils;
+
+import com.google.common.collect.Lists;
/**
* Hive specific implementation of alter
@@ -128,7 +134,10 @@ public class HiveAlterHandler implements
if(MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
List<Partition> parts = msdb.getPartitions(dbname, name, -1);
for (Partition part : parts) {
+ List<FieldSchema> oldCols = part.getSd().getCols();
part.getSd().setCols(newt.getSd().getCols());
+ String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
+ updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part);
msdb.alterPartition(dbname, name, part.getValues(), part);
}
} else {
@@ -209,6 +218,13 @@ public class HiveAlterHandler implements
Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
altps.add(ObjectPair.create(part, part.getSd().getLocation()));
part.getSd().setLocation(newPartLocPath.toString());
+ String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
+ try {
+ //existing partition column stats is no longer valid, remove them
+ msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null);
+ } catch (InvalidInputException iie) {
+ throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
+ }
msdb.alterPartition(dbname, name, part.getValues(), part);
}
}
@@ -219,6 +235,7 @@ public class HiveAlterHandler implements
// alterPartition()
MetaStoreUtils.updateUnpartitionedTableStatsFast(db, newt, wh, false, true);
}
+ updateTableColumnStatsForAlterTable(msdb, oldt, newt);
// now finally call alter table
msdb.alterTable(dbname, name, newt);
// commit the changes
@@ -311,6 +328,7 @@ public class HiveAlterHandler implements
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl)) {
MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true);
}
+ updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
} catch (InvalidObjectException e) {
throw new InvalidOperationException("alter is not possible");
@@ -349,6 +367,15 @@ public class HiveAlterHandler implements
// if the external partition is renamed, the file should not change
if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
new_part.getSd().setLocation(oldPart.getSd().getLocation());
+ String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
+ try {
+ //existing partition column stats is no longer valid, remove
+ msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
+ } catch (NoSuchObjectException nsoe) {
+ //ignore
+ } catch (InvalidInputException iie) {
+ throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
+ }
msdb.alterPartition(dbname, name, part_vals, new_part);
} else {
try {
@@ -395,6 +422,15 @@ public class HiveAlterHandler implements
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl)) {
MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true);
}
+ String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
+ try {
+ //existing partition column stats is no longer valid, remove
+ msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
+ } catch (NoSuchObjectException nsoe) {
+ //ignore
+ } catch (InvalidInputException iie) {
+ throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
+ }
msdb.alterPartition(dbname, name, part_vals, new_part);
}
}
@@ -461,6 +497,7 @@ public class HiveAlterHandler implements
if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl)) {
MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true);
}
+ updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
}
msdb.alterPartitions(dbname, name, partValsList, new_parts);
} catch (InvalidObjectException e) {
@@ -508,4 +545,131 @@ public class HiveAlterHandler implements
return new Path(currentUri.getScheme(), currentUri.getAuthority(),
defaultNewPath.toUri().getPath());
}
+
+ private void updatePartColumnStatsForAlterColumns(RawStore msdb, Partition oldPartition,
+ String oldPartName, List<String> partVals, List<FieldSchema> oldCols, Partition newPart)
+ throws MetaException, InvalidObjectException {
+ String dbName = oldPartition.getDbName();
+ String tableName = oldPartition.getTableName();
+ try {
+ List<String> oldPartNames = Lists.newArrayList(oldPartName);
+ List<String> oldColNames = new ArrayList<String>(oldCols.size());
+ for (FieldSchema oldCol : oldCols) {
+ oldColNames.add(oldCol.getName());
+ }
+ List<FieldSchema> newCols = newPart.getSd().getCols();
+ List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbName, tableName,
+ oldPartNames, oldColNames);
+ assert (partsColStats.size() <= 1);
+ for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop
+ List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ boolean found =false;
+ for (FieldSchema newCol : newCols) {
+ if (statsObj.getColName().equals(newCol.getName())
+ && statsObj.getColType().equals(newCol.getType())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals,
+ statsObj.getColName());
+ }
+ }
+ }
+ } catch (NoSuchObjectException nsoe) {
+ LOG.debug("Could not find db entry." + nsoe);
+ //ignore
+ } catch (InvalidInputException iie) {
+ throw new InvalidObjectException
+ ("Invalid input to update partition column stats in alter table change columns" + iie);
+ }
+ }
+
+ private void updatePartColumnStats(RawStore msdb, String dbName, String tableName,
+ List<String> partVals, Partition newPart) throws MetaException, InvalidObjectException {
+ dbName = HiveStringUtils.normalizeIdentifier(dbName);
+ tableName = HiveStringUtils.normalizeIdentifier(tableName);
+ String newDbName = HiveStringUtils.normalizeIdentifier(newPart.getDbName());
+ String newTableName = HiveStringUtils.normalizeIdentifier(newPart.getTableName());
+
+ Table oldTable = msdb.getTable(dbName, tableName);
+ if (oldTable == null) {
+ return;
+ }
+
+ try {
+ String oldPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), partVals);
+ String newPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), newPart.getValues());
+ if (!dbName.equals(newDbName) || !tableName.equals(newTableName)
+ || !oldPartName.equals(newPartName)) {
+ msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, null);
+ } else {
+ Partition oldPartition = msdb.getPartition(dbName, tableName, partVals);
+ if (oldPartition == null) {
+ return;
+ }
+ if (oldPartition.getSd() != null && newPart.getSd() != null) {
+ List<FieldSchema> oldCols = oldPartition.getSd().getCols();
+ if (!MetaStoreUtils.areSameColumns(oldCols, newPart.getSd().getCols())) {
+ updatePartColumnStatsForAlterColumns(msdb, oldPartition, oldPartName, partVals, oldCols, newPart);
+ }
+ }
+ }
+ } catch (NoSuchObjectException nsoe) {
+ LOG.debug("Could not find db entry." + nsoe);
+ //ignore
+ } catch (InvalidInputException iie) {
+ throw new InvalidObjectException("Invalid input to update partition column stats." + iie);
+ }
+ }
+
+ private void updateTableColumnStatsForAlterTable(RawStore msdb, Table oldTable, Table newTable)
+ throws MetaException, InvalidObjectException {
+ String dbName = oldTable.getDbName();
+ String tableName = oldTable.getTableName();
+ String newDbName = HiveStringUtils.normalizeIdentifier(newTable.getDbName());
+ String newTableName = HiveStringUtils.normalizeIdentifier(newTable.getTableName());
+
+ try {
+ if (!dbName.equals(newDbName) || !tableName.equals(newTableName)) {
+ msdb.deleteTableColumnStatistics(dbName, tableName, null);
+ } else {
+ List<FieldSchema> oldCols = oldTable.getSd().getCols();
+ List<FieldSchema> newCols = newTable.getSd().getCols();
+ if (!MetaStoreUtils.areSameColumns(oldCols, newCols)) {
+ List<String> oldColNames = new ArrayList<String>(oldCols.size());
+ for (FieldSchema oldCol : oldCols) {
+ oldColNames.add(oldCol.getName());
+ }
+
+ ColumnStatistics cs = msdb.getTableColumnStatistics(dbName, tableName, oldColNames);
+ if (cs == null) {
+ return;
+ }
+
+ List<ColumnStatisticsObj> statsObjs = cs.getStatsObj();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ boolean found = false;
+ for (FieldSchema newCol : newCols) {
+ if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
+ && statsObj.getColType().equals(newCol.getType())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
+ }
+ }
+ }
+ }
+ } catch (NoSuchObjectException nsoe) {
+ LOG.debug("Could not find db entry." + nsoe);
+ } catch (InvalidInputException e) {
+ //should not happen since the input were verified before passed in
+ throw new InvalidObjectException("Invalid inputs to update table column stats: " + e);
+ }
+ }
}
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Sat Mar 28 14:03:43 2015
@@ -66,6 +66,7 @@ import org.apache.hadoop.hive.common.met
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
import org.apache.hadoop.hive.metastore.api.AggrStats;
@@ -3445,13 +3446,22 @@ public class HiveMetaStore extends Thrif
@Override
public List<FieldSchema> get_fields(String db, String tableName)
throws MetaException, UnknownTableException, UnknownDBException {
- startFunction("get_fields", ": db=" + db + "tbl=" + tableName);
+ return get_fields_with_environment_context(db, tableName, null);
+ }
+
+ @Override
+ public List<FieldSchema> get_fields_with_environment_context(String db, String tableName,
+ final EnvironmentContext envContext)
+ throws MetaException, UnknownTableException, UnknownDBException {
+ startFunction("get_fields_with_environment_context", ": db=" + db + "tbl=" + tableName);
String[] names = tableName.split("\\.");
String base_table_name = names[0];
Table tbl;
List<FieldSchema> ret = null;
Exception ex = null;
+ ClassLoader orgHiveLoader = null;
+ Configuration curConf = hiveConf;
try {
try {
tbl = get_table_core(db, base_table_name);
@@ -3464,7 +3474,18 @@ public class HiveMetaStore extends Thrif
ret = tbl.getSd().getCols();
} else {
try {
- Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl, false);
+ if (envContext != null) {
+ String addedJars = envContext.getProperties().get("hive.added.jars.path");
+ if (org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) {
+ //for thread safe
+ curConf = getConf();
+ orgHiveLoader = curConf.getClassLoader();
+ ClassLoader loader = MetaStoreUtils.addToClassPath(orgHiveLoader, org.apache.commons.lang.StringUtils.split(addedJars, ","));
+ curConf.setClassLoader(loader);
+ }
+ }
+
+ Deserializer s = MetaStoreUtils.getDeserializer(curConf, tbl, false);
ret = MetaStoreUtils.getFieldsFromDeserializer(tableName, s);
} catch (SerDeException e) {
StringUtils.stringifyException(e);
@@ -3483,7 +3504,10 @@ public class HiveMetaStore extends Thrif
throw newMetaException(e);
}
} finally {
- endFunction("get_fields", ret != null, ex, tableName);
+ if (orgHiveLoader != null) {
+ curConf.setClassLoader(orgHiveLoader);
+ }
+ endFunction("get_fields_with_environment_context", ret != null, ex, tableName);
}
return ret;
@@ -3505,7 +3529,30 @@ public class HiveMetaStore extends Thrif
@Override
public List<FieldSchema> get_schema(String db, String tableName)
throws MetaException, UnknownTableException, UnknownDBException {
- startFunction("get_schema", ": db=" + db + "tbl=" + tableName);
+ return get_schema_with_environment_context(db,tableName, null);
+ }
+
+
+ /**
+ * Return the schema of the table. This function includes partition columns
+ * in addition to the regular columns.
+ *
+ * @param db
+ * Name of the database
+ * @param tableName
+ * Name of the table
+ * @param envContext
+ * Store session based properties
+ * @return List of columns, each column is a FieldSchema structure
+ * @throws MetaException
+ * @throws UnknownTableException
+ * @throws UnknownDBException
+ */
+ @Override
+ public List<FieldSchema> get_schema_with_environment_context(String db, String tableName,
+ final EnvironmentContext envContext)
+ throws MetaException, UnknownTableException, UnknownDBException {
+ startFunction("get_schema_with_environment_context", ": db=" + db + "tbl=" + tableName);
boolean success = false;
Exception ex = null;
try {
@@ -3518,7 +3565,7 @@ public class HiveMetaStore extends Thrif
} catch (NoSuchObjectException e) {
throw new UnknownTableException(e.getMessage());
}
- List<FieldSchema> fieldSchemas = get_fields(db, base_table_name);
+ List<FieldSchema> fieldSchemas = get_fields_with_environment_context(db, base_table_name,envContext);
if (tbl == null || fieldSchemas == null) {
throw new UnknownTableException(tableName + " doesn't exist");
@@ -3545,7 +3592,7 @@ public class HiveMetaStore extends Thrif
throw me;
}
} finally {
- endFunction("get_schema", success, ex, tableName);
+ endFunction("get_schema_with_environment_context", success, ex, tableName);
}
}
@@ -5520,6 +5567,12 @@ public class HiveMetaStore extends Thrif
}
@Override
+ public void add_dynamic_partitions(AddDynamicPartitions rqst)
+ throws NoSuchTxnException, TxnAbortedException, TException {
+ getTxnHandler().addDynamicPartitions(rqst);
+ }
+
+ @Override
public GetPrincipalsInRoleResponse get_principals_in_role(GetPrincipalsInRoleRequest request)
throws MetaException, TException {
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Sat Mar 28 14:03:43 2015
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
import org.apache.hadoop.hive.metastore.api.AggrStats;
@@ -294,6 +295,11 @@ public class HiveMetaStoreClient impleme
}
@Override
+ public void setHiveAddedJars(String addedJars) {
+ HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars);
+ }
+
+ @Override
public void reconnect() throws MetaException {
if (localMetaStore) {
// For direct DB connections we don't yet support reestablishing connections.
@@ -1480,7 +1486,15 @@ public class HiveMetaStoreClient impleme
public List<FieldSchema> getSchema(String db, String tableName)
throws MetaException, TException, UnknownTableException,
UnknownDBException {
- return deepCopyFieldSchemas(client.get_schema(db, tableName));
+ EnvironmentContext envCxt = null;
+ String addedJars = conf.getVar(ConfVars.HIVEADDEDJARS);
+ if(org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) {
+ Map<String, String> props = new HashMap<String, String>();
+ props.put("hive.added.jars.path", addedJars);
+ envCxt = new EnvironmentContext(props);
+ }
+
+ return deepCopyFieldSchemas(client.get_schema_with_environment_context(db, tableName, envCxt));
}
@Override
@@ -1902,6 +1916,12 @@ public class HiveMetaStoreClient impleme
}
@Override
+ public void addDynamicPartitions(long txnId, String dbName, String tableName,
+ List<String> partNames) throws TException {
+ client.add_dynamic_partitions(new AddDynamicPartitions(txnId, dbName, tableName, partNames));
+ }
+
+ @Override
public NotificationEventResponse getNextNotification(long lastEventId, int maxEvents,
NotificationFilter filter) throws TException {
NotificationEventRequest rqst = new NotificationEventRequest(lastEventId);
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Sat Mar 28 14:03:43 2015
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
@@ -95,6 +96,12 @@ public interface IMetaStoreClient {
boolean isCompatibleWith(HiveConf conf);
/**
+ * Set added jars path info to MetaStoreClient.
+ * @param addedJars the hive.added.jars.path. It is qualified paths separated by commas.
+ */
+ void setHiveAddedJars(String addedJars);
+
+ /**
* Tries to reconnect this MetaStoreClient to the MetaStore.
*/
void reconnect() throws MetaException;
@@ -1345,6 +1352,18 @@ public interface IMetaStoreClient {
ShowCompactResponse showCompactions() throws TException;
/**
+ * Send a list of partitions to the metastore to indicate which partitions were loaded
+ * dynamically.
+ * @param txnId id of the transaction
+ * @param dbName database name
+ * @param tableName table name
+ * @param partNames partition name, as constructed by Warehouse.makePartName
+ * @throws TException
+ */
+ void addDynamicPartitions(long txnId, String dbName, String tableName, List<String> partNames)
+ throws TException;
+
+ /**
* A filter provided by the client that determines if a given notification event should be
* returned.
*/
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Sat Mar 28 14:03:43 2015
@@ -25,7 +25,10 @@ import java.lang.reflect.InvocationTarge
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -39,6 +42,7 @@ import java.util.regex.Pattern;
import com.google.common.base.Predicates;
import com.google.common.collect.Maps;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -559,23 +563,26 @@ public class MetaStoreUtils {
}
static boolean isCascadeNeededInAlterTable(Table oldTable, Table newTable) {
+ //currently cascade only supports add/replace columns and
+ //changing column type/position/name/comments
List<FieldSchema> oldCols = oldTable.getSd().getCols();
List<FieldSchema> newCols = newTable.getSd().getCols();
+ return !areSameColumns(oldCols, newCols);
+ }
- //currently cascade only supports add/replace columns and
- //changing column type/position/name/comments
+ static boolean areSameColumns(List<FieldSchema> oldCols, List<FieldSchema> newCols) {
if (oldCols.size() != newCols.size()) {
- return true;
+ return false;
} else {
for (int i = 0; i < oldCols.size(); i++) {
FieldSchema oldCol = oldCols.get(i);
FieldSchema newCol = newCols.get(i);
if(!oldCol.equals(newCol)) {
- return true;
+ return false;
}
}
}
- return false;
+ return true;
}
/**
@@ -1653,4 +1660,52 @@ public class MetaStoreUtils {
}
}
+
+ /**
+ * Create a URL from a string representing a path to a local file.
+ * The path string can be just a path, or can start with file:/, file:///
+ * @param onestr path string
+ * @return
+ */
+ private static URL urlFromPathString(String onestr) {
+ URL oneurl = null;
+ try {
+ if (StringUtils.indexOf(onestr, "file:/") == 0) {
+ oneurl = new URL(onestr);
+ } else {
+ oneurl = new File(onestr).toURL();
+ }
+ } catch (Exception err) {
+ LOG.error("Bad URL " + onestr + ", ignoring path");
+ }
+ return oneurl;
+ }
+
+ /**
+ * Add new elements to the classpath.
+ *
+ * @param newPaths
+ * Array of classpath elements
+ */
+ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception {
+ URLClassLoader loader = (URLClassLoader) cloader;
+ List<URL> curPath = Arrays.asList(loader.getURLs());
+ ArrayList<URL> newPath = new ArrayList<URL>();
+
+ // get a list with the current classpath components
+ for (URL onePath : curPath) {
+ newPath.add(onePath);
+ }
+ curPath = newPath;
+
+ for (String onestr : newPaths) {
+ URL oneurl = urlFromPathString(onestr);
+ if (oneurl != null && !curPath.contains(oneurl)) {
+ curPath.add(oneurl);
+ }
+ }
+
+ return new URLClassLoader(curPath.toArray(new URL[0]), loader);
+ }
+
}
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java Sat Mar 28 14:03:43 2015
@@ -24,6 +24,7 @@ import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.lang.reflect.UndeclaredThrowableException;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.ClassUtils;
import org.apache.hadoop.conf.Configuration;
@@ -93,9 +94,28 @@ public class RawStoreProxy implements In
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
Object ret = null;
+ boolean isTimerStarted = false;
try {
+ try {
+ if (!Deadline.isStarted()) {
+ Deadline.startTimer(method.getName());
+ isTimerStarted = true;
+ }
+ } catch (MetaException e) {
+ // Deadline was not registered yet.
+ long timeout = HiveConf.getTimeVar(hiveConf,
+ HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
+ Deadline.registerIfNot(timeout);
+ Deadline.startTimer(method.getName());
+ isTimerStarted = true;
+ }
+
ret = method.invoke(base, args);
+
+ if (isTimerStarted) {
+ Deadline.stopTimer();
+ }
} catch (UndeclaredThrowableException e) {
throw e.getCause();
} catch (InvocationTargetException e) {
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java Sat Mar 28 14:03:43 2015
@@ -135,7 +135,8 @@ public class RetryingHMSHandler implemen
caughtException = e.getCause();
} else if (e.getCause() instanceof NoSuchObjectException || e.getTargetException().getCause() instanceof NoSuchObjectException) {
String methodName = method.getName();
- if (!methodName.startsWith("get_table") && !methodName.startsWith("get_partition") && !methodName.startsWith("get_function")) {
+ if (!methodName.startsWith("get_database") && !methodName.startsWith("get_table")
+ && !methodName.startsWith("get_partition") && !methodName.startsWith("get_function")) {
LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
}
throw e.getCause();
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java Sat Mar 28 14:03:43 2015
@@ -22,7 +22,7 @@ import org.apache.hadoop.hive.metastore.
/**
* Information on a possible or running compaction.
*/
-public class CompactionInfo {
+public class CompactionInfo implements Comparable<CompactionInfo> {
public long id;
public String dbname;
public String tableName;
@@ -68,4 +68,9 @@ public class CompactionInfo {
public boolean isMajorCompaction() {
return CompactionType.MAJOR == type;
}
+
+ @Override
+ public int compareTo(CompactionInfo o) {
+ return getFullPartitionName().compareTo(o.getFullPartitionName());
+ }
}
Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Sat Mar 28 14:03:43 2015
@@ -783,6 +783,48 @@ public class TxnHandler {
}
}
+ public void addDynamicPartitions(AddDynamicPartitions rqst)
+ throws NoSuchTxnException, TxnAbortedException, MetaException {
+ Connection dbConn = null;
+ Statement stmt = null;
+ try {
+ try {
+ dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+ stmt = dbConn.createStatement();
+ // Heartbeat this first to make sure the transaction is still valid.
+ heartbeatTxn(dbConn, rqst.getTxnid());
+ for (String partName : rqst.getPartitionnames()) {
+ StringBuilder buff = new StringBuilder();
+ buff.append("insert into TXN_COMPONENTS (tc_txnid, tc_database, tc_table, tc_partition) values (");
+ buff.append(rqst.getTxnid());
+ buff.append(", '");
+ buff.append(rqst.getDbname());
+ buff.append("', '");
+ buff.append(rqst.getTablename());
+ buff.append("', '");
+ buff.append(partName);
+ buff.append("')");
+ String s = buff.toString();
+ LOG.debug("Going to execute update <" + s + ">");
+ stmt.executeUpdate(s);
+ }
+ LOG.debug("Going to commit");
+ dbConn.commit();
+ } catch (SQLException e) {
+ LOG.debug("Going to rollback");
+ rollbackDBConn(dbConn);
+ checkRetryable(dbConn, e, "addDynamicPartitions");
+ throw new MetaException("Unable to insert into from transaction database " +
+ StringUtils.stringifyException(e));
+ } finally {
+ closeStmt(stmt);
+ closeDbConn(dbConn);
+ }
+ } catch (RetryException e) {
+ addDynamicPartitions(rqst);
+ }
+ }
+
/**
* For testing only, do not use.
*/
Modified: hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java (original)
+++ hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java Sat Mar 28 14:03:43 2015
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.metastore.txn;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.log4j.Level;
@@ -26,8 +28,11 @@ import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
import static junit.framework.Assert.*;
@@ -38,6 +43,7 @@ public class TestCompactionTxnHandler {
private HiveConf conf = new HiveConf();
private CompactionTxnHandler txnHandler;
+ static final private Log LOG = LogFactory.getLog(TestCompactionTxnHandler.class);
public TestCompactionTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
@@ -417,6 +423,40 @@ public class TestCompactionTxnHandler {
assertEquals(3, txnList.getOpen_txnsSize());
}
+ @Test
+ public void addDynamicPartitions() throws Exception {
+ String dbName = "default";
+ String tableName = "adp_table";
+ OpenTxnsResponse openTxns = txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost"));
+ long txnId = openTxns.getTxn_ids().get(0);
+ // lock a table, as in dynamic partitions
+ LockComponent lc = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, dbName);
+ lc.setTablename(tableName);
+ LockRequest lr = new LockRequest(Arrays.asList(lc), "me", "localhost");
+ lr.setTxnid(txnId);
+ LockResponse lock = txnHandler.lock(new LockRequest(Arrays.asList(lc), "me", "localhost"));
+ assertEquals(LockState.ACQUIRED, lock.getState());
+
+ txnHandler.addDynamicPartitions(new AddDynamicPartitions(txnId, dbName, tableName,
+ Arrays.asList("ds=yesterday", "ds=today")));
+ txnHandler.commitTxn(new CommitTxnRequest(txnId));
+
+ Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(1000);
+ assertEquals(2, potentials.size());
+ SortedSet<CompactionInfo> sorted = new TreeSet<CompactionInfo>(potentials);
+
+ int i = 0;
+ for (CompactionInfo ci : sorted) {
+ assertEquals(dbName, ci.dbname);
+ assertEquals(tableName, ci.tableName);
+ switch (i++) {
+ case 0: assertEquals("ds=today", ci.partName); break;
+ case 1: assertEquals("ds=yesterday", ci.partName); break;
+ default: throw new RuntimeException("What?");
+ }
+ }
+ }
+
@Before
public void setUp() throws Exception {
TxnDbUtil.prepDb();
Modified: hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java (original)
+++ hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java Sat Mar 28 14:03:43 2015
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.metastore
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreThread;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
Modified: hive/branches/spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/pom.xml?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/pom.xml (original)
+++ hive/branches/spark/pom.xml Sat Mar 28 14:03:43 2015
@@ -100,7 +100,7 @@
<antlr.version>3.4</antlr.version>
<avro.version>1.7.5</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
- <calcite.version>1.0.0-incubating</calcite.version>
+ <calcite.version>1.1.0-incubating</calcite.version>
<datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
<datanucleus-core.version>3.2.10</datanucleus-core.version>
<datanucleus-rdbms.version>3.2.9</datanucleus-rdbms.version>
Modified: hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (original)
+++ hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java Sat Mar 28 14:03:43 2015
@@ -10603,6 +10603,21 @@ public final class OrcProto {
*/
org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncodingOrBuilder getColumnsOrBuilder(
int index);
+
+ // optional string writerTimezone = 3;
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ boolean hasWriterTimezone();
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ java.lang.String getWriterTimezone();
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ com.google.protobuf.ByteString
+ getWriterTimezoneBytes();
}
/**
* Protobuf type {@code orc.proto.StripeFooter}
@@ -10671,6 +10686,11 @@ public final class OrcProto {
columns_.add(input.readMessage(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.PARSER, extensionRegistry));
break;
}
+ case 26: {
+ bitField0_ |= 0x00000001;
+ writerTimezone_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -10716,6 +10736,7 @@ public final class OrcProto {
return PARSER;
}
+ private int bitField0_;
// repeated .orc.proto.Stream streams = 1;
public static final int STREAMS_FIELD_NUMBER = 1;
private java.util.List<org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream> streams_;
@@ -10788,9 +10809,53 @@ public final class OrcProto {
return columns_.get(index);
}
+ // optional string writerTimezone = 3;
+ public static final int WRITERTIMEZONE_FIELD_NUMBER = 3;
+ private java.lang.Object writerTimezone_;
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public boolean hasWriterTimezone() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public java.lang.String getWriterTimezone() {
+ java.lang.Object ref = writerTimezone_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ writerTimezone_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public com.google.protobuf.ByteString
+ getWriterTimezoneBytes() {
+ java.lang.Object ref = writerTimezone_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ writerTimezone_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
private void initFields() {
streams_ = java.util.Collections.emptyList();
columns_ = java.util.Collections.emptyList();
+ writerTimezone_ = "";
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -10810,6 +10875,9 @@ public final class OrcProto {
for (int i = 0; i < columns_.size(); i++) {
output.writeMessage(2, columns_.get(i));
}
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(3, getWriterTimezoneBytes());
+ }
getUnknownFields().writeTo(output);
}
@@ -10827,6 +10895,10 @@ public final class OrcProto {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(2, columns_.get(i));
}
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(3, getWriterTimezoneBytes());
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -10957,6 +11029,8 @@ public final class OrcProto {
} else {
columnsBuilder_.clear();
}
+ writerTimezone_ = "";
+ bitField0_ = (bitField0_ & ~0x00000004);
return this;
}
@@ -10984,6 +11058,7 @@ public final class OrcProto {
public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter buildPartial() {
org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter(this);
int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
if (streamsBuilder_ == null) {
if (((bitField0_ & 0x00000001) == 0x00000001)) {
streams_ = java.util.Collections.unmodifiableList(streams_);
@@ -11002,6 +11077,11 @@ public final class OrcProto {
} else {
result.columns_ = columnsBuilder_.build();
}
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.writerTimezone_ = writerTimezone_;
+ result.bitField0_ = to_bitField0_;
onBuilt();
return result;
}
@@ -11069,6 +11149,11 @@ public final class OrcProto {
}
}
}
+ if (other.hasWriterTimezone()) {
+ bitField0_ |= 0x00000004;
+ writerTimezone_ = other.writerTimezone_;
+ onChanged();
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -11576,6 +11661,80 @@ public final class OrcProto {
return columnsBuilder_;
}
+ // optional string writerTimezone = 3;
+ private java.lang.Object writerTimezone_ = "";
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public boolean hasWriterTimezone() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public java.lang.String getWriterTimezone() {
+ java.lang.Object ref = writerTimezone_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ writerTimezone_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public com.google.protobuf.ByteString
+ getWriterTimezoneBytes() {
+ java.lang.Object ref = writerTimezone_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ writerTimezone_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public Builder setWriterTimezone(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ writerTimezone_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public Builder clearWriterTimezone() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ writerTimezone_ = getDefaultInstance().getWriterTimezone();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string writerTimezone = 3;</code>
+ */
+ public Builder setWriterTimezoneBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ writerTimezone_ = value;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:orc.proto.StripeFooter)
}
@@ -18921,40 +19080,41 @@ public final class OrcProto {
"ng\022,\n\004kind\030\001 \001(\0162\036.orc.proto.ColumnEncod" +
"ing.Kind\022\026\n\016dictionarySize\030\002 \001(\r\"D\n\004Kind" +
"\022\n\n\006DIRECT\020\000\022\016\n\nDICTIONARY\020\001\022\r\n\tDIRECT_V",
- "2\020\002\022\021\n\rDICTIONARY_V2\020\003\"^\n\014StripeFooter\022\"" +
+ "2\020\002\022\021\n\rDICTIONARY_V2\020\003\"v\n\014StripeFooter\022\"" +
"\n\007streams\030\001 \003(\0132\021.orc.proto.Stream\022*\n\007co" +
- "lumns\030\002 \003(\0132\031.orc.proto.ColumnEncoding\"\341" +
- "\002\n\004Type\022\"\n\004kind\030\001 \001(\0162\024.orc.proto.Type.K" +
- "ind\022\024\n\010subtypes\030\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030" +
- "\003 \003(\t\022\025\n\rmaximumLength\030\004 \001(\r\022\021\n\tprecisio" +
- "n\030\005 \001(\r\022\r\n\005scale\030\006 \001(\r\"\321\001\n\004Kind\022\013\n\007BOOLE" +
- "AN\020\000\022\010\n\004BYTE\020\001\022\t\n\005SHORT\020\002\022\007\n\003INT\020\003\022\010\n\004LO" +
- "NG\020\004\022\t\n\005FLOAT\020\005\022\n\n\006DOUBLE\020\006\022\n\n\006STRING\020\007\022" +
- "\n\n\006BINARY\020\010\022\r\n\tTIMESTAMP\020\t\022\010\n\004LIST\020\n\022\007\n\003",
- "MAP\020\013\022\n\n\006STRUCT\020\014\022\t\n\005UNION\020\r\022\013\n\007DECIMAL\020" +
- "\016\022\010\n\004DATE\020\017\022\013\n\007VARCHAR\020\020\022\010\n\004CHAR\020\021\"x\n\021St" +
- "ripeInformation\022\016\n\006offset\030\001 \001(\004\022\023\n\013index" +
- "Length\030\002 \001(\004\022\022\n\ndataLength\030\003 \001(\004\022\024\n\014foot" +
- "erLength\030\004 \001(\004\022\024\n\014numberOfRows\030\005 \001(\004\"/\n\020" +
- "UserMetadataItem\022\014\n\004name\030\001 \001(\t\022\r\n\005value\030" +
- "\002 \001(\014\"A\n\020StripeStatistics\022-\n\010colStats\030\001 " +
- "\003(\0132\033.orc.proto.ColumnStatistics\"<\n\010Meta" +
- "data\0220\n\013stripeStats\030\001 \003(\0132\033.orc.proto.St" +
- "ripeStatistics\"\222\002\n\006Footer\022\024\n\014headerLengt",
- "h\030\001 \001(\004\022\025\n\rcontentLength\030\002 \001(\004\022-\n\007stripe" +
- "s\030\003 \003(\0132\034.orc.proto.StripeInformation\022\036\n" +
- "\005types\030\004 \003(\0132\017.orc.proto.Type\022-\n\010metadat" +
- "a\030\005 \003(\0132\033.orc.proto.UserMetadataItem\022\024\n\014" +
- "numberOfRows\030\006 \001(\004\022/\n\nstatistics\030\007 \003(\0132\033" +
- ".orc.proto.ColumnStatistics\022\026\n\016rowIndexS" +
- "tride\030\010 \001(\r\"\305\001\n\nPostScript\022\024\n\014footerLeng" +
- "th\030\001 \001(\004\022/\n\013compression\030\002 \001(\0162\032.orc.prot" +
- "o.CompressionKind\022\034\n\024compressionBlockSiz" +
- "e\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001\022\026\n\016metadata",
- "Length\030\005 \001(\004\022\025\n\rwriterVersion\030\006 \001(\r\022\016\n\005m" +
- "agic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004NONE\020\000" +
- "\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003B\"\n org.a" +
- "pache.hadoop.hive.ql.io.orc"
+ "lumns\030\002 \003(\0132\031.orc.proto.ColumnEncoding\022\026" +
+ "\n\016writerTimezone\030\003 \001(\t\"\341\002\n\004Type\022\"\n\004kind\030" +
+ "\001 \001(\0162\024.orc.proto.Type.Kind\022\024\n\010subtypes\030" +
+ "\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030\003 \003(\t\022\025\n\rmaximum" +
+ "Length\030\004 \001(\r\022\021\n\tprecision\030\005 \001(\r\022\r\n\005scale" +
+ "\030\006 \001(\r\"\321\001\n\004Kind\022\013\n\007BOOLEAN\020\000\022\010\n\004BYTE\020\001\022\t" +
+ "\n\005SHORT\020\002\022\007\n\003INT\020\003\022\010\n\004LONG\020\004\022\t\n\005FLOAT\020\005\022" +
+ "\n\n\006DOUBLE\020\006\022\n\n\006STRING\020\007\022\n\n\006BINARY\020\010\022\r\n\tT",
+ "IMESTAMP\020\t\022\010\n\004LIST\020\n\022\007\n\003MAP\020\013\022\n\n\006STRUCT\020" +
+ "\014\022\t\n\005UNION\020\r\022\013\n\007DECIMAL\020\016\022\010\n\004DATE\020\017\022\013\n\007V" +
+ "ARCHAR\020\020\022\010\n\004CHAR\020\021\"x\n\021StripeInformation\022" +
+ "\016\n\006offset\030\001 \001(\004\022\023\n\013indexLength\030\002 \001(\004\022\022\n\n" +
+ "dataLength\030\003 \001(\004\022\024\n\014footerLength\030\004 \001(\004\022\024" +
+ "\n\014numberOfRows\030\005 \001(\004\"/\n\020UserMetadataItem" +
+ "\022\014\n\004name\030\001 \001(\t\022\r\n\005value\030\002 \001(\014\"A\n\020StripeS" +
+ "tatistics\022-\n\010colStats\030\001 \003(\0132\033.orc.proto." +
+ "ColumnStatistics\"<\n\010Metadata\0220\n\013stripeSt" +
+ "ats\030\001 \003(\0132\033.orc.proto.StripeStatistics\"\222",
+ "\002\n\006Footer\022\024\n\014headerLength\030\001 \001(\004\022\025\n\rconte" +
+ "ntLength\030\002 \001(\004\022-\n\007stripes\030\003 \003(\0132\034.orc.pr" +
+ "oto.StripeInformation\022\036\n\005types\030\004 \003(\0132\017.o" +
+ "rc.proto.Type\022-\n\010metadata\030\005 \003(\0132\033.orc.pr" +
+ "oto.UserMetadataItem\022\024\n\014numberOfRows\030\006 \001" +
+ "(\004\022/\n\nstatistics\030\007 \003(\0132\033.orc.proto.Colum" +
+ "nStatistics\022\026\n\016rowIndexStride\030\010 \001(\r\"\305\001\n\n" +
+ "PostScript\022\024\n\014footerLength\030\001 \001(\004\022/\n\013comp" +
+ "ression\030\002 \001(\0162\032.orc.proto.CompressionKin" +
+ "d\022\034\n\024compressionBlockSize\030\003 \001(\004\022\023\n\007versi",
+ "on\030\004 \003(\rB\002\020\001\022\026\n\016metadataLength\030\005 \001(\004\022\025\n\r" +
+ "writerVersion\030\006 \001(\r\022\016\n\005magic\030\300> \001(\t*:\n\017C" +
+ "ompressionKind\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SN" +
+ "APPY\020\002\022\007\n\003LZO\020\003B\"\n org.apache.hadoop.hiv" +
+ "e.ql.io.orc"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -19056,7 +19216,7 @@ public final class OrcProto {
internal_static_orc_proto_StripeFooter_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_orc_proto_StripeFooter_descriptor,
- new java.lang.String[] { "Streams", "Columns", });
+ new java.lang.String[] { "Streams", "Columns", "WriterTimezone", });
internal_static_orc_proto_Type_descriptor =
getDescriptor().getMessageTypes().get(16);
internal_static_orc_proto_Type_fieldAccessorTable = new
Modified: hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt (original)
+++ hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt Sat Mar 28 14:03:43 2015
@@ -130,7 +130,7 @@ public class <ClassName> extends VectorE
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(1)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
+ VectorExpressionDescriptor.ArgumentType.getType("<VectorExprArgType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Sat Mar 28 14:03:43 2015
@@ -440,15 +440,13 @@ public class Driver implements CommandPr
sem.validate();
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ANALYZE);
- plan = new QueryPlan(command, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
- SessionState.get().getCommandType());
+ // Command should be redacted before passing it to the QueryPlan in order
+ // to avoid returning sensitive data
+ String queryStr = HookUtils.redactLogString(conf, command);
+
+ plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
+ SessionState.get().getCommandType());
- String queryStr = plan.getQueryStr();
- List<Redactor> queryRedactors = getHooks(ConfVars.QUERYREDACTORHOOKS, Redactor.class);
- for (Redactor redactor : queryRedactors) {
- redactor.setConf(conf);
- queryStr = redactor.redactQuery(queryStr);
- }
conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);
conf.set("mapreduce.workflow.id", "hive_" + queryId);
@@ -973,6 +971,7 @@ public class Driver implements CommandPr
if (txnId == SessionState.NO_CURRENT_TXN) {
txnId = txnMgr.openTxn(userFromUGI);
ss.setCurrentTxn(txnId);
+ LOG.debug("Setting current transaction to " + txnId);
}
// Set the transaction id in all of the acid file sinks
if (acidSinks != null) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java Sat Mar 28 14:03:43 2015
@@ -149,7 +149,7 @@ public class QueryPlan implements Serial
return userid
+ "_"
- + String.format("%1$4d%2$02d%3$02d%4$02d%5$02d%5$02d", gc
+ + String.format("%1$4d%2$02d%3$02d%4$02d%5$02d%6$02d", gc
.get(Calendar.YEAR), gc.get(Calendar.MONTH) + 1, gc
.get(Calendar.DAY_OF_MONTH), gc.get(Calendar.HOUR_OF_DAY), gc
.get(Calendar.MINUTE), gc.get(Calendar.SECOND))
@@ -435,7 +435,7 @@ public class QueryPlan implements Serial
return "\"" + key + "\":" + getJSONValue(value) + ",";
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("rawtypes")
private String getJSONList(List list) {
if (list == null) {
return "null";
@@ -451,7 +451,7 @@ public class QueryPlan implements Serial
return sb.toString();
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("rawtypes")
public String getJSONMap(Map map) {
if (map == null) {
return "null";
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java Sat Mar 28 14:03:43 2015
@@ -17,6 +17,13 @@
*/
package org.apache.hadoop.hive.ql.exec;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Future;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -28,11 +35,6 @@ import org.apache.hadoop.hive.ql.plan.Dy
import org.apache.hadoop.hive.ql.plan.FileMergeDesc;
import org.apache.hadoop.mapred.JobConf;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Set;
-
/**
* Fast file merge operator for ORC and RCfile. This is an abstract class which
* does not process any rows. Refer {@link org.apache.hadoop.hive.ql.exec.OrcFileMergeOperator}
@@ -63,8 +65,8 @@ public abstract class AbstractFileMergeO
protected transient DynamicPartitionCtx dpCtx;
@Override
- public void initializeOp(Configuration hconf) throws HiveException {
- super.initializeOp(hconf);
+ public Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+ Collection<Future<?>> result = super.initializeOp(hconf);
this.jc = new JobConf(hconf);
incompatFileSet = new HashSet<Path>();
autoDelete = false;
@@ -92,6 +94,7 @@ public abstract class AbstractFileMergeO
throw new HiveException("Failed to initialize AbstractFileMergeOperator",
e);
}
+ return result;
}
// sets up temp and task temp path
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java Sat Mar 28 14:03:43 2015
@@ -20,7 +20,9 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.persistence.RowContainer;
@@ -57,7 +59,7 @@ public abstract class AbstractMapJoinOpe
@Override
@SuppressWarnings("unchecked")
- protected void initializeOp(Configuration hconf) throws HiveException {
+ protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
if (conf.getGenJoinKeys()) {
int tagLen = conf.getTagLength();
joinKeys = new List[tagLen];
@@ -66,7 +68,7 @@ public abstract class AbstractMapJoinOpe
inputObjInspectors,NOTSKIPBIGTABLE, tagLen);
}
- super.initializeOp(hconf);
+ Collection<Future<?>> result = super.initializeOp(hconf);
numMapRowsRead = 0;
@@ -81,7 +83,7 @@ public abstract class AbstractMapJoinOpe
!hasFilter(posBigTable), reporter);
storage[posBigTable] = bigPosRC;
- initializeChildren(hconf);
+ return result;
}
@Override
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java Sat Mar 28 14:03:43 2015
@@ -20,7 +20,9 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.Collections;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -50,11 +52,14 @@ public class AppMasterEventOperator exte
protected transient long MAX_SIZE;
@Override
- public void initializeOp(Configuration hconf) throws HiveException {
+ public Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+ Collection<Future<?>> result = super.initializeOp(hconf);
+
MAX_SIZE = HiveConf.getLongVar(hconf, ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE);
serializer =
(Serializer) ReflectionUtils.newInstance(conf.getTable().getDeserializerClass(), null);
initDataBuffer(false);
+ return result;
}
protected void initDataBuffer(boolean skipPruning) throws HiveException {
@@ -71,7 +76,7 @@ public class AppMasterEventOperator exte
}
@Override
- public void processOp(Object row, int tag) throws HiveException {
+ public void process(Object row, int tag) throws HiveException {
if (hasReachedMaxSize) {
return;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java Sat Mar 28 14:03:43 2015
@@ -20,11 +20,13 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.plan.CollectDesc;
+import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -41,16 +43,17 @@ public class CollectOperator extends Ope
transient int maxSize;
@Override
- protected void initializeOp(Configuration hconf) throws HiveException {
- super.initializeOp(hconf);
+ protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+ Collection<Future<?>> result = super.initializeOp(hconf);
rowList = new ArrayList<Object>();
maxSize = conf.getBufferSize().intValue();
+ return result;
}
boolean firstRow = true;
@Override
- public void processOp(Object row, int tag) throws HiveException {
+ public void process(Object row, int tag) throws HiveException {
ObjectInspector rowInspector = inputObjInspectors[tag];
if (firstRow) {
firstRow = false;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Sat Mar 28 14:03:43 2015
@@ -267,7 +267,7 @@ public class ColumnStatsTask extends Tas
private List<ColumnStatistics> constructColumnStatsFromPackedRows() throws HiveException, MetaException, IOException {
- String dbName = SessionState.get().getCurrentDatabase();
+ String currentDb = SessionState.get().getCurrentDatabase();
String tableName = work.getColStats().getTableName();
String partName = null;
List<String> colName = work.getColStats().getColName();
@@ -286,7 +286,7 @@ public class ColumnStatsTask extends Tas
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<Object> list = soi.getStructFieldsDataAsList(packedRow.o);
- Table tbl = db.getTable(dbName,tableName);
+ Table tbl = db.getTable(currentDb,tableName);
List<FieldSchema> partColSchema = tbl.getPartCols();
// Partition columns are appended at end, we only care about stats column
int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size();
@@ -313,8 +313,8 @@ public class ColumnStatsTask extends Tas
}
partName = Warehouse.makePartName(partColSchema, partVals);
}
-
- ColumnStatisticsDesc statsDesc = getColumnStatsDesc(dbName, tableName, partName, isTblLevel);
+ String [] names = Utilities.getDbTableName(currentDb, tableName);
+ ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, isTblLevel);
ColumnStatistics colStats = new ColumnStatistics();
colStats.setStatsDesc(statsDesc);
colStats.setStatsObj(statsObjs);
@@ -368,7 +368,7 @@ public class ColumnStatsTask extends Tas
return persistPartitionStats();
}
} catch (Exception e) {
- LOG.info(e);
+ LOG.error("Failed to run column stats task", e);
}
return 1;
}