You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/03/28 15:03:49 UTC

svn commit: r1669775 [10/35] - in /hive/branches/spark: ./ ant/src/org/apache/hadoop/hive/ant/ beeline/src/java/org/apache/hive/beeline/ common/src/java/org/apache/hadoop/hive/common/ common/src/java/org/apache/hadoop/hive/common/type/ common/src/java/...

Modified: hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Sat Mar 28 14:03:43 2015
@@ -8160,6 +8160,118 @@ class ShowCompactResponse:
   def __ne__(self, other):
     return not (self == other)
 
+class AddDynamicPartitions:
+  """
+  Attributes:
+   - txnid
+   - dbname
+   - tablename
+   - partitionnames
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I64, 'txnid', None, None, ), # 1
+    (2, TType.STRING, 'dbname', None, None, ), # 2
+    (3, TType.STRING, 'tablename', None, None, ), # 3
+    (4, TType.LIST, 'partitionnames', (TType.STRING,None), None, ), # 4
+  )
+
+  def __init__(self, txnid=None, dbname=None, tablename=None, partitionnames=None,):
+    self.txnid = txnid
+    self.dbname = dbname
+    self.tablename = tablename
+    self.partitionnames = partitionnames
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I64:
+          self.txnid = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRING:
+          self.dbname = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.STRING:
+          self.tablename = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.LIST:
+          self.partitionnames = []
+          (_etype437, _size434) = iprot.readListBegin()
+          for _i438 in xrange(_size434):
+            _elem439 = iprot.readString();
+            self.partitionnames.append(_elem439)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('AddDynamicPartitions')
+    if self.txnid is not None:
+      oprot.writeFieldBegin('txnid', TType.I64, 1)
+      oprot.writeI64(self.txnid)
+      oprot.writeFieldEnd()
+    if self.dbname is not None:
+      oprot.writeFieldBegin('dbname', TType.STRING, 2)
+      oprot.writeString(self.dbname)
+      oprot.writeFieldEnd()
+    if self.tablename is not None:
+      oprot.writeFieldBegin('tablename', TType.STRING, 3)
+      oprot.writeString(self.tablename)
+      oprot.writeFieldEnd()
+    if self.partitionnames is not None:
+      oprot.writeFieldBegin('partitionnames', TType.LIST, 4)
+      oprot.writeListBegin(TType.STRING, len(self.partitionnames))
+      for iter440 in self.partitionnames:
+        oprot.writeString(iter440)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.txnid is None:
+      raise TProtocol.TProtocolException(message='Required field txnid is unset!')
+    if self.dbname is None:
+      raise TProtocol.TProtocolException(message='Required field dbname is unset!')
+    if self.tablename is None:
+      raise TProtocol.TProtocolException(message='Required field tablename is unset!')
+    if self.partitionnames is None:
+      raise TProtocol.TProtocolException(message='Required field partitionnames is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
 class NotificationEventRequest:
   """
   Attributes:
@@ -8388,11 +8500,11 @@ class NotificationEventResponse:
       if fid == 1:
         if ftype == TType.LIST:
           self.events = []
-          (_etype437, _size434) = iprot.readListBegin()
-          for _i438 in xrange(_size434):
-            _elem439 = NotificationEvent()
-            _elem439.read(iprot)
-            self.events.append(_elem439)
+          (_etype444, _size441) = iprot.readListBegin()
+          for _i445 in xrange(_size441):
+            _elem446 = NotificationEvent()
+            _elem446.read(iprot)
+            self.events.append(_elem446)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -8409,8 +8521,8 @@ class NotificationEventResponse:
     if self.events is not None:
       oprot.writeFieldBegin('events', TType.LIST, 1)
       oprot.writeListBegin(TType.STRUCT, len(self.events))
-      for iter440 in self.events:
-        iter440.write(oprot)
+      for iter447 in self.events:
+        iter447.write(oprot)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
@@ -8521,10 +8633,10 @@ class InsertEventRequestData:
       if fid == 1:
         if ftype == TType.LIST:
           self.filesAdded = []
-          (_etype444, _size441) = iprot.readListBegin()
-          for _i445 in xrange(_size441):
-            _elem446 = iprot.readString();
-            self.filesAdded.append(_elem446)
+          (_etype451, _size448) = iprot.readListBegin()
+          for _i452 in xrange(_size448):
+            _elem453 = iprot.readString();
+            self.filesAdded.append(_elem453)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -8541,8 +8653,8 @@ class InsertEventRequestData:
     if self.filesAdded is not None:
       oprot.writeFieldBegin('filesAdded', TType.LIST, 1)
       oprot.writeListBegin(TType.STRING, len(self.filesAdded))
-      for iter447 in self.filesAdded:
-        oprot.writeString(iter447)
+      for iter454 in self.filesAdded:
+        oprot.writeString(iter454)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
@@ -8685,10 +8797,10 @@ class FireEventRequest:
       elif fid == 5:
         if ftype == TType.LIST:
           self.partitionVals = []
-          (_etype451, _size448) = iprot.readListBegin()
-          for _i452 in xrange(_size448):
-            _elem453 = iprot.readString();
-            self.partitionVals.append(_elem453)
+          (_etype458, _size455) = iprot.readListBegin()
+          for _i459 in xrange(_size455):
+            _elem460 = iprot.readString();
+            self.partitionVals.append(_elem460)
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
@@ -8721,8 +8833,8 @@ class FireEventRequest:
     if self.partitionVals is not None:
       oprot.writeFieldBegin('partitionVals', TType.LIST, 5)
       oprot.writeListBegin(TType.STRING, len(self.partitionVals))
-      for iter454 in self.partitionVals:
-        oprot.writeString(iter454)
+      for iter461 in self.partitionVals:
+        oprot.writeString(iter461)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
     oprot.writeFieldStop()

Modified: hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Sat Mar 28 14:03:43 2015
@@ -1994,6 +1994,32 @@ class ShowCompactResponse
   ::Thrift::Struct.generate_accessors self
 end
 
+class AddDynamicPartitions
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  TXNID = 1
+  DBNAME = 2
+  TABLENAME = 3
+  PARTITIONNAMES = 4
+
+  FIELDS = {
+    TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'},
+    DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+    TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'},
+    PARTITIONNAMES => {:type => ::Thrift::Types::LIST, :name => 'partitionnames', :element => {:type => ::Thrift::Types::STRING}}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txnid is unset!') unless @txnid
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablename is unset!') unless @tablename
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partitionnames is unset!') unless @partitionnames
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
 class NotificationEventRequest
   include ::Thrift::Struct, ::Thrift::Struct_Union
   LASTEVENT = 1

Modified: hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb (original)
+++ hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb Sat Mar 28 14:03:43 2015
@@ -228,6 +228,24 @@ module ThriftHiveMetastore
       raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_fields failed: unknown result')
     end
 
+    def get_fields_with_environment_context(db_name, table_name, environment_context)
+      send_get_fields_with_environment_context(db_name, table_name, environment_context)
+      return recv_get_fields_with_environment_context()
+    end
+
+    def send_get_fields_with_environment_context(db_name, table_name, environment_context)
+      send_message('get_fields_with_environment_context', Get_fields_with_environment_context_args, :db_name => db_name, :table_name => table_name, :environment_context => environment_context)
+    end
+
+    def recv_get_fields_with_environment_context()
+      result = receive_message(Get_fields_with_environment_context_result)
+      return result.success unless result.success.nil?
+      raise result.o1 unless result.o1.nil?
+      raise result.o2 unless result.o2.nil?
+      raise result.o3 unless result.o3.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_fields_with_environment_context failed: unknown result')
+    end
+
     def get_schema(db_name, table_name)
       send_get_schema(db_name, table_name)
       return recv_get_schema()
@@ -246,6 +264,24 @@ module ThriftHiveMetastore
       raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_schema failed: unknown result')
     end
 
+    def get_schema_with_environment_context(db_name, table_name, environment_context)
+      send_get_schema_with_environment_context(db_name, table_name, environment_context)
+      return recv_get_schema_with_environment_context()
+    end
+
+    def send_get_schema_with_environment_context(db_name, table_name, environment_context)
+      send_message('get_schema_with_environment_context', Get_schema_with_environment_context_args, :db_name => db_name, :table_name => table_name, :environment_context => environment_context)
+    end
+
+    def recv_get_schema_with_environment_context()
+      result = receive_message(Get_schema_with_environment_context_result)
+      return result.success unless result.success.nil?
+      raise result.o1 unless result.o1.nil?
+      raise result.o2 unless result.o2.nil?
+      raise result.o3 unless result.o3.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_schema_with_environment_context failed: unknown result')
+    end
+
     def create_table(tbl)
       send_create_table(tbl)
       recv_create_table()
@@ -1977,6 +2013,22 @@ module ThriftHiveMetastore
       raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'show_compact failed: unknown result')
     end
 
+    def add_dynamic_partitions(rqst)
+      send_add_dynamic_partitions(rqst)
+      recv_add_dynamic_partitions()
+    end
+
+    def send_add_dynamic_partitions(rqst)
+      send_message('add_dynamic_partitions', Add_dynamic_partitions_args, :rqst => rqst)
+    end
+
+    def recv_add_dynamic_partitions()
+      result = receive_message(Add_dynamic_partitions_result)
+      raise result.o1 unless result.o1.nil?
+      raise result.o2 unless result.o2.nil?
+      return
+    end
+
     def get_next_notification(rqst)
       send_get_next_notification(rqst)
       return recv_get_next_notification()
@@ -2194,6 +2246,21 @@ module ThriftHiveMetastore
       write_result(result, oprot, 'get_fields', seqid)
     end
 
+    def process_get_fields_with_environment_context(seqid, iprot, oprot)
+      args = read_args(iprot, Get_fields_with_environment_context_args)
+      result = Get_fields_with_environment_context_result.new()
+      begin
+        result.success = @handler.get_fields_with_environment_context(args.db_name, args.table_name, args.environment_context)
+      rescue ::MetaException => o1
+        result.o1 = o1
+      rescue ::UnknownTableException => o2
+        result.o2 = o2
+      rescue ::UnknownDBException => o3
+        result.o3 = o3
+      end
+      write_result(result, oprot, 'get_fields_with_environment_context', seqid)
+    end
+
     def process_get_schema(seqid, iprot, oprot)
       args = read_args(iprot, Get_schema_args)
       result = Get_schema_result.new()
@@ -2209,6 +2276,21 @@ module ThriftHiveMetastore
       write_result(result, oprot, 'get_schema', seqid)
     end
 
+    def process_get_schema_with_environment_context(seqid, iprot, oprot)
+      args = read_args(iprot, Get_schema_with_environment_context_args)
+      result = Get_schema_with_environment_context_result.new()
+      begin
+        result.success = @handler.get_schema_with_environment_context(args.db_name, args.table_name, args.environment_context)
+      rescue ::MetaException => o1
+        result.o1 = o1
+      rescue ::UnknownTableException => o2
+        result.o2 = o2
+      rescue ::UnknownDBException => o3
+        result.o3 = o3
+      end
+      write_result(result, oprot, 'get_schema_with_environment_context', seqid)
+    end
+
     def process_create_table(seqid, iprot, oprot)
       args = read_args(iprot, Create_table_args)
       result = Create_table_result.new()
@@ -3538,6 +3620,19 @@ module ThriftHiveMetastore
       write_result(result, oprot, 'show_compact', seqid)
     end
 
+    def process_add_dynamic_partitions(seqid, iprot, oprot)
+      args = read_args(iprot, Add_dynamic_partitions_args)
+      result = Add_dynamic_partitions_result.new()
+      begin
+        @handler.add_dynamic_partitions(args.rqst)
+      rescue ::NoSuchTxnException => o1
+        result.o1 = o1
+      rescue ::TxnAbortedException => o2
+        result.o2 = o2
+      end
+      write_result(result, oprot, 'add_dynamic_partitions', seqid)
+    end
+
     def process_get_next_notification(seqid, iprot, oprot)
       args = read_args(iprot, Get_next_notification_args)
       result = Get_next_notification_result.new()
@@ -4030,6 +4125,48 @@ module ThriftHiveMetastore
     ::Thrift::Struct.generate_accessors self
   end
 
+  class Get_fields_with_environment_context_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    DB_NAME = 1
+    TABLE_NAME = 2
+    ENVIRONMENT_CONTEXT = 3
+
+    FIELDS = {
+      DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+      TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+      ENVIRONMENT_CONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environment_context', :class => ::EnvironmentContext}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Get_fields_with_environment_context_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+    O1 = 1
+    O2 = 2
+    O3 = 3
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+      O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException},
+      O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::UnknownTableException},
+      O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::UnknownDBException}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
   class Get_schema_args
     include ::Thrift::Struct, ::Thrift::Struct_Union
     DB_NAME = 1
@@ -4070,6 +4207,48 @@ module ThriftHiveMetastore
     ::Thrift::Struct.generate_accessors self
   end
 
+  class Get_schema_with_environment_context_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    DB_NAME = 1
+    TABLE_NAME = 2
+    ENVIRONMENT_CONTEXT = 3
+
+    FIELDS = {
+      DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+      TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+      ENVIRONMENT_CONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environment_context', :class => ::EnvironmentContext}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Get_schema_with_environment_context_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+    O1 = 1
+    O2 = 2
+    O3 = 3
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+      O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException},
+      O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::UnknownTableException},
+      O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::UnknownDBException}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
   class Create_table_args
     include ::Thrift::Struct, ::Thrift::Struct_Union
     TBL = 1
@@ -8040,6 +8219,40 @@ module ThriftHiveMetastore
     }
 
     def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Add_dynamic_partitions_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    RQST = 1
+
+    FIELDS = {
+      RQST => {:type => ::Thrift::Types::STRUCT, :name => 'rqst', :class => ::AddDynamicPartitions}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Add_dynamic_partitions_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    O1 = 1
+    O2 = 2
+
+    FIELDS = {
+      O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::NoSuchTxnException},
+      O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::TxnAbortedException}
+    }
+
+    def struct_fields; FIELDS; end
 
     def validate
     end

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java Sat Mar 28 14:03:43 2015
@@ -100,6 +100,21 @@ public class Deadline {
   }
 
   /**
+   * Check whether the timer is started.
+   * @return
+   * @throws MetaException
+   */
+  public static boolean isStarted() throws MetaException {
+    Deadline deadline = getCurrentDeadline();
+    if (deadline != null) {
+      return deadline.startTime >= 0;
+    } else {
+      throw newMetaException(new DeadlineException("The threadlocal Deadline is null," +
+          " please register it firstly."));
+    }
+  }
+
+  /**
    * start the timer before a method is invoked.
    * @param method
    */

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Sat Mar 28 14:03:43 2015
@@ -33,8 +33,11 @@ import org.apache.hadoop.hive.common.Fil
 import org.apache.hadoop.hive.common.ObjectPair;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
 import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -42,6 +45,9 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hive.common.util.HiveStringUtils;
+
+import com.google.common.collect.Lists;
 
 /**
  * Hive specific implementation of alter
@@ -128,7 +134,10 @@ public class HiveAlterHandler implements
         if(MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
           List<Partition> parts = msdb.getPartitions(dbname, name, -1);
           for (Partition part : parts) {
+            List<FieldSchema> oldCols = part.getSd().getCols();
             part.getSd().setCols(newt.getSd().getCols());
+            String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
+            updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part);
             msdb.alterPartition(dbname, name, part.getValues(), part);
           }
         } else {
@@ -209,6 +218,13 @@ public class HiveAlterHandler implements
             Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
             altps.add(ObjectPair.create(part, part.getSd().getLocation()));
             part.getSd().setLocation(newPartLocPath.toString());
+            String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
+            try {
+              //existing partition column stats is no longer valid, remove them
+              msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null);
+            } catch (InvalidInputException iie) {
+              throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
+            }
             msdb.alterPartition(dbname, name, part.getValues(), part);
           }
         }
@@ -219,6 +235,7 @@ public class HiveAlterHandler implements
           // alterPartition()
           MetaStoreUtils.updateUnpartitionedTableStatsFast(db, newt, wh, false, true);
       }
+      updateTableColumnStatsForAlterTable(msdb, oldt, newt);
       // now finally call alter table
       msdb.alterTable(dbname, name, newt);
       // commit the changes
@@ -311,6 +328,7 @@ public class HiveAlterHandler implements
         if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl)) {
           MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true);
         }
+        updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
         msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
       } catch (InvalidObjectException e) {
         throw new InvalidOperationException("alter is not possible");
@@ -349,6 +367,15 @@ public class HiveAlterHandler implements
       // if the external partition is renamed, the file should not change
       if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
         new_part.getSd().setLocation(oldPart.getSd().getLocation());
+        String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
+        try {
+          //existing partition column stats is no longer valid, remove
+          msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
+        } catch (NoSuchObjectException nsoe) {
+          //ignore
+        } catch (InvalidInputException iie) {
+          throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
+        }
         msdb.alterPartition(dbname, name, part_vals, new_part);
       } else {
         try {
@@ -395,6 +422,15 @@ public class HiveAlterHandler implements
           if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl)) {
             MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true);
           }
+          String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
+          try {
+            //existing partition column stats is no longer valid, remove
+            msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
+          } catch (NoSuchObjectException nsoe) {
+            //ignore
+          } catch (InvalidInputException iie) {
+            throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
+          }
           msdb.alterPartition(dbname, name, part_vals, new_part);
         }
       }
@@ -461,6 +497,7 @@ public class HiveAlterHandler implements
         if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl)) {
           MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true);
         }
+        updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
       }
       msdb.alterPartitions(dbname, name, partValsList, new_parts);
     } catch (InvalidObjectException e) {
@@ -508,4 +545,131 @@ public class HiveAlterHandler implements
     return new Path(currentUri.getScheme(), currentUri.getAuthority(),
         defaultNewPath.toUri().getPath());
   }
+
+  private void updatePartColumnStatsForAlterColumns(RawStore msdb, Partition oldPartition,
+      String oldPartName, List<String> partVals, List<FieldSchema> oldCols, Partition newPart)
+          throws MetaException, InvalidObjectException {
+    String dbName = oldPartition.getDbName();
+    String tableName = oldPartition.getTableName();
+    try {
+      List<String> oldPartNames = Lists.newArrayList(oldPartName);
+      List<String> oldColNames = new ArrayList<String>(oldCols.size());
+      for (FieldSchema oldCol : oldCols) {
+        oldColNames.add(oldCol.getName());
+      }
+      List<FieldSchema> newCols = newPart.getSd().getCols();
+      List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbName, tableName,
+          oldPartNames, oldColNames);
+      assert (partsColStats.size() <= 1);
+      for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop
+        List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj();
+        for (ColumnStatisticsObj statsObj : statsObjs) {
+          boolean found =false;
+          for (FieldSchema newCol : newCols) {
+            if (statsObj.getColName().equals(newCol.getName())
+                && statsObj.getColType().equals(newCol.getType())) {
+              found = true;
+              break;
+            }
+          }
+          if (!found) {
+            msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals,
+                statsObj.getColName());
+          }
+        }
+      }
+    } catch (NoSuchObjectException nsoe) {
+      LOG.debug("Could not find db entry." + nsoe);
+      //ignore
+    } catch (InvalidInputException iie) {
+      throw new InvalidObjectException
+      ("Invalid input to update partition column stats in alter table change columns" + iie);
+    }
+  }
+
+  private void updatePartColumnStats(RawStore msdb, String dbName, String tableName,
+      List<String> partVals, Partition newPart) throws MetaException, InvalidObjectException {
+    dbName = HiveStringUtils.normalizeIdentifier(dbName);
+    tableName = HiveStringUtils.normalizeIdentifier(tableName);
+    String newDbName = HiveStringUtils.normalizeIdentifier(newPart.getDbName());
+    String newTableName = HiveStringUtils.normalizeIdentifier(newPart.getTableName());
+
+    Table oldTable = msdb.getTable(dbName, tableName);
+    if (oldTable == null) {
+      return;
+    }
+
+    try {
+      String oldPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), partVals);
+      String newPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), newPart.getValues());
+      if (!dbName.equals(newDbName) || !tableName.equals(newTableName)
+          || !oldPartName.equals(newPartName)) {
+        msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, null);
+      } else {
+        Partition oldPartition = msdb.getPartition(dbName, tableName, partVals);
+        if (oldPartition == null) {
+          return;
+        }
+        if (oldPartition.getSd() != null && newPart.getSd() != null) {
+        List<FieldSchema> oldCols = oldPartition.getSd().getCols();
+          if (!MetaStoreUtils.areSameColumns(oldCols, newPart.getSd().getCols())) {
+            updatePartColumnStatsForAlterColumns(msdb, oldPartition, oldPartName, partVals, oldCols, newPart);
+          }
+        }
+      }
+    } catch (NoSuchObjectException nsoe) {
+      LOG.debug("Could not find db entry." + nsoe);
+      //ignore
+    } catch (InvalidInputException iie) {
+      throw new InvalidObjectException("Invalid input to update partition column stats." + iie);
+    }
+  }
+
+  private void updateTableColumnStatsForAlterTable(RawStore msdb, Table oldTable, Table newTable)
+      throws MetaException, InvalidObjectException {
+    String dbName = oldTable.getDbName();
+    String tableName = oldTable.getTableName();
+    String newDbName = HiveStringUtils.normalizeIdentifier(newTable.getDbName());
+    String newTableName = HiveStringUtils.normalizeIdentifier(newTable.getTableName());
+
+    try {
+      if (!dbName.equals(newDbName) || !tableName.equals(newTableName)) {
+        msdb.deleteTableColumnStatistics(dbName, tableName, null);
+      } else {
+        List<FieldSchema> oldCols = oldTable.getSd().getCols();
+        List<FieldSchema> newCols = newTable.getSd().getCols();
+        if (!MetaStoreUtils.areSameColumns(oldCols, newCols)) {
+          List<String> oldColNames = new ArrayList<String>(oldCols.size());
+          for (FieldSchema oldCol : oldCols) {
+            oldColNames.add(oldCol.getName());
+          }
+
+          ColumnStatistics cs = msdb.getTableColumnStatistics(dbName, tableName, oldColNames);
+          if (cs == null) {
+            return;
+          }
+
+          List<ColumnStatisticsObj> statsObjs = cs.getStatsObj();
+          for (ColumnStatisticsObj statsObj : statsObjs) {
+            boolean found = false;
+            for (FieldSchema newCol : newCols) {
+              if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
+                  && statsObj.getColType().equals(newCol.getType())) {
+                found = true;
+                break;
+              }
+            }
+            if (!found) {
+              msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
+            }
+          }
+        }
+      }
+    } catch (NoSuchObjectException nsoe) {
+      LOG.debug("Could not find db entry." + nsoe);
+    } catch (InvalidInputException e) {
+      //should not happen since the input were verified before passed in
+      throw new InvalidObjectException("Invalid inputs to update table column stats: " + e);
+    }
+  }
 }

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Sat Mar 28 14:03:43 2015
@@ -66,6 +66,7 @@ import org.apache.hadoop.hive.common.met
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
@@ -3445,13 +3446,22 @@ public class HiveMetaStore extends Thrif
     @Override
     public List<FieldSchema> get_fields(String db, String tableName)
         throws MetaException, UnknownTableException, UnknownDBException {
-      startFunction("get_fields", ": db=" + db + "tbl=" + tableName);
+      return get_fields_with_environment_context(db, tableName, null);
+    }
+
+    @Override
+    public List<FieldSchema> get_fields_with_environment_context(String db, String tableName,
+        final EnvironmentContext envContext)
+        throws MetaException, UnknownTableException, UnknownDBException {
+      startFunction("get_fields_with_environment_context", ": db=" + db + "tbl=" + tableName);
       String[] names = tableName.split("\\.");
       String base_table_name = names[0];
 
       Table tbl;
       List<FieldSchema> ret = null;
       Exception ex = null;
+      ClassLoader orgHiveLoader = null;
+      Configuration curConf = hiveConf;
       try {
         try {
           tbl = get_table_core(db, base_table_name);
@@ -3464,7 +3474,18 @@ public class HiveMetaStore extends Thrif
           ret = tbl.getSd().getCols();
         } else {
           try {
-            Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl, false);
+            if (envContext != null) {
+              String addedJars = envContext.getProperties().get("hive.added.jars.path");
+              if (org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) {
+                //for thread safe
+                curConf = getConf();
+                orgHiveLoader = curConf.getClassLoader();
+                ClassLoader loader = MetaStoreUtils.addToClassPath(orgHiveLoader, org.apache.commons.lang.StringUtils.split(addedJars, ","));
+                curConf.setClassLoader(loader);
+              }
+            }
+
+            Deserializer s = MetaStoreUtils.getDeserializer(curConf, tbl, false);
             ret = MetaStoreUtils.getFieldsFromDeserializer(tableName, s);
           } catch (SerDeException e) {
             StringUtils.stringifyException(e);
@@ -3483,7 +3504,10 @@ public class HiveMetaStore extends Thrif
           throw newMetaException(e);
         }
       } finally {
-        endFunction("get_fields", ret != null, ex, tableName);
+        if (orgHiveLoader != null) {
+          curConf.setClassLoader(orgHiveLoader);
+        }
+        endFunction("get_fields_with_environment_context", ret != null, ex, tableName);
       }
 
       return ret;
@@ -3505,7 +3529,30 @@ public class HiveMetaStore extends Thrif
     @Override
     public List<FieldSchema> get_schema(String db, String tableName)
         throws MetaException, UnknownTableException, UnknownDBException {
-      startFunction("get_schema", ": db=" + db + "tbl=" + tableName);
+      return get_schema_with_environment_context(db,tableName, null);
+    }
+
+
+    /**
+     * Return the schema of the table. This function includes partition columns
+     * in addition to the regular columns.
+     *
+     * @param db
+     *          Name of the database
+     * @param tableName
+     *          Name of the table
+     * @param envContext
+     *          Store session based properties
+     * @return List of columns, each column is a FieldSchema structure
+     * @throws MetaException
+     * @throws UnknownTableException
+     * @throws UnknownDBException
+     */
+    @Override
+    public List<FieldSchema> get_schema_with_environment_context(String db, String tableName,
+          final EnvironmentContext envContext)
+        throws MetaException, UnknownTableException, UnknownDBException {
+      startFunction("get_schema_with_environment_context", ": db=" + db + "tbl=" + tableName);
       boolean success = false;
       Exception ex = null;
       try {
@@ -3518,7 +3565,7 @@ public class HiveMetaStore extends Thrif
         } catch (NoSuchObjectException e) {
           throw new UnknownTableException(e.getMessage());
         }
-        List<FieldSchema> fieldSchemas = get_fields(db, base_table_name);
+        List<FieldSchema> fieldSchemas = get_fields_with_environment_context(db, base_table_name,envContext);
 
         if (tbl == null || fieldSchemas == null) {
           throw new UnknownTableException(tableName + " doesn't exist");
@@ -3545,7 +3592,7 @@ public class HiveMetaStore extends Thrif
           throw me;
         }
       } finally {
-        endFunction("get_schema", success, ex, tableName);
+        endFunction("get_schema_with_environment_context", success, ex, tableName);
       }
     }
 
@@ -5520,6 +5567,12 @@ public class HiveMetaStore extends Thrif
     }
 
     @Override
+    public void add_dynamic_partitions(AddDynamicPartitions rqst)
+        throws NoSuchTxnException, TxnAbortedException, TException {
+      getTxnHandler().addDynamicPartitions(rqst);
+    }
+
+    @Override
     public GetPrincipalsInRoleResponse get_principals_in_role(GetPrincipalsInRoleRequest request)
         throws MetaException, TException {
 

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Sat Mar 28 14:03:43 2015
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.conf.HiveConfUtil;
 import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
@@ -294,6 +295,11 @@ public class HiveMetaStoreClient impleme
   }
 
   @Override
+  public void setHiveAddedJars(String addedJars) {
+    HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars);
+  }
+
+  @Override
   public void reconnect() throws MetaException {
     if (localMetaStore) {
       // For direct DB connections we don't yet support reestablishing connections.
@@ -1480,7 +1486,15 @@ public class HiveMetaStoreClient impleme
   public List<FieldSchema> getSchema(String db, String tableName)
       throws MetaException, TException, UnknownTableException,
       UnknownDBException {
-    return deepCopyFieldSchemas(client.get_schema(db, tableName));
+      EnvironmentContext envCxt = null;
+      String addedJars = conf.getVar(ConfVars.HIVEADDEDJARS);
+      if(org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) {
+         Map<String, String> props = new HashMap<String, String>();
+         props.put("hive.added.jars.path", addedJars);
+         envCxt = new EnvironmentContext(props);
+       }
+
+    return deepCopyFieldSchemas(client.get_schema_with_environment_context(db, tableName, envCxt));
   }
 
   @Override
@@ -1902,6 +1916,12 @@ public class HiveMetaStoreClient impleme
   }
 
   @Override
+  public void addDynamicPartitions(long txnId, String dbName, String tableName,
+                                   List<String> partNames) throws TException {
+    client.add_dynamic_partitions(new AddDynamicPartitions(txnId, dbName, tableName, partNames));
+  }
+
+  @Override
   public NotificationEventResponse getNextNotification(long lastEventId, int maxEvents,
                                                        NotificationFilter filter) throws TException {
     NotificationEventRequest rqst = new NotificationEventRequest(lastEventId);

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Sat Mar 28 14:03:43 2015
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore
 
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
 import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
@@ -95,6 +96,12 @@ public interface IMetaStoreClient {
   boolean isCompatibleWith(HiveConf conf);
 
   /**
+   * Set added jars path info to MetaStoreClient.
+   * @param addedJars the hive.added.jars.path. It is qualified paths separated by commas.
+   */
+  void setHiveAddedJars(String addedJars);
+
+  /**
    *  Tries to reconnect this MetaStoreClient to the MetaStore.
    */
   void reconnect() throws MetaException;
@@ -1345,6 +1352,18 @@ public interface IMetaStoreClient {
   ShowCompactResponse showCompactions() throws TException;
 
   /**
+   * Send a list of partitions to the metastore to indicate which partitions were loaded
+   * dynamically.
+   * @param txnId id of the transaction
+   * @param dbName database name
+   * @param tableName table name
+   * @param partNames partition name, as constructed by Warehouse.makePartName
+   * @throws TException
+   */
+  void addDynamicPartitions(long txnId, String dbName, String tableName, List<String> partNames)
+    throws TException;
+
+  /**
    * A filter provided by the client that determines if a given notification event should be
    * returned.
    */

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Sat Mar 28 14:03:43 2015
@@ -25,7 +25,10 @@ import java.lang.reflect.InvocationTarge
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -39,6 +42,7 @@ import java.util.regex.Pattern;
 
 import com.google.common.base.Predicates;
 import com.google.common.collect.Maps;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -559,23 +563,26 @@ public class MetaStoreUtils {
   }
 
   static boolean isCascadeNeededInAlterTable(Table oldTable, Table newTable) {
+    //currently cascade only supports add/replace columns and
+    //changing column type/position/name/comments
     List<FieldSchema> oldCols = oldTable.getSd().getCols();
     List<FieldSchema> newCols = newTable.getSd().getCols();
+    return !areSameColumns(oldCols, newCols);
+  }
 
-    //currently cascade only supports add/replace columns and
-    //changing column type/position/name/comments
+  static boolean areSameColumns(List<FieldSchema> oldCols, List<FieldSchema> newCols) {
     if (oldCols.size() != newCols.size()) {
-      return true;
+      return false;
     } else {
       for (int i = 0; i < oldCols.size(); i++) {
         FieldSchema oldCol = oldCols.get(i);
         FieldSchema newCol = newCols.get(i);
         if(!oldCol.equals(newCol)) {
-          return true;
+          return false;
         }
       }
     }
-    return false;
+    return true;
   }
 
   /**
@@ -1653,4 +1660,52 @@ public class MetaStoreUtils {
     }
   }
 
+
+  /**
+   * Create a URL from a string representing a path to a local file.
+   * The path string can be just a path, or can start with file:/, file:///
+   * @param onestr  path string
+   * @return
+   */
+  private static URL urlFromPathString(String onestr) {
+    URL oneurl = null;
+    try {
+      if (StringUtils.indexOf(onestr, "file:/") == 0) {
+        oneurl = new URL(onestr);
+      } else {
+        oneurl = new File(onestr).toURL();
+      }
+    } catch (Exception err) {
+      LOG.error("Bad URL " + onestr + ", ignoring path");
+    }
+    return oneurl;
+  }
+
+  /**
+   * Add new elements to the classpath.
+   *
+   * @param newPaths
+   *          Array of classpath elements
+   */
+  public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception {
+    URLClassLoader loader = (URLClassLoader) cloader;
+    List<URL> curPath = Arrays.asList(loader.getURLs());
+    ArrayList<URL> newPath = new ArrayList<URL>();
+
+    // get a list with the current classpath components
+    for (URL onePath : curPath) {
+      newPath.add(onePath);
+    }
+    curPath = newPath;
+
+    for (String onestr : newPaths) {
+      URL oneurl = urlFromPathString(onestr);
+      if (oneurl != null && !curPath.contains(oneurl)) {
+        curPath.add(oneurl);
+      }
+    }
+
+    return new URLClassLoader(curPath.toArray(new URL[0]), loader);
+  }
+
 }

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java Sat Mar 28 14:03:43 2015
@@ -24,6 +24,7 @@ import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang.ClassUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -93,9 +94,28 @@ public class RawStoreProxy implements In
   @Override
   public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
     Object ret = null;
+    boolean isTimerStarted = false;
 
     try {
+      try {
+        if (!Deadline.isStarted()) {
+          Deadline.startTimer(method.getName());
+          isTimerStarted = true;
+        }
+      } catch (MetaException e) {
+        // Deadline was not registered yet.
+        long timeout = HiveConf.getTimeVar(hiveConf,
+            HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
+        Deadline.registerIfNot(timeout);
+        Deadline.startTimer(method.getName());
+        isTimerStarted = true;
+      }
+
       ret = method.invoke(base, args);
+
+      if (isTimerStarted) {
+        Deadline.stopTimer();
+      }
     } catch (UndeclaredThrowableException e) {
       throw e.getCause();
     } catch (InvocationTargetException e) {

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java Sat Mar 28 14:03:43 2015
@@ -135,7 +135,8 @@ public class RetryingHMSHandler implemen
           caughtException = e.getCause();
         } else if (e.getCause() instanceof NoSuchObjectException || e.getTargetException().getCause() instanceof NoSuchObjectException) {
           String methodName = method.getName();
-          if (!methodName.startsWith("get_table") && !methodName.startsWith("get_partition") && !methodName.startsWith("get_function")) {
+          if (!methodName.startsWith("get_database") && !methodName.startsWith("get_table")
+              && !methodName.startsWith("get_partition") && !methodName.startsWith("get_function")) {
             LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
           }
           throw e.getCause();

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java Sat Mar 28 14:03:43 2015
@@ -22,7 +22,7 @@ import org.apache.hadoop.hive.metastore.
 /**
  * Information on a possible or running compaction.
  */
-public class CompactionInfo {
+public class CompactionInfo implements Comparable<CompactionInfo> {
   public long id;
   public String dbname;
   public String tableName;
@@ -68,4 +68,9 @@ public class CompactionInfo {
   public boolean isMajorCompaction() {
     return CompactionType.MAJOR == type;
   }
+
+  @Override
+  public int compareTo(CompactionInfo o) {
+    return getFullPartitionName().compareTo(o.getFullPartitionName());
+  }
 }

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Sat Mar 28 14:03:43 2015
@@ -783,6 +783,48 @@ public class TxnHandler {
     }
   }
 
+  public void addDynamicPartitions(AddDynamicPartitions rqst)
+      throws NoSuchTxnException,  TxnAbortedException, MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        // Heartbeat this first to make sure the transaction is still valid.
+        heartbeatTxn(dbConn, rqst.getTxnid());
+        for (String partName : rqst.getPartitionnames()) {
+          StringBuilder buff = new StringBuilder();
+          buff.append("insert into TXN_COMPONENTS (tc_txnid, tc_database, tc_table, tc_partition) values (");
+          buff.append(rqst.getTxnid());
+          buff.append(", '");
+          buff.append(rqst.getDbname());
+          buff.append("', '");
+          buff.append(rqst.getTablename());
+          buff.append("', '");
+          buff.append(partName);
+          buff.append("')");
+          String s = buff.toString();
+          LOG.debug("Going to execute update <" + s + ">");
+          stmt.executeUpdate(s);
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "addDynamicPartitions");
+        throw new MetaException("Unable to insert into from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      addDynamicPartitions(rqst);
+    }
+  }
+
   /**
    * For testing only, do not use.
    */

Modified: hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java (original)
+++ hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java Sat Mar 28 14:03:43 2015
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.metastore.txn;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.*;
 import org.apache.log4j.Level;
@@ -26,8 +28,11 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 import static junit.framework.Assert.*;
 
@@ -38,6 +43,7 @@ public class TestCompactionTxnHandler {
 
   private HiveConf conf = new HiveConf();
   private CompactionTxnHandler txnHandler;
+  static final private Log LOG = LogFactory.getLog(TestCompactionTxnHandler.class);
 
   public TestCompactionTxnHandler() throws Exception {
     TxnDbUtil.setConfValues(conf);
@@ -417,6 +423,40 @@ public class TestCompactionTxnHandler {
     assertEquals(3, txnList.getOpen_txnsSize());
   }
 
+  @Test
+  public void addDynamicPartitions() throws Exception {
+    String dbName = "default";
+    String tableName = "adp_table";
+    OpenTxnsResponse openTxns = txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost"));
+    long txnId = openTxns.getTxn_ids().get(0);
+    // lock a table, as in dynamic partitions
+    LockComponent lc = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, dbName);
+    lc.setTablename(tableName);
+    LockRequest lr = new LockRequest(Arrays.asList(lc), "me", "localhost");
+    lr.setTxnid(txnId);
+    LockResponse lock = txnHandler.lock(new LockRequest(Arrays.asList(lc), "me", "localhost"));
+    assertEquals(LockState.ACQUIRED, lock.getState());
+
+    txnHandler.addDynamicPartitions(new AddDynamicPartitions(txnId, dbName, tableName,
+        Arrays.asList("ds=yesterday", "ds=today")));
+    txnHandler.commitTxn(new CommitTxnRequest(txnId));
+
+    Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(1000);
+    assertEquals(2, potentials.size());
+    SortedSet<CompactionInfo> sorted = new TreeSet<CompactionInfo>(potentials);
+
+    int i = 0;
+    for (CompactionInfo ci : sorted) {
+      assertEquals(dbName, ci.dbname);
+      assertEquals(tableName, ci.tableName);
+      switch (i++) {
+      case 0: assertEquals("ds=today", ci.partName); break;
+      case 1: assertEquals("ds=yesterday", ci.partName); break;
+      default: throw new RuntimeException("What?");
+      }
+    }
+  }
+
   @Before
   public void setUp() throws Exception {
     TxnDbUtil.prepDb();

Modified: hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java (original)
+++ hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java Sat Mar 28 14:03:43 2015
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.metastore
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreThread;
 import org.apache.hadoop.hive.metastore.api.*;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;

Modified: hive/branches/spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/pom.xml?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/pom.xml (original)
+++ hive/branches/spark/pom.xml Sat Mar 28 14:03:43 2015
@@ -100,7 +100,7 @@
     <antlr.version>3.4</antlr.version>
     <avro.version>1.7.5</avro.version>
     <bonecp.version>0.8.0.RELEASE</bonecp.version>
-    <calcite.version>1.0.0-incubating</calcite.version>
+    <calcite.version>1.1.0-incubating</calcite.version>
     <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
     <datanucleus-core.version>3.2.10</datanucleus-core.version>
     <datanucleus-rdbms.version>3.2.9</datanucleus-rdbms.version>

Modified: hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (original)
+++ hive/branches/spark/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java Sat Mar 28 14:03:43 2015
@@ -10603,6 +10603,21 @@ public final class OrcProto {
      */
     org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncodingOrBuilder getColumnsOrBuilder(
         int index);
+
+    // optional string writerTimezone = 3;
+    /**
+     * <code>optional string writerTimezone = 3;</code>
+     */
+    boolean hasWriterTimezone();
+    /**
+     * <code>optional string writerTimezone = 3;</code>
+     */
+    java.lang.String getWriterTimezone();
+    /**
+     * <code>optional string writerTimezone = 3;</code>
+     */
+    com.google.protobuf.ByteString
+        getWriterTimezoneBytes();
   }
   /**
    * Protobuf type {@code orc.proto.StripeFooter}
@@ -10671,6 +10686,11 @@ public final class OrcProto {
               columns_.add(input.readMessage(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.PARSER, extensionRegistry));
               break;
             }
+            case 26: {
+              bitField0_ |= 0x00000001;
+              writerTimezone_ = input.readBytes();
+              break;
+            }
           }
         }
       } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -10716,6 +10736,7 @@ public final class OrcProto {
       return PARSER;
     }
 
+    private int bitField0_;
     // repeated .orc.proto.Stream streams = 1;
     public static final int STREAMS_FIELD_NUMBER = 1;
     private java.util.List<org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream> streams_;
@@ -10788,9 +10809,53 @@ public final class OrcProto {
       return columns_.get(index);
     }
 
+    // optional string writerTimezone = 3;
+    public static final int WRITERTIMEZONE_FIELD_NUMBER = 3;
+    private java.lang.Object writerTimezone_;
+    /**
+     * <code>optional string writerTimezone = 3;</code>
+     */
+    public boolean hasWriterTimezone() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * <code>optional string writerTimezone = 3;</code>
+     */
+    public java.lang.String getWriterTimezone() {
+      java.lang.Object ref = writerTimezone_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          writerTimezone_ = s;
+        }
+        return s;
+      }
+    }
+    /**
+     * <code>optional string writerTimezone = 3;</code>
+     */
+    public com.google.protobuf.ByteString
+        getWriterTimezoneBytes() {
+      java.lang.Object ref = writerTimezone_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        writerTimezone_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
     private void initFields() {
       streams_ = java.util.Collections.emptyList();
       columns_ = java.util.Collections.emptyList();
+      writerTimezone_ = "";
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -10810,6 +10875,9 @@ public final class OrcProto {
       for (int i = 0; i < columns_.size(); i++) {
         output.writeMessage(2, columns_.get(i));
       }
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(3, getWriterTimezoneBytes());
+      }
       getUnknownFields().writeTo(output);
     }
 
@@ -10827,6 +10895,10 @@ public final class OrcProto {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(2, columns_.get(i));
       }
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, getWriterTimezoneBytes());
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -10957,6 +11029,8 @@ public final class OrcProto {
         } else {
           columnsBuilder_.clear();
         }
+        writerTimezone_ = "";
+        bitField0_ = (bitField0_ & ~0x00000004);
         return this;
       }
 
@@ -10984,6 +11058,7 @@ public final class OrcProto {
       public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter buildPartial() {
         org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter(this);
         int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
         if (streamsBuilder_ == null) {
           if (((bitField0_ & 0x00000001) == 0x00000001)) {
             streams_ = java.util.Collections.unmodifiableList(streams_);
@@ -11002,6 +11077,11 @@ public final class OrcProto {
         } else {
           result.columns_ = columnsBuilder_.build();
         }
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.writerTimezone_ = writerTimezone_;
+        result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
       }
@@ -11069,6 +11149,11 @@ public final class OrcProto {
             }
           }
         }
+        if (other.hasWriterTimezone()) {
+          bitField0_ |= 0x00000004;
+          writerTimezone_ = other.writerTimezone_;
+          onChanged();
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -11576,6 +11661,80 @@ public final class OrcProto {
         return columnsBuilder_;
       }
 
+      // optional string writerTimezone = 3;
+      private java.lang.Object writerTimezone_ = "";
+      /**
+       * <code>optional string writerTimezone = 3;</code>
+       */
+      public boolean hasWriterTimezone() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      /**
+       * <code>optional string writerTimezone = 3;</code>
+       */
+      public java.lang.String getWriterTimezone() {
+        java.lang.Object ref = writerTimezone_;
+        if (!(ref instanceof java.lang.String)) {
+          java.lang.String s = ((com.google.protobuf.ByteString) ref)
+              .toStringUtf8();
+          writerTimezone_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * <code>optional string writerTimezone = 3;</code>
+       */
+      public com.google.protobuf.ByteString
+          getWriterTimezoneBytes() {
+        java.lang.Object ref = writerTimezone_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b = 
+              com.google.protobuf.ByteString.copyFromUtf8(
+                  (java.lang.String) ref);
+          writerTimezone_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * <code>optional string writerTimezone = 3;</code>
+       */
+      public Builder setWriterTimezone(
+          java.lang.String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        writerTimezone_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional string writerTimezone = 3;</code>
+       */
+      public Builder clearWriterTimezone() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        writerTimezone_ = getDefaultInstance().getWriterTimezone();
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional string writerTimezone = 3;</code>
+       */
+      public Builder setWriterTimezoneBytes(
+          com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        writerTimezone_ = value;
+        onChanged();
+        return this;
+      }
+
       // @@protoc_insertion_point(builder_scope:orc.proto.StripeFooter)
     }
 
@@ -18921,40 +19080,41 @@ public final class OrcProto {
       "ng\022,\n\004kind\030\001 \001(\0162\036.orc.proto.ColumnEncod" +
       "ing.Kind\022\026\n\016dictionarySize\030\002 \001(\r\"D\n\004Kind" +
       "\022\n\n\006DIRECT\020\000\022\016\n\nDICTIONARY\020\001\022\r\n\tDIRECT_V",
-      "2\020\002\022\021\n\rDICTIONARY_V2\020\003\"^\n\014StripeFooter\022\"" +
+      "2\020\002\022\021\n\rDICTIONARY_V2\020\003\"v\n\014StripeFooter\022\"" +
       "\n\007streams\030\001 \003(\0132\021.orc.proto.Stream\022*\n\007co" +
-      "lumns\030\002 \003(\0132\031.orc.proto.ColumnEncoding\"\341" +
-      "\002\n\004Type\022\"\n\004kind\030\001 \001(\0162\024.orc.proto.Type.K" +
-      "ind\022\024\n\010subtypes\030\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030" +
-      "\003 \003(\t\022\025\n\rmaximumLength\030\004 \001(\r\022\021\n\tprecisio" +
-      "n\030\005 \001(\r\022\r\n\005scale\030\006 \001(\r\"\321\001\n\004Kind\022\013\n\007BOOLE" +
-      "AN\020\000\022\010\n\004BYTE\020\001\022\t\n\005SHORT\020\002\022\007\n\003INT\020\003\022\010\n\004LO" +
-      "NG\020\004\022\t\n\005FLOAT\020\005\022\n\n\006DOUBLE\020\006\022\n\n\006STRING\020\007\022" +
-      "\n\n\006BINARY\020\010\022\r\n\tTIMESTAMP\020\t\022\010\n\004LIST\020\n\022\007\n\003",
-      "MAP\020\013\022\n\n\006STRUCT\020\014\022\t\n\005UNION\020\r\022\013\n\007DECIMAL\020" +
-      "\016\022\010\n\004DATE\020\017\022\013\n\007VARCHAR\020\020\022\010\n\004CHAR\020\021\"x\n\021St" +
-      "ripeInformation\022\016\n\006offset\030\001 \001(\004\022\023\n\013index" +
-      "Length\030\002 \001(\004\022\022\n\ndataLength\030\003 \001(\004\022\024\n\014foot" +
-      "erLength\030\004 \001(\004\022\024\n\014numberOfRows\030\005 \001(\004\"/\n\020" +
-      "UserMetadataItem\022\014\n\004name\030\001 \001(\t\022\r\n\005value\030" +
-      "\002 \001(\014\"A\n\020StripeStatistics\022-\n\010colStats\030\001 " +
-      "\003(\0132\033.orc.proto.ColumnStatistics\"<\n\010Meta" +
-      "data\0220\n\013stripeStats\030\001 \003(\0132\033.orc.proto.St" +
-      "ripeStatistics\"\222\002\n\006Footer\022\024\n\014headerLengt",
-      "h\030\001 \001(\004\022\025\n\rcontentLength\030\002 \001(\004\022-\n\007stripe" +
-      "s\030\003 \003(\0132\034.orc.proto.StripeInformation\022\036\n" +
-      "\005types\030\004 \003(\0132\017.orc.proto.Type\022-\n\010metadat" +
-      "a\030\005 \003(\0132\033.orc.proto.UserMetadataItem\022\024\n\014" +
-      "numberOfRows\030\006 \001(\004\022/\n\nstatistics\030\007 \003(\0132\033" +
-      ".orc.proto.ColumnStatistics\022\026\n\016rowIndexS" +
-      "tride\030\010 \001(\r\"\305\001\n\nPostScript\022\024\n\014footerLeng" +
-      "th\030\001 \001(\004\022/\n\013compression\030\002 \001(\0162\032.orc.prot" +
-      "o.CompressionKind\022\034\n\024compressionBlockSiz" +
-      "e\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001\022\026\n\016metadata",
-      "Length\030\005 \001(\004\022\025\n\rwriterVersion\030\006 \001(\r\022\016\n\005m" +
-      "agic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004NONE\020\000" +
-      "\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003B\"\n org.a" +
-      "pache.hadoop.hive.ql.io.orc"
+      "lumns\030\002 \003(\0132\031.orc.proto.ColumnEncoding\022\026" +
+      "\n\016writerTimezone\030\003 \001(\t\"\341\002\n\004Type\022\"\n\004kind\030" +
+      "\001 \001(\0162\024.orc.proto.Type.Kind\022\024\n\010subtypes\030" +
+      "\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030\003 \003(\t\022\025\n\rmaximum" +
+      "Length\030\004 \001(\r\022\021\n\tprecision\030\005 \001(\r\022\r\n\005scale" +
+      "\030\006 \001(\r\"\321\001\n\004Kind\022\013\n\007BOOLEAN\020\000\022\010\n\004BYTE\020\001\022\t" +
+      "\n\005SHORT\020\002\022\007\n\003INT\020\003\022\010\n\004LONG\020\004\022\t\n\005FLOAT\020\005\022" +
+      "\n\n\006DOUBLE\020\006\022\n\n\006STRING\020\007\022\n\n\006BINARY\020\010\022\r\n\tT",
+      "IMESTAMP\020\t\022\010\n\004LIST\020\n\022\007\n\003MAP\020\013\022\n\n\006STRUCT\020" +
+      "\014\022\t\n\005UNION\020\r\022\013\n\007DECIMAL\020\016\022\010\n\004DATE\020\017\022\013\n\007V" +
+      "ARCHAR\020\020\022\010\n\004CHAR\020\021\"x\n\021StripeInformation\022" +
+      "\016\n\006offset\030\001 \001(\004\022\023\n\013indexLength\030\002 \001(\004\022\022\n\n" +
+      "dataLength\030\003 \001(\004\022\024\n\014footerLength\030\004 \001(\004\022\024" +
+      "\n\014numberOfRows\030\005 \001(\004\"/\n\020UserMetadataItem" +
+      "\022\014\n\004name\030\001 \001(\t\022\r\n\005value\030\002 \001(\014\"A\n\020StripeS" +
+      "tatistics\022-\n\010colStats\030\001 \003(\0132\033.orc.proto." +
+      "ColumnStatistics\"<\n\010Metadata\0220\n\013stripeSt" +
+      "ats\030\001 \003(\0132\033.orc.proto.StripeStatistics\"\222",
+      "\002\n\006Footer\022\024\n\014headerLength\030\001 \001(\004\022\025\n\rconte" +
+      "ntLength\030\002 \001(\004\022-\n\007stripes\030\003 \003(\0132\034.orc.pr" +
+      "oto.StripeInformation\022\036\n\005types\030\004 \003(\0132\017.o" +
+      "rc.proto.Type\022-\n\010metadata\030\005 \003(\0132\033.orc.pr" +
+      "oto.UserMetadataItem\022\024\n\014numberOfRows\030\006 \001" +
+      "(\004\022/\n\nstatistics\030\007 \003(\0132\033.orc.proto.Colum" +
+      "nStatistics\022\026\n\016rowIndexStride\030\010 \001(\r\"\305\001\n\n" +
+      "PostScript\022\024\n\014footerLength\030\001 \001(\004\022/\n\013comp" +
+      "ression\030\002 \001(\0162\032.orc.proto.CompressionKin" +
+      "d\022\034\n\024compressionBlockSize\030\003 \001(\004\022\023\n\007versi",
+      "on\030\004 \003(\rB\002\020\001\022\026\n\016metadataLength\030\005 \001(\004\022\025\n\r" +
+      "writerVersion\030\006 \001(\r\022\016\n\005magic\030\300> \001(\t*:\n\017C" +
+      "ompressionKind\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SN" +
+      "APPY\020\002\022\007\n\003LZO\020\003B\"\n org.apache.hadoop.hiv" +
+      "e.ql.io.orc"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -19056,7 +19216,7 @@ public final class OrcProto {
           internal_static_orc_proto_StripeFooter_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_orc_proto_StripeFooter_descriptor,
-              new java.lang.String[] { "Streams", "Columns", });
+              new java.lang.String[] { "Streams", "Columns", "WriterTimezone", });
           internal_static_orc_proto_Type_descriptor =
             getDescriptor().getMessageTypes().get(16);
           internal_static_orc_proto_Type_fieldAccessorTable = new

Modified: hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt (original)
+++ hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ColumnUnaryMinus.txt Sat Mar 28 14:03:43 2015
@@ -130,7 +130,7 @@ public class <ClassName> extends VectorE
             VectorExpressionDescriptor.Mode.PROJECTION)
         .setNumArguments(1)
         .setArgumentTypes(
-            VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
+            VectorExpressionDescriptor.ArgumentType.getType("<VectorExprArgType>"))
         .setInputExpressionTypes(
             VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
   }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Sat Mar 28 14:03:43 2015
@@ -440,15 +440,13 @@ public class Driver implements CommandPr
       sem.validate();
       perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ANALYZE);
 
-      plan = new QueryPlan(command, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
-        SessionState.get().getCommandType());
+      // Command should be redacted before passing it to the QueryPlan in order
+      // to avoid returning sensitive data
+      String queryStr = HookUtils.redactLogString(conf, command);
+
+      plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
+          SessionState.get().getCommandType());
 
-      String queryStr = plan.getQueryStr();
-      List<Redactor> queryRedactors = getHooks(ConfVars.QUERYREDACTORHOOKS, Redactor.class);
-      for (Redactor redactor : queryRedactors) {
-        redactor.setConf(conf);
-        queryStr = redactor.redactQuery(queryStr);
-      }
       conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);
 
       conf.set("mapreduce.workflow.id", "hive_" + queryId);
@@ -973,6 +971,7 @@ public class Driver implements CommandPr
         if (txnId == SessionState.NO_CURRENT_TXN) {
           txnId = txnMgr.openTxn(userFromUGI);
           ss.setCurrentTxn(txnId);
+          LOG.debug("Setting current transaction to " + txnId);
         }
         // Set the transaction id in all of the acid file sinks
         if (acidSinks != null) {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java Sat Mar 28 14:03:43 2015
@@ -149,7 +149,7 @@ public class QueryPlan implements Serial
 
     return userid
         + "_"
-        + String.format("%1$4d%2$02d%3$02d%4$02d%5$02d%5$02d", gc
+        + String.format("%1$4d%2$02d%3$02d%4$02d%5$02d%6$02d", gc
         .get(Calendar.YEAR), gc.get(Calendar.MONTH) + 1, gc
         .get(Calendar.DAY_OF_MONTH), gc.get(Calendar.HOUR_OF_DAY), gc
         .get(Calendar.MINUTE), gc.get(Calendar.SECOND))
@@ -435,7 +435,7 @@ public class QueryPlan implements Serial
     return "\"" + key + "\":" + getJSONValue(value) + ",";
   }
 
-  @SuppressWarnings("unchecked")
+  @SuppressWarnings("rawtypes")
   private String getJSONList(List list) {
     if (list == null) {
       return "null";
@@ -451,7 +451,7 @@ public class QueryPlan implements Serial
     return sb.toString();
   }
 
-  @SuppressWarnings("unchecked")
+  @SuppressWarnings("rawtypes")
   public String getJSONMap(Map map) {
     if (map == null) {
       return "null";

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java Sat Mar 28 14:03:43 2015
@@ -17,6 +17,13 @@
  */
 package org.apache.hadoop.hive.ql.exec;
 
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Future;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -28,11 +35,6 @@ import org.apache.hadoop.hive.ql.plan.Dy
 import org.apache.hadoop.hive.ql.plan.FileMergeDesc;
 import org.apache.hadoop.mapred.JobConf;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Set;
-
 /**
  * Fast file merge operator for ORC and RCfile. This is an abstract class which
  * does not process any rows. Refer {@link org.apache.hadoop.hive.ql.exec.OrcFileMergeOperator}
@@ -63,8 +65,8 @@ public abstract class AbstractFileMergeO
   protected transient DynamicPartitionCtx dpCtx;
 
   @Override
-  public void initializeOp(Configuration hconf) throws HiveException {
-    super.initializeOp(hconf);
+  public Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+    Collection<Future<?>> result = super.initializeOp(hconf);
     this.jc = new JobConf(hconf);
     incompatFileSet = new HashSet<Path>();
     autoDelete = false;
@@ -92,6 +94,7 @@ public abstract class AbstractFileMergeO
       throw new HiveException("Failed to initialize AbstractFileMergeOperator",
           e);
     }
+    return result;
   }
 
   // sets up temp and task temp path

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java Sat Mar 28 14:03:43 2015
@@ -20,7 +20,9 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.Future;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.exec.persistence.RowContainer;
@@ -57,7 +59,7 @@ public abstract class AbstractMapJoinOpe
 
   @Override
   @SuppressWarnings("unchecked")
-  protected void initializeOp(Configuration hconf) throws HiveException {
+  protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
     if (conf.getGenJoinKeys()) {
       int tagLen = conf.getTagLength();
       joinKeys = new List[tagLen];
@@ -66,7 +68,7 @@ public abstract class AbstractMapJoinOpe
           inputObjInspectors,NOTSKIPBIGTABLE, tagLen);
     }
 
-    super.initializeOp(hconf);
+    Collection<Future<?>> result = super.initializeOp(hconf);
 
     numMapRowsRead = 0;
 
@@ -81,7 +83,7 @@ public abstract class AbstractMapJoinOpe
         !hasFilter(posBigTable), reporter);
     storage[posBigTable] = bigPosRC;
 
-    initializeChildren(hconf);
+    return result;
   }
 
   @Override

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java Sat Mar 28 14:03:43 2015
@@ -20,7 +20,9 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.concurrent.Future;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -50,11 +52,14 @@ public class AppMasterEventOperator exte
   protected transient long MAX_SIZE;
 
   @Override
-  public void initializeOp(Configuration hconf) throws HiveException {
+  public Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+    Collection<Future<?>> result = super.initializeOp(hconf);
+
     MAX_SIZE = HiveConf.getLongVar(hconf, ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE);
     serializer =
         (Serializer) ReflectionUtils.newInstance(conf.getTable().getDeserializerClass(), null);
     initDataBuffer(false);
+    return result;
   }
 
   protected void initDataBuffer(boolean skipPruning) throws HiveException {
@@ -71,7 +76,7 @@ public class AppMasterEventOperator exte
   }
 
   @Override
-  public void processOp(Object row, int tag) throws HiveException {
+  public void process(Object row, int tag) throws HiveException {
     if (hasReachedMaxSize) {
       return;
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java Sat Mar 28 14:03:43 2015
@@ -20,11 +20,13 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Future;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.ql.plan.CollectDesc;
+import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -41,16 +43,17 @@ public class CollectOperator extends Ope
   transient int maxSize;
 
   @Override
-  protected void initializeOp(Configuration hconf) throws HiveException {
-    super.initializeOp(hconf);
+  protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+    Collection<Future<?>> result = super.initializeOp(hconf);
     rowList = new ArrayList<Object>();
     maxSize = conf.getBufferSize().intValue();
+    return result;
   }
 
   boolean firstRow = true;
 
   @Override
-  public void processOp(Object row, int tag) throws HiveException {
+  public void process(Object row, int tag) throws HiveException {
     ObjectInspector rowInspector = inputObjInspectors[tag];
     if (firstRow) {
       firstRow = false;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1669775&r1=1669774&r2=1669775&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Sat Mar 28 14:03:43 2015
@@ -267,7 +267,7 @@ public class ColumnStatsTask extends Tas
 
   private List<ColumnStatistics> constructColumnStatsFromPackedRows() throws HiveException, MetaException, IOException {
 
-    String dbName = SessionState.get().getCurrentDatabase();
+    String currentDb = SessionState.get().getCurrentDatabase();
     String tableName = work.getColStats().getTableName();
     String partName = null;
     List<String> colName = work.getColStats().getColName();
@@ -286,7 +286,7 @@ public class ColumnStatsTask extends Tas
       List<? extends StructField> fields = soi.getAllStructFieldRefs();
       List<Object> list = soi.getStructFieldsDataAsList(packedRow.o);
 
-      Table tbl = db.getTable(dbName,tableName);
+      Table tbl = db.getTable(currentDb,tableName);
       List<FieldSchema> partColSchema = tbl.getPartCols();
       // Partition columns are appended at end, we only care about stats column
       int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size();
@@ -313,8 +313,8 @@ public class ColumnStatsTask extends Tas
         }
         partName = Warehouse.makePartName(partColSchema, partVals);
       }
-
-      ColumnStatisticsDesc statsDesc = getColumnStatsDesc(dbName, tableName, partName, isTblLevel);
+      String [] names = Utilities.getDbTableName(currentDb, tableName);
+      ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, isTblLevel);
       ColumnStatistics colStats = new ColumnStatistics();
       colStats.setStatsDesc(statsDesc);
       colStats.setStatsObj(statsObjs);
@@ -368,7 +368,7 @@ public class ColumnStatsTask extends Tas
         return persistPartitionStats();
       }
     } catch (Exception e) {
-        LOG.info(e);
+      LOG.error("Failed to run column stats task", e);
     }
     return 1;
   }