You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/01/09 08:41:19 UTC
svn commit: r1650453 [1/9] - in /hive/branches/spark: ./
common/src/java/org/apache/hadoop/hive/conf/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/
hcatalog/server-exten...
Author: szehon
Date: Fri Jan 9 07:41:17 2015
New Revision: 1650453
URL: http://svn.apache.org/r1650453
Log:
HIVE-9323 : Merge from trunk to spark 1/8/2015
Added:
hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
- copied unchanged from r1650429, hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
hive/branches/spark/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/
- copied from r1650429, hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/
hive/branches/spark/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CurrentNotificationEventId.java
- copied unchanged from r1650429, hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CurrentNotificationEventId.java
hive/branches/spark/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
- copied unchanged from r1650429, hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
hive/branches/spark/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventRequest.java
- copied unchanged from r1650429, hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventRequest.java
hive/branches/spark/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventResponse.java
- copied unchanged from r1650429, hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventResponse.java
hive/branches/spark/metastore/src/model/org/apache/hadoop/hive/metastore/model/MNotificationLog.java
- copied unchanged from r1650429, hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MNotificationLog.java
hive/branches/spark/metastore/src/model/org/apache/hadoop/hive/metastore/model/MNotificationNextId.java
- copied unchanged from r1650429, hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MNotificationNextId.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
- copied unchanged from r1650429, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
hive/branches/spark/ql/src/test/queries/clientpositive/expr_cached.q
- copied unchanged from r1650429, hive/trunk/ql/src/test/queries/clientpositive/expr_cached.q
hive/branches/spark/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q
- copied unchanged from r1650429, hive/trunk/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q
hive/branches/spark/ql/src/test/queries/clientpositive/union_remove_6_subq.q
- copied unchanged from r1650429, hive/trunk/ql/src/test/queries/clientpositive/union_remove_6_subq.q
hive/branches/spark/ql/src/test/queries/clientpositive/vector_char_mapjoin1.q
- copied unchanged from r1650429, hive/trunk/ql/src/test/queries/clientpositive/vector_char_mapjoin1.q
hive/branches/spark/ql/src/test/queries/clientpositive/vector_varchar_mapjoin1.q
- copied unchanged from r1650429, hive/trunk/ql/src/test/queries/clientpositive/vector_varchar_mapjoin1.q
hive/branches/spark/ql/src/test/results/clientpositive/expr_cached.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/expr_cached.q.out
hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
- copied unchanged from r1650429, hive/trunk/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
Modified:
hive/branches/spark/ (props changed)
hive/branches/spark/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/spark/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
hive/branches/spark/itests/hcatalog-unit/pom.xml
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
hive/branches/spark/metastore/if/hive_metastore.thrift
hive/branches/spark/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
hive/branches/spark/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
hive/branches/spark/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
hive/branches/spark/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
hive/branches/spark/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
hive/branches/spark/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
hive/branches/spark/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
hive/branches/spark/metastore/src/gen/thrift/gen-php/metastore/Types.php
hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
hive/branches/spark/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
hive/branches/spark/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
hive/branches/spark/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
hive/branches/spark/metastore/src/model/package.jdo
hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
hive/branches/spark/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
hive/branches/spark/ql/src/test/queries/clientpositive/groupby_cube1.q
hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join16.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join17.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join19.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join22.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join27.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join33.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join8.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join9.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join_nulls.q.out
hive/branches/spark/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/cross_product_check_2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/explain_logical.q.out
hive/branches/spark/ql/src/test/results/clientpositive/groupby_cube1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out
hive/branches/spark/ql/src/test/results/clientpositive/groupby_rollup1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out
hive/branches/spark/ql/src/test/results/clientpositive/input23.q.out
hive/branches/spark/ql/src/test/results/clientpositive/join17.q.out
hive/branches/spark/ql/src/test/results/clientpositive/join32_lessSize.q.out
hive/branches/spark/ql/src/test/results/clientpositive/join34.q.out
hive/branches/spark/ql/src/test/results/clientpositive/join35.q.out
hive/branches/spark/ql/src/test/results/clientpositive/join7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/join9.q.out
hive/branches/spark/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hive/branches/spark/ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out
hive/branches/spark/ql/src/test/results/clientpositive/metadataonly1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/multiMapJoin2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/ppd_join_filter.q.out
hive/branches/spark/ql/src/test/results/clientpositive/ppd_union_view.q.out
hive/branches/spark/ql/src/test/results/clientpositive/ppd_vc.q.out
hive/branches/spark/ql/src/test/results/clientpositive/router_join_ppr.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/auto_join7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/subquery_views.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/cross_join.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vector_left_outer_join.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out
hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java
Propchange: hive/branches/spark/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 9 07:41:17 2015
@@ -2,4 +2,4 @@
/hive/branches/cbo:1605012-1627125
/hive/branches/tez:1494760-1622766
/hive/branches/vectorization:1466908-1527856
-/hive/trunk:1608589-1649690,1650434
+/hive/trunk:1608589-1650429
Modified: hive/branches/spark/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/spark/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Jan 9 07:41:17 2015
@@ -421,6 +421,9 @@ public class HiveConf extends Configurat
"If true, the metastore Thrift interface will be secured with SASL. Clients must authenticate with Kerberos."),
METASTORE_USE_THRIFT_FRAMED_TRANSPORT("hive.metastore.thrift.framed.transport.enabled", false,
"If true, the metastore Thrift interface will use TFramedTransport. When false (default) a standard TTransport is used."),
+ METASTORE_USE_THRIFT_COMPACT_PROTOCOL("hive.metastore.thrift.compact.protocol.enabled", false,
+ "If true, the metastore Thrift interface will use TCompactProtocol. When false (default) TBinaryProtocol will be used.\n" +
+ "Setting it to true will break compatibility with older clients running TBinaryProtocol."),
METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS("hive.cluster.delegation.token.store.class",
"org.apache.hadoop.hive.thrift.MemoryTokenStore",
"The delegation token store implementation. Set to org.apache.hadoop.hive.thrift.ZooKeeperTokenStore for load-balanced cluster."),
@@ -486,6 +489,9 @@ public class HiveConf extends Configurat
METASTORE_PRE_EVENT_LISTENERS("hive.metastore.pre.event.listeners", "",
"List of comma separated listeners for metastore events."),
METASTORE_EVENT_LISTENERS("hive.metastore.event.listeners", "", ""),
+ METASTORE_EVENT_DB_LISTENER_TTL("hive.metastore.event.db.listener.timetolive", "86400s",
+ new TimeValidator(TimeUnit.SECONDS),
+ "time after which events will be removed from the database listener queue"),
METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS("hive.metastore.authorization.storage.checks", false,
"Should the metastore do authorization checks against the underlying storage (usually hdfs) \n" +
"for operations like drop-partition (disallow the drop-partition if the user in\n" +
@@ -870,6 +876,11 @@ public class HiveConf extends Configurat
HIVE_RCFILE_TOLERATE_CORRUPTIONS("hive.io.rcfile.tolerate.corruptions", false, ""),
HIVE_RCFILE_RECORD_BUFFER_SIZE("hive.io.rcfile.record.buffer.size", 4194304, ""), // 4M
+ PARQUET_MEMORY_POOL_RATIO("parquet.memory.pool.ratio", 0.5f,
+ "Maximum fraction of heap that can be used by Parquet file writers in one task.\n" +
+ "It is for avoiding OutOfMemory error in tasks. Work with Parquet 1.6.0 and above.\n" +
+ "This config parameter is defined in Parquet, so that it does not start with 'hive.'."),
+
HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f,
"Maximum fraction of heap that can be used by ORC file writers"),
HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null,
Modified: hive/branches/spark/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hive/branches/spark/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Fri Jan 9 07:41:17 2015
@@ -488,6 +488,14 @@ public class HBaseStorageHandler extends
}
}
+ private static Class counterClass = null;
+ static {
+ try {
+ counterClass = Class.forName("org.cliffc.high_scale_lib.Counter");
+ } catch (ClassNotFoundException cnfe) {
+ // this dependency is removed for HBase 1.0
+ }
+ }
@Override
public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
try {
@@ -498,9 +506,13 @@ public class HBaseStorageHandler extends
* will not be required once Hive bumps up its hbase version). At that time , we will
* only need TableMapReduceUtil.addDependencyJars(jobConf) here.
*/
- TableMapReduceUtil.addDependencyJars(
- jobConf, HBaseStorageHandler.class, TableInputFormatBase.class,
- org.cliffc.high_scale_lib.Counter.class); // this will be removed for HBase 1.0
+ if (counterClass != null) {
+ TableMapReduceUtil.addDependencyJars(
+ jobConf, HBaseStorageHandler.class, TableInputFormatBase.class, counterClass);
+ } else {
+ TableMapReduceUtil.addDependencyJars(
+ jobConf, HBaseStorageHandler.class, TableInputFormatBase.class);
+ }
Set<String> merged = new LinkedHashSet<String>(jobConf.getStringCollection("tmpjars"));
Job copy = new Job(jobConf);
Modified: hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java (original)
+++ hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java Fri Jan 9 07:41:17 2015
@@ -19,13 +19,18 @@
package org.apache.hive.hcatalog.messaging.json;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hive.hcatalog.messaging.AddPartitionMessage;
+// TODO, once HIVE-9175 is committed
+// import org.apache.hive.hcatalog.messaging.AlterTableMessage;
import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage;
import org.apache.hive.hcatalog.messaging.CreateTableMessage;
import org.apache.hive.hcatalog.messaging.DropDatabaseMessage;
@@ -42,6 +47,9 @@ import java.util.*;
*/
public class JSONMessageFactory extends MessageFactory {
+ private static final Log LOG = LogFactory.getLog(JSONMessageFactory.class.getName());
+
+
private static JSONMessageDeserializer deserializer = new JSONMessageDeserializer();
@Override
@@ -62,31 +70,40 @@ public class JSONMessageFactory extends
@Override
public CreateDatabaseMessage buildCreateDatabaseMessage(Database db) {
return new JSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
- System.currentTimeMillis() / 1000);
+ now());
}
@Override
public DropDatabaseMessage buildDropDatabaseMessage(Database db) {
return new JSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
- System.currentTimeMillis() / 1000);
+ now());
}
@Override
public CreateTableMessage buildCreateTableMessage(Table table) {
return new JSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
- table.getTableName(), System.currentTimeMillis()/1000);
+ table.getTableName(), now());
}
+ // TODO Once HIVE-9175 is committed
+ /*
+ @Override
+ public AlterTableMessage buildAlterTableMessage(Table before, Table after) {
+ return new JSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), now());
+ }
+ */
+
@Override
public DropTableMessage buildDropTableMessage(Table table) {
return new JSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), table.getTableName(),
- System.currentTimeMillis()/1000);
+ now());
}
@Override
public AddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) {
return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
- table.getTableName(), getPartitionKeyValues(table, partitions), System.currentTimeMillis()/1000);
+ table.getTableName(), getPartitionKeyValues(table, partitions), now());
}
@Override
@@ -100,8 +117,11 @@ public class JSONMessageFactory extends
@Override
public DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) {
return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
- partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
- System.currentTimeMillis()/1000);
+ partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), now());
+ }
+
+ private long now() {
+ return System.currentTimeMillis() / 1000;
}
private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) {
Modified: hive/branches/spark/itests/hcatalog-unit/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hcatalog-unit/pom.xml?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/itests/hcatalog-unit/pom.xml (original)
+++ hive/branches/spark/itests/hcatalog-unit/pom.xml Fri Jan 9 07:41:17 2015
@@ -60,6 +60,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-server-extensions</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-hbase-handler</artifactId>
<version>${project.version}</version>
Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Fri Jan 9 07:41:17 2015
@@ -178,6 +178,7 @@ minitez.query.files.shared=alter_merge_2
vector_bucket.q,\
vector_cast_constant.q,\
vector_char_4.q,\
+ vector_char_mapjoin1.q,\
vector_char_simple.q,\
vector_coalesce.q,\
vector_coalesce_2.q,\
@@ -212,6 +213,7 @@ minitez.query.files.shared=alter_merge_2
vector_reduce_groupby_decimal.q,\
vector_string_concat.q,\
vector_varchar_4.q,\
+ vector_varchar_mapjoin1.q,\
vector_varchar_simple.q,\
vectorization_0.q,\
vectorization_1.q,\
Modified: hive/branches/spark/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/if/hive_metastore.thrift?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/metastore/if/hive_metastore.thrift (original)
+++ hive/branches/spark/metastore/if/hive_metastore.thrift Fri Jan 9 07:41:17 2015
@@ -642,6 +642,29 @@ struct ShowCompactResponse {
1: required list<ShowCompactResponseElement> compacts,
}
+struct NotificationEventRequest {
+ 1: required i64 lastEvent,
+ 2: optional i32 maxEvents,
+}
+
+struct NotificationEvent {
+ 1: required i64 eventId,
+ 2: required i32 eventTime,
+ 3: required string eventType,
+ 4: optional string dbName,
+ 5: optional string tableName,
+ 6: required string message,
+}
+
+struct NotificationEventResponse {
+ 1: required list<NotificationEvent> events,
+}
+
+struct CurrentNotificationEventId {
+ 1: required i64 eventId,
+}
+
+
exception MetaException {
1: string message
}
@@ -1107,6 +1130,10 @@ service ThriftHiveMetastore extends fb30
HeartbeatTxnRangeResponse heartbeat_txn_range(1:HeartbeatTxnRangeRequest txns)
void compact(1:CompactionRequest rqst)
ShowCompactResponse show_compact(1:ShowCompactRequest rqst)
+
+ // Notification logging calls
+ NotificationEventResponse getNextNotification(1:NotificationEventRequest rqst)
+ CurrentNotificationEventId getCurrentNotificationEventId()
}
// * Note about the DDL_TIME: When creating or altering a table or a partition,