You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ke...@apache.org on 2012/08/25 01:27:04 UTC
svn commit: r1377146 - in /hive/trunk:
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/queries/positive/
hbase-handler/src/test/results/positive/
ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/
ql/src/test/queries/clientpo...
Author: kevinwilfong
Date: Fri Aug 24 23:27:04 2012
New Revision: 1377146
URL: http://svn.apache.org/viewvc?rev=1377146&view=rev
Log:
HIVE-3396. hive.stats.reliable config causes FileSinkOperator to fail when writing empty file. (kevinwilfong reviewed by njain)
Added:
hive/trunk/hbase-handler/src/test/queries/positive/hbase_stats_empty_partition.q
hive/trunk/hbase-handler/src/test/results/positive/hbase_stats_empty_partition.q.out
hive/trunk/ql/src/test/queries/clientpositive/stats_empty_partition.q
hive/trunk/ql/src/test/results/clientpositive/stats_empty_partition.q.out
Modified:
hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java?rev=1377146&r1=1377145&r2=1377146&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java Fri Aug 24 23:27:04 2012
@@ -68,6 +68,11 @@ public class HBaseStatsPublisher impleme
// Write in HBase
+ if (stats.isEmpty()) {
+ // If there are no stats to publish, nothing to do.
+ return true;
+ }
+
if (!HBaseStatsUtils.isValidStatisticSet(stats.keySet())) {
LOG.warn("Warning. Invalid statistic: " + stats.keySet().toString()
+ ", supported stats: "
Added: hive/trunk/hbase-handler/src/test/queries/positive/hbase_stats_empty_partition.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_stats_empty_partition.q?rev=1377146&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_stats_empty_partition.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_stats_empty_partition.q Fri Aug 24 23:27:04 2012
@@ -0,0 +1,13 @@
+-- This test verifies that writing an empty partition succeeds when
+-- hive.stats.reliable is set to true.
+
+create table tmptable(key string, value string) partitioned by (part string);
+
+set hive.stats.autogather=true;
+set hive.stats.reliable=true;
+
+set hive.stats.dbclass=hbase;
+
+insert overwrite table tmptable partition (part = '1') select * from src where key = 'no_such_value';
+
+describe formatted tmptable partition (part = '1');
Added: hive/trunk/hbase-handler/src/test/results/positive/hbase_stats_empty_partition.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_stats_empty_partition.q.out?rev=1377146&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_stats_empty_partition.q.out (added)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_stats_empty_partition.q.out Fri Aug 24 23:27:04 2012
@@ -0,0 +1,61 @@
+PREHOOK: query: -- This test verifies that writing an empty partition succeeds when
+-- hive.stats.reliable is set to true.
+
+create table tmptable(key string, value string) partitioned by (part string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- This test verifies that writing an empty partition succeeds when
+-- hive.stats.reliable is set to true.
+
+create table tmptable(key string, value string) partitioned by (part string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tmptable
+PREHOOK: query: insert overwrite table tmptable partition (part = '1') select * from src where key = 'no_such_value'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmptable@part=1
+POSTHOOK: query: insert overwrite table tmptable partition (part = '1') select * from src where key = 'no_such_value'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmptable@part=1
+POSTHOOK: Lineage: tmptable PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tmptable PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe formatted tmptable partition (part = '1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted tmptable partition (part = '1')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: tmptable PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tmptable PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+# col_name data_type comment
+
+key string None
+value string None
+
+# Partition Information
+# col_name data_type comment
+
+part string None
+
+# Detailed Partition Information
+Partition Value: [1]
+Database: default
+Table: tmptable
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ numFiles 1
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1377146&r1=1377145&r2=1377146&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Fri Aug 24 23:27:04 2012
@@ -119,6 +119,11 @@ public class JDBCStatsPublisher implemen
@Override
public boolean publishStat(String fileID, Map<String, String> stats) {
+ if (stats.isEmpty()) {
+ // If there are no stats to publish, nothing to do.
+ return true;
+ }
+
if (conn == null) {
LOG.error("JDBC connection is null. Cannot publish stats without JDBC connection.");
return false;
Added: hive/trunk/ql/src/test/queries/clientpositive/stats_empty_partition.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_empty_partition.q?rev=1377146&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_empty_partition.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_empty_partition.q Fri Aug 24 23:27:04 2012
@@ -0,0 +1,11 @@
+-- This test verifies that writing an empty partition succeeds when
+-- hive.stats.reliable is set to true.
+
+create table tmptable(key string, value string) partitioned by (part string);
+
+set hive.stats.autogather=true;
+set hive.stats.reliable=true;
+
+insert overwrite table tmptable partition (part = '1') select * from src where key = 'no_such_value';
+
+describe formatted tmptable partition (part = '1');
Added: hive/trunk/ql/src/test/results/clientpositive/stats_empty_partition.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats_empty_partition.q.out?rev=1377146&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/stats_empty_partition.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/stats_empty_partition.q.out Fri Aug 24 23:27:04 2012
@@ -0,0 +1,61 @@
+PREHOOK: query: -- This test verifies that writing an empty partition succeeds when
+-- hive.stats.reliable is set to true.
+
+create table tmptable(key string, value string) partitioned by (part string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- This test verifies that writing an empty partition succeeds when
+-- hive.stats.reliable is set to true.
+
+create table tmptable(key string, value string) partitioned by (part string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tmptable
+PREHOOK: query: insert overwrite table tmptable partition (part = '1') select * from src where key = 'no_such_value'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmptable@part=1
+POSTHOOK: query: insert overwrite table tmptable partition (part = '1') select * from src where key = 'no_such_value'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmptable@part=1
+POSTHOOK: Lineage: tmptable PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tmptable PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe formatted tmptable partition (part = '1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted tmptable partition (part = '1')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: tmptable PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tmptable PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+# col_name data_type comment
+
+key string None
+value string None
+
+# Partition Information
+# col_name data_type comment
+
+part string None
+
+# Detailed Partition Information
+Partition Value: [1]
+Database: default
+Table: tmptable
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ numFiles 1
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1