You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2013/04/09 02:43:45 UTC
svn commit: r1465849 - in /hive/trunk:
ql/src/test/queries/clientpositive/avro_compression_enabled.q
ql/src/test/results/clientpositive/avro_compression_enabled.q.out
serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
Author: navis
Date: Tue Apr 9 00:43:45 2013
New Revision: 1465849
URL: http://svn.apache.org/r1465849
Log:
HIVE-3308 Mixing avro and snappy gives null values (Bennie Schut via Navis)
Added:
hive/trunk/ql/src/test/queries/clientpositive/avro_compression_enabled.q
hive/trunk/ql/src/test/results/clientpositive/avro_compression_enabled.q.out
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
Added: hive/trunk/ql/src/test/queries/clientpositive/avro_compression_enabled.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/avro_compression_enabled.q?rev=1465849&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/avro_compression_enabled.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/avro_compression_enabled.q Tue Apr 9 00:43:45 2013
@@ -0,0 +1,43 @@
+-- verify that new joins bring in correct schemas (including evolved schemas)
+
+CREATE TABLE doctors4
+ROW FORMAT
+SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+TBLPROPERTIES ('avro.schema.literal'='{
+ "namespace": "testing.hive.avro.serde",
+ "name": "doctors",
+ "type": "record",
+ "fields": [
+ {
+ "name":"number",
+ "type":"int",
+ "doc":"Order of playing the role"
+ },
+ {
+ "name":"first_name",
+ "type":"string",
+ "doc":"first name of actor playing role"
+ },
+ {
+ "name":"last_name",
+ "type":"string",
+ "doc":"last name of actor playing role"
+ },
+ {
+ "name":"extra_field",
+ "type":"string",
+ "doc:":"an extra field not in the original file",
+ "default":"fishfingers and custard"
+ }
+ ]
+}');
+
+LOAD DATA LOCAL INPATH '../data/files/doctors.avro' INTO TABLE doctors4;
+
+set hive.exec.compress.output=true;
+
+select count(*) from src;
+
Added: hive/trunk/ql/src/test/results/clientpositive/avro_compression_enabled.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/avro_compression_enabled.q.out?rev=1465849&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/avro_compression_enabled.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/avro_compression_enabled.q.out Tue Apr 9 00:43:45 2013
@@ -0,0 +1,90 @@
+PREHOOK: query: -- verify that new joins bring in correct schemas (including evolved schemas)
+
+CREATE TABLE doctors4
+ROW FORMAT
+SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+TBLPROPERTIES ('avro.schema.literal'='{
+ "namespace": "testing.hive.avro.serde",
+ "name": "doctors",
+ "type": "record",
+ "fields": [
+ {
+ "name":"number",
+ "type":"int",
+ "doc":"Order of playing the role"
+ },
+ {
+ "name":"first_name",
+ "type":"string",
+ "doc":"first name of actor playing role"
+ },
+ {
+ "name":"last_name",
+ "type":"string",
+ "doc":"last name of actor playing role"
+ },
+ {
+ "name":"extra_field",
+ "type":"string",
+ "doc:":"an extra field not in the original file",
+ "default":"fishfingers and custard"
+ }
+ ]
+}')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- verify that new joins bring in correct schemas (including evolved schemas)
+
+CREATE TABLE doctors4
+ROW FORMAT
+SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+TBLPROPERTIES ('avro.schema.literal'='{
+ "namespace": "testing.hive.avro.serde",
+ "name": "doctors",
+ "type": "record",
+ "fields": [
+ {
+ "name":"number",
+ "type":"int",
+ "doc":"Order of playing the role"
+ },
+ {
+ "name":"first_name",
+ "type":"string",
+ "doc":"first name of actor playing role"
+ },
+ {
+ "name":"last_name",
+ "type":"string",
+ "doc":"last name of actor playing role"
+ },
+ {
+ "name":"extra_field",
+ "type":"string",
+ "doc:":"an extra field not in the original file",
+ "default":"fishfingers and custard"
+ }
+ ]
+}')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@doctors4
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/doctors.avro' INTO TABLE doctors4
+PREHOOK: type: LOAD
+PREHOOK: Output: default@doctors4
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/doctors.avro' INTO TABLE doctors4
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@doctors4
+PREHOOK: query: select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java?rev=1465849&r1=1465848&r2=1465849&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java Tue Apr 9 00:43:45 2013
@@ -61,8 +61,6 @@ public class AvroSerDe extends AbstractS
if(configuration == null) {
LOG.info("Configuration null, not inserting schema");
} else {
- // force output files to have a .avro extension
- configuration.set("hive.output.file.extension", ".avro");
configuration.set(AvroSerdeUtils.AVRO_SERDE_SCHEMA, schema.toString(false));
}