You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/07/16 09:53:07 UTC
[49/50] [abbrv] hive git commit: HIVE-11277: Merge master to parquet
06/16/2015 [Parquet branch]
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
index f7adb39,d82e93c..e1b6dd8
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
@@@ -17,10 -17,9 +17,9 @@@ import java.io.IOException
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
- import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
import org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.RecordReader;
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
index 3cd48f8,3261e4b..cb25a4f
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
@@@ -13,11 -13,12 +13,11 @@@
*/
package org.apache.hadoop.hive.ql.io.parquet.convert;
-import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.io.api.GroupConverter;
- import parquet.io.api.RecordMaterializer;
- import parquet.schema.GroupType;
- import parquet.schema.MessageType;
+ import org.apache.parquet.io.api.GroupConverter;
+ import org.apache.parquet.io.api.RecordMaterializer;
+ import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.MessageTypeParser;
++import org.apache.parquet.schema.MessageType;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
index 7915111,06f3d32..99d3643
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
@@@ -20,13 -20,13 +20,15 @@@ package org.apache.hadoop.hive.ql.io.pa
import com.google.common.base.Preconditions;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.io.api.Converter;
- import parquet.schema.GroupType;
- import parquet.schema.Type;
+ import org.apache.hadoop.io.ArrayWritable;
+ import org.apache.hadoop.io.Writable;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.Type;
public class HiveCollectionConverter extends HiveGroupConverter {
private final GroupType collectionType;
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
index c22c045,b1ca85a..30001cb
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
@@@ -13,14 -13,14 +13,13 @@@
*/
package org.apache.hadoop.hive.ql.io.parquet.convert;
--import org.apache.hadoop.io.Writable;
- import parquet.io.api.Converter;
- import parquet.io.api.GroupConverter;
- import parquet.io.api.PrimitiveConverter;
- import parquet.schema.GroupType;
- import parquet.schema.OriginalType;
- import parquet.schema.PrimitiveType;
- import parquet.schema.Type;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.io.api.GroupConverter;
+ import org.apache.parquet.io.api.PrimitiveConverter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.OriginalType;
+ import org.apache.parquet.schema.PrimitiveType;
+ import org.apache.parquet.schema.Type;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
index 4d06c36,9c35a9f..bd706fa
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
@@@ -18,10 -17,11 +18,10 @@@ import java.util.Arrays
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.io.ArrayWritable;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.io.api.Converter;
- import parquet.schema.GroupType;
- import parquet.schema.Type;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.Type;
/**
*
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
index c84caec,c0af291..3b36eaf
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
@@@ -22,13 -22,15 +22,13 @@@ import java.util.ArrayList
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.io.ArrayWritable;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.column.Dictionary;
- import parquet.io.api.Binary;
- import parquet.io.api.Converter;
- import parquet.io.api.PrimitiveConverter;
- import parquet.schema.GroupType;
- import parquet.schema.PrimitiveType;
+ import org.apache.parquet.column.Dictionary;
+ import org.apache.parquet.io.api.Binary;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.io.api.PrimitiveConverter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.PrimitiveType;
/**
* Converters for repeated fields need to know when the parent field starts and
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
index a5856bf,8b8b92d..c5ad201
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
@@@ -29,17 -29,17 +29,17 @@@ import org.apache.hadoop.hive.serde2.ob
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
import org.apache.hadoop.util.StringUtils;
- import parquet.hadoop.api.InitContext;
- import parquet.hadoop.api.ReadSupport;
- import parquet.io.api.RecordMaterializer;
- import parquet.schema.GroupType;
- import parquet.schema.MessageType;
- import parquet.schema.Type;
- import parquet.schema.Types;
- import parquet.schema.PrimitiveType.PrimitiveTypeName;
+ import org.apache.parquet.hadoop.api.InitContext;
+ import org.apache.parquet.hadoop.api.ReadSupport;
+ import org.apache.parquet.io.api.RecordMaterializer;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.MessageType;
+ import org.apache.parquet.schema.Type;
+ import org.apache.parquet.schema.Types;
+ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
/**
*
@@@ -246,9 -245,9 +246,9 @@@ public class DataWritableReadSupport ex
* @return Record Materialize for Hive
*/
@Override
- public RecordMaterializer<ArrayWritable> prepareForRead(final Configuration configuration,
+ public RecordMaterializer<ObjectArrayWritable> prepareForRead(final Configuration configuration,
final Map<String, String> keyValueMetaData, final MessageType fileSchema,
- final parquet.hadoop.api.ReadSupport.ReadContext readContext) {
+ final org.apache.parquet.hadoop.api.ReadSupport.ReadContext readContext) {
final Map<String, String> metadata = readContext.getReadSupportMetadata();
if (metadata == null) {
throw new IllegalStateException("ReadContext not initialized properly. " +
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
index dc4f896,e1bf8e2..58eabf0
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
@@@ -34,10 -33,11 +34,10 @@@ import org.apache.hadoop.hive.serde2.ty
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
- import parquet.hadoop.ParquetOutputFormat;
- import parquet.hadoop.ParquetWriter;
+ import org.apache.parquet.hadoop.ParquetOutputFormat;
+ import org.apache.parquet.hadoop.ParquetWriter;
/**
*
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
index 17d8feb,4b69642..e3e875e
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
@@@ -20,18 -20,22 +20,18 @@@ package org.apache.hadoop.hive.ql.io.pa
import java.util.List;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.io.ArrayWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
import org.junit.Assert;
import org.junit.Test;
- import parquet.io.api.RecordConsumer;
- import parquet.schema.MessageType;
- import parquet.schema.Types;
+ import org.apache.parquet.io.api.RecordConsumer;
+ import org.apache.parquet.schema.MessageType;
+ import org.apache.parquet.schema.Types;
- import static parquet.schema.OriginalType.LIST;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+ import static org.apache.parquet.schema.OriginalType.LIST;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
public class TestArrayCompatibility extends AbstractTestParquetDirect {
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
index 5f586e4,7049139..71f4acf
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
@@@ -353,21 -358,19 +353,19 @@@ public class TestDataWritableWriter
+ " }\n"
+ "}\n";
- ArrayWritable hiveRecord = createGroup(
+ ObjectArrayWritable hiveRecord = createGroup(
createGroup(
createArray(
- createArray(
- createString("key1"),
- createInt(1)
- ),
- createArray(
- createString("key2"),
- createInt(2)
- ),
- createArray(
- createString("key3"),
- createNull()
- )
+ createString("key1"),
+ createInt(1)
+ ),
+ createArray(
+ createString("key2"),
+ createInt(2)
+ ),
+ createArray(
+ createString("key3"),
+ createNull()
)
)
);
@@@ -424,15 -427,11 +422,11 @@@
+ " }\n"
+ "}\n";
- ArrayWritable hiveRecord = createGroup(
- createArray(
+ ObjectArrayWritable hiveRecord = createGroup(
+ createGroup(
createArray(
- createGroup(
- createArray(
- createInt(1),
- createInt(2)
- )
- )
+ createInt(1),
+ createInt(2)
)
)
);
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
index 1b98cd3,518eefd..fdf4b0f
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
@@@ -15,10 -15,10 +15,10 @@@ package org.apache.hadoop.hive.ql.io.pa
import static org.mockito.Mockito.mock;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
import org.junit.Test;
- import parquet.hadoop.ParquetInputFormat;
+ import org.apache.parquet.hadoop.ParquetInputFormat;
public class TestMapredParquetInputFormat {
@Test
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRowGroupFilter.java
----------------------------------------------------------------------