You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2015/05/27 21:29:12 UTC
hive git commit: HIVE-9605: Remove parquet nested objects from
wrapper writable objects (Sergio Pena, reviewed by Ferdinand Xu)
Repository: hive
Updated Branches:
refs/heads/master 14a046b03 -> 175e2863a
HIVE-9605: Remove parquet nested objects from wrapper writable objects (Sergio Pena, reviewed by Ferdinand Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/175e2863
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/175e2863
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/175e2863
Branch: refs/heads/master
Commit: 175e2863a22b97fa4e465e75e8956dd2183775ac
Parents: 14a046b
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed May 27 14:27:58 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed May 27 14:27:58 2015 -0500
----------------------------------------------------------------------
.../benchmark/storage/ColumnarStorageBench.java | 4 +-
.../convert/HiveCollectionConverter.java | 5 +--
.../io/parquet/convert/HiveGroupConverter.java | 10 -----
.../hive/ql/io/parquet/convert/Repeated.java | 8 ++--
.../serde/AbstractParquetMapInspector.java | 13 +++---
.../serde/DeepParquetHiveMapInspector.java | 6 +--
.../serde/ParquetHiveArrayInspector.java | 43 +++++--------------
.../serde/StandardParquetHiveMapInspector.java | 6 +--
.../ql/io/parquet/TestArrayCompatibility.java | 44 ++++++++++----------
.../ql/io/parquet/TestDataWritableWriter.java | 44 ++++++++------------
.../hive/ql/io/parquet/TestMapStructures.java | 36 ++++++++--------
.../hive/ql/io/parquet/TestParquetSerDe.java | 8 +---
.../serde/TestAbstractParquetMapInspector.java | 4 +-
.../serde/TestDeepParquetHiveMapInspector.java | 4 +-
.../serde/TestParquetHiveArrayInspector.java | 3 +-
.../TestStandardParquetHiveMapInspector.java | 4 +-
16 files changed, 92 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
----------------------------------------------------------------------
diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
index 61c2eb4..4f6985c 100644
--- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
+++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
@@ -209,13 +209,13 @@ public class ColumnarStorageBench {
case LIST: {
List<TypeInfo> elementType = new ArrayList<TypeInfo>();
elementType.add(((ListTypeInfo) type).getListElementTypeInfo());
- fields[pos++] = record(createRecord(elementType));
+ fields[pos++] = createRecord(elementType);
} break;
case MAP: {
List<TypeInfo> keyValueType = new ArrayList<TypeInfo>();
keyValueType.add(((MapTypeInfo) type).getMapKeyTypeInfo());
keyValueType.add(((MapTypeInfo) type).getMapValueTypeInfo());
- fields[pos++] = record(record(createRecord(keyValueType)));
+ fields[pos++] = record(createRecord(keyValueType));
} break;
case STRUCT: {
List<TypeInfo> elementType = ((StructTypeInfo) type).getAllStructFieldTypeInfos();
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
index 0fd538e..80405bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.io.parquet.convert;
import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Writable;
@@ -83,8 +82,8 @@ public class HiveCollectionConverter extends HiveGroupConverter {
@Override
public void end() {
- parent.set(index, wrapList(new ArrayWritable(
- Writable.class, list.toArray(new Writable[list.size()]))));
+ parent.set(index, new ArrayWritable(
+ Writable.class, list.toArray(new Writable[0])));
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
index 4809f9b..c6d03a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
@@ -13,7 +13,6 @@
*/
package org.apache.hadoop.hive.ql.io.parquet.convert;
-import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Writable;
import parquet.io.api.Converter;
import parquet.io.api.GroupConverter;
@@ -72,15 +71,6 @@ public abstract class HiveGroupConverter extends GroupConverter implements Conve
return getConverterFromDescription(type.asGroupType(), index, parent);
}
- /**
- * The original list and map conversion didn't remove the synthetic layer and
- * the ObjectInspector had to remove it. This is a temporary fix that adds an
- * extra layer for the ObjectInspector to remove.
- */
- static ArrayWritable wrapList(ArrayWritable list) {
- return new ArrayWritable(Writable.class, new Writable[] {list});
- }
-
public abstract void set(int index, Writable value);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
index 0130aef..4becd20 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
@@ -125,8 +125,8 @@ public interface Repeated extends ConverterParent {
@Override
public void parentEnd() {
- parent.set(index, HiveGroupConverter.wrapList(new ArrayWritable(
- Writable.class, list.toArray(new Writable[list.size()]))));
+ parent.set(index, new ArrayWritable(
+ Writable.class, list.toArray(new Writable[list.size()])));
}
@Override
@@ -185,8 +185,8 @@ public interface Repeated extends ConverterParent {
@Override
public void parentEnd() {
- parent.set(index, wrapList(new ArrayWritable(
- Writable.class, list.toArray(new Writable[list.size()]))));
+ parent.set(index, new ArrayWritable(
+ Writable.class, list.toArray(new Writable[list.size()])));
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
index 62c61fc..49bf1c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
@@ -59,15 +59,12 @@ public abstract class AbstractParquetMapInspector implements SettableMapObjectIn
}
if (data instanceof ArrayWritable) {
- final Writable[] mapContainer = ((ArrayWritable) data).get();
-
- if (mapContainer == null || mapContainer.length == 0) {
+ final Writable[] mapArray = ((ArrayWritable) data).get();
+ if (mapArray == null || mapArray.length == 0) {
return null;
}
- final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get();
final Map<Writable, Writable> map = new LinkedHashMap<Writable, Writable>();
-
for (final Writable obj : mapArray) {
final ArrayWritable mapObj = (ArrayWritable) obj;
final Writable[] arr = mapObj.get();
@@ -91,12 +88,12 @@ public abstract class AbstractParquetMapInspector implements SettableMapObjectIn
}
if (data instanceof ArrayWritable) {
- final Writable[] mapContainer = ((ArrayWritable) data).get();
+ final Writable[] mapArray = ((ArrayWritable) data).get();
- if (mapContainer == null || mapContainer.length == 0) {
+ if (mapArray == null || mapArray.length == 0) {
return -1;
} else {
- return ((ArrayWritable) mapContainer[0]).get().length;
+ return mapArray.length;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
index d38c641..143d72e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
@@ -40,14 +40,12 @@ public class DeepParquetHiveMapInspector extends AbstractParquetMapInspector {
}
if (data instanceof ArrayWritable) {
- final Writable[] mapContainer = ((ArrayWritable) data).get();
+ final Writable[] mapArray = ((ArrayWritable) data).get();
- if (mapContainer == null || mapContainer.length == 0) {
+ if (mapArray == null || mapArray.length == 0) {
return null;
}
- final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get();
-
for (final Writable obj : mapArray) {
final ArrayWritable mapObj = (ArrayWritable) obj;
final Writable[] arr = mapObj.get();
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
index 53ca31d..bde0dcb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
@@ -56,20 +56,13 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector {
}
if (data instanceof ArrayWritable) {
- final Writable[] listContainer = ((ArrayWritable) data).get();
-
- if (listContainer == null || listContainer.length == 0) {
- return null;
- }
-
- final Writable subObj = listContainer[0];
-
- if (subObj == null) {
+ final Writable[] array = ((ArrayWritable) data).get();
+ if (array == null || array.length == 0) {
return null;
}
- if (index >= 0 && index < ((ArrayWritable) subObj).get().length) {
- return ((ArrayWritable) subObj).get()[index];
+ if (index >= 0 && index < array.length) {
+ return array[index];
} else {
return null;
}
@@ -85,19 +78,12 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector {
}
if (data instanceof ArrayWritable) {
- final Writable[] listContainer = ((ArrayWritable) data).get();
-
- if (listContainer == null || listContainer.length == 0) {
+ final Writable[] array = ((ArrayWritable) data).get();
+ if (array == null || array.length == 0) {
return -1;
}
- final Writable subObj = listContainer[0];
-
- if (subObj == null) {
- return 0;
- }
-
- return ((ArrayWritable) subObj).get().length;
+ return array.length;
}
throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName());
@@ -110,21 +96,12 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector {
}
if (data instanceof ArrayWritable) {
- final Writable[] listContainer = ((ArrayWritable) data).get();
-
- if (listContainer == null || listContainer.length == 0) {
+ final Writable[] array = ((ArrayWritable) data).get();
+ if (array == null || array.length == 0) {
return null;
}
- final Writable subObj = listContainer[0];
-
- if (subObj == null) {
- return null;
- }
-
- final Writable[] array = ((ArrayWritable) subObj).get();
- final List<Writable> list = new ArrayList<Writable>();
-
+ final List<Writable> list = new ArrayList<Writable>(array.length);
for (final Writable obj : array) {
list.add(obj);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
index 5aa1448..22250b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
@@ -37,12 +37,12 @@ public class StandardParquetHiveMapInspector extends AbstractParquetMapInspector
return null;
}
if (data instanceof ArrayWritable) {
- final Writable[] mapContainer = ((ArrayWritable) data).get();
+ final Writable[] mapArray = ((ArrayWritable) data).get();
- if (mapContainer == null || mapContainer.length == 0) {
+ if (mapArray == null || mapArray.length == 0) {
return null;
}
- final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get();
+
for (final Writable obj : mapArray) {
final ArrayWritable mapObj = (ArrayWritable) obj;
final Writable[] arr = mapObj.get();
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
index f349aa0..e0ea262 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
@@ -62,8 +62,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
- new IntWritable(34), new IntWritable(35), new IntWritable(36)));
+ ArrayWritable expected = list(
+ new IntWritable(34), new IntWritable(35), new IntWritable(36));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -109,9 +109,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new FloatWritable(1.0f), new FloatWritable(1.0f)),
- record(new FloatWritable(2.0f), new FloatWritable(2.0f))));
+ record(new FloatWritable(2.0f), new FloatWritable(2.0f)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -148,8 +148,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
- new IntWritable(34), new IntWritable(35), new IntWritable(36)));
+ ArrayWritable expected = list(
+ new IntWritable(34), new IntWritable(35), new IntWritable(36));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -198,9 +198,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new LongWritable(1234L)),
- record(new LongWritable(2345L))));
+ record(new LongWritable(2345L)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -237,8 +237,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
- new IntWritable(34), new IntWritable(35), new IntWritable(36)));
+ ArrayWritable expected = list(
+ new IntWritable(34), new IntWritable(35), new IntWritable(36));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -287,9 +287,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new LongWritable(1234L)),
- record(new LongWritable(2345L))));
+ record(new LongWritable(2345L)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -339,9 +339,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
new LongWritable(1234L),
- new LongWritable(2345L)));
+ new LongWritable(2345L));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -397,9 +397,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new DoubleWritable(0.0), new DoubleWritable(0.0)),
- record(new DoubleWritable(0.0), new DoubleWritable(180.0))));
+ record(new DoubleWritable(0.0), new DoubleWritable(180.0)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -473,10 +473,10 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new DoubleWritable(0.0), new DoubleWritable(0.0)),
null,
- record(new DoubleWritable(0.0), new DoubleWritable(180.0))));
+ record(new DoubleWritable(0.0), new DoubleWritable(180.0)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -546,9 +546,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new DoubleWritable(0.0), new DoubleWritable(180.0)),
- record(new DoubleWritable(0.0), new DoubleWritable(0.0))));
+ record(new DoubleWritable(0.0), new DoubleWritable(0.0)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -619,9 +619,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new DoubleWritable(0.0), new DoubleWritable(180.0)),
- record(new DoubleWritable(0.0), new DoubleWritable(0.0))));
+ record(new DoubleWritable(0.0), new DoubleWritable(0.0)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
index 8f03c5b..73425e3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
@@ -311,13 +311,11 @@ public class TestDataWritableWriter {
+ "}\n";
ArrayWritable hiveRecord = createGroup(
- createGroup(
- createArray(
- createInt(1),
- createNull(),
- createInt(2)
- )
- )
+ createArray(
+ createInt(1),
+ createNull(),
+ createInt(2)
+ )
);
// Write record to Parquet format
@@ -363,18 +361,16 @@ public class TestDataWritableWriter {
ArrayWritable hiveRecord = createGroup(
createGroup(
createArray(
- createArray(
- createString("key1"),
- createInt(1)
- ),
- createArray(
- createString("key2"),
- createInt(2)
- ),
- createArray(
- createString("key3"),
- createNull()
- )
+ createString("key1"),
+ createInt(1)
+ ),
+ createArray(
+ createString("key2"),
+ createInt(2)
+ ),
+ createArray(
+ createString("key3"),
+ createNull()
)
)
);
@@ -432,14 +428,10 @@ public class TestDataWritableWriter {
+ "}\n";
ArrayWritable hiveRecord = createGroup(
- createGroup(
+ createArray(
createArray(
- createGroup(
- createArray(
- createInt(1),
- createInt(2)
- )
- )
+ createInt(1),
+ createInt(2)
)
)
);
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
index 6fc83e0..32264a4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
@@ -82,9 +82,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new Text("lettuce"), new IntWritable(34)),
- record(new Text("cabbage"), new IntWritable(18))));
+ record(new Text("cabbage"), new IntWritable(18)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -149,10 +149,10 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
+ ArrayWritable expected = list(
record(new Text("lettuce"), new IntWritable(34)),
record(new Text("kale"), null),
- record(new Text("cabbage"), new IntWritable(18))));
+ record(new Text("cabbage"), new IntWritable(18)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -230,9 +230,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
- record(new Text("green"), list(new Text("lettuce"), new Text("kale"), null)),
- record(new Text("brown"), null)));
+ ArrayWritable expected = list(
+ record(new Text("green"), record(new Text("lettuce"), new Text("kale"), null)),
+ record(new Text("brown"), null));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -325,9 +325,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
- record(new Text("low"), list(new IntWritable(34), new IntWritable(35), null)),
- record(new Text("high"), list(new IntWritable(340), new IntWritable(360)))));
+ ArrayWritable expected = list(
+ record(new Text("low"), record(new IntWritable(34), new IntWritable(35), null)),
+ record(new Text("high"), record(new IntWritable(340), new IntWritable(360))));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -386,9 +386,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(record(
+ ArrayWritable expected = list(record(
record(new IntWritable(7), new IntWritable(22)),
- new DoubleWritable(3.14))));
+ new DoubleWritable(3.14)));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -447,9 +447,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(record(
+ ArrayWritable expected = list(record(
new DoubleWritable(3.14),
- record(new IntWritable(7), new IntWritable(22)))));
+ record(new IntWritable(7), new IntWritable(22))));
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
@@ -542,13 +542,13 @@ public class TestMapStructures extends AbstractTestParquetDirect {
}
});
- ArrayWritable expected = record(list(
- record(new Text("a"), list(
+ ArrayWritable expected = list(
+ record(new Text("a"), record(
record(new Text("b"), new IntWritable(1)))),
- record(new Text("b"), list(
+ record(new Text("b"), record(
record(new Text("a"), new IntWritable(-1)),
record(new Text("b"), new IntWritable(-2))))
- ));
+ );
List<ArrayWritable> records = read(test);
Assert.assertEquals("Should have only one record", 1, records.size());
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
index 21f889a..dbb2795 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
@@ -55,7 +55,6 @@ public class TestParquetSerDe extends TestCase {
arr[4] = new DoubleWritable((double) 5.3);
arr[5] = new BytesWritable("hive and hadoop and parquet. Big family.".getBytes("UTF-8"));
arr[6] = new BytesWritable("parquetSerde binary".getBytes("UTF-8"));
- final Writable[] mapContainer = new Writable[1];
final Writable[] map = new Writable[3];
for (int i = 0; i < 3; ++i) {
final Writable[] pair = new Writable[2];
@@ -63,16 +62,13 @@ public class TestParquetSerDe extends TestCase {
pair[1] = new IntWritable(i);
map[i] = new ArrayWritable(Writable.class, pair);
}
- mapContainer[0] = new ArrayWritable(Writable.class, map);
- arr[7] = new ArrayWritable(Writable.class, mapContainer);
+ arr[7] = new ArrayWritable(Writable.class, map);
- final Writable[] arrayContainer = new Writable[1];
final Writable[] array = new Writable[5];
for (int i = 0; i < 5; ++i) {
array[i] = new BytesWritable(("elem_" + i).getBytes("UTF-8"));
}
- arrayContainer[0] = new ArrayWritable(Writable.class, array);
- arr[8] = new ArrayWritable(Writable.class, arrayContainer);
+ arr[8] = new ArrayWritable(Writable.class, array);
final ArrayWritable arrWritable = new ArrayWritable(Writable.class, arr);
// Test
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
index ef05150..f5d9cb4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
@@ -71,11 +71,9 @@ public class TestAbstractParquetMapInspector extends TestCase {
final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};
- final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
+ final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{
new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});
- final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});
-
final Map<Writable, Writable> expected = new HashMap<Writable, Writable>();
expected.put(new IntWritable(0), new IntWritable(1));
expected.put(new IntWritable(2), new IntWritable(3));
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
index 8646ff4..1ca6861 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
@@ -58,11 +58,9 @@ public class TestDeepParquetHiveMapInspector extends TestCase {
final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};
- final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
+ final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{
new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});
- final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});
-
assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0)));
assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2)));
assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new ShortWritable((short) 0)));
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
index f3a24af..0ce654d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
@@ -58,9 +58,8 @@ public class TestParquetHiveArrayInspector extends TestCase {
@Test
public void testRegularList() {
- final ArrayWritable internalList = new ArrayWritable(Writable.class,
+ final ArrayWritable list = new ArrayWritable(Writable.class,
new Writable[]{new IntWritable(3), new IntWritable(5), new IntWritable(1)});
- final ArrayWritable list = new ArrayWritable(ArrayWritable.class, new ArrayWritable[]{internalList});
final List<Writable> expected = new ArrayList<Writable>();
expected.add(new IntWritable(3));
http://git-wip-us.apache.org/repos/asf/hive/blob/175e2863/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
index 278419f..950956a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
@@ -57,11 +57,9 @@ public class TestStandardParquetHiveMapInspector extends TestCase {
final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};
- final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
+ final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{
new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});
- final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});
-
assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0)));
assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2)));
assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 0)));