You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2015/04/15 18:42:48 UTC
svn commit: r1673861 - in
/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet:
convert/DataWritableRecordConverter.java read/DataWritableReadSupport.java
read/ParquetRecordReaderWrapper.java
Author: spena
Date: Wed Apr 15 16:42:47 2015
New Revision: 1673861
URL: http://svn.apache.org/r1673861
Log:
HIVE-10016: Remove duplicated Hive table schema parsing in DataWritableReadSupport (Dong Chen via Sergio Pena)
Modified:
hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java?rev=1673861&r1=1673860&r2=1673861&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java Wed Apr 15 16:42:47 2015
@@ -13,12 +13,11 @@
*/
package org.apache.hadoop.hive.ql.io.parquet.convert;
-import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
import org.apache.hadoop.io.ArrayWritable;
import parquet.io.api.GroupConverter;
import parquet.io.api.RecordMaterializer;
import parquet.schema.GroupType;
-import parquet.schema.MessageTypeParser;
+import parquet.schema.MessageType;
import java.util.Map;
@@ -31,9 +30,11 @@ public class DataWritableRecordConverter
private final HiveStructConverter root;
- public DataWritableRecordConverter(final GroupType requestedSchema, final Map<String, String> metadata) {
+ public DataWritableRecordConverter(final GroupType requestedSchema,
+ final MessageType tableSchema,
+ final Map<String, String> metadata) {
this.root = new HiveStructConverter(requestedSchema,
- MessageTypeParser.parseMessageType(metadata.get(DataWritableReadSupport.HIVE_TABLE_AS_PARQUET_SCHEMA)), metadata);
+ tableSchema, metadata);
}
@Override
Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java?rev=1673861&r1=1673860&r2=1673861&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java Wed Apr 15 16:42:47 2015
@@ -52,6 +52,7 @@ public class DataWritableReadSupport ext
public static final String HIVE_TABLE_AS_PARQUET_SCHEMA = "HIVE_TABLE_SCHEMA";
public static final String PARQUET_COLUMN_INDEX_ACCESS = "parquet.column.index.access";
+ private MessageType tableSchema;
/**
* From a string which columns names (including hive column), return a list
@@ -205,7 +206,6 @@ public class DataWritableReadSupport ext
if (columnNames != null) {
List<String> columnNamesList = getColumnNames(columnNames);
- MessageType tableSchema;
if (indexAccess) {
List<Integer> indexSequence = new ArrayList<Integer>();
@@ -229,6 +229,7 @@ public class DataWritableReadSupport ext
return new ReadContext(requestedSchemaByUser, contextMetadata);
} else {
+ tableSchema = fileSchema;
contextMetadata.put(HIVE_TABLE_AS_PARQUET_SCHEMA, fileSchema.toString());
return new ReadContext(fileSchema, contextMetadata);
}
@@ -258,6 +259,11 @@ public class DataWritableReadSupport ext
metadata.put(key, String.valueOf(HiveConf.getBoolVar(
configuration, HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION)));
}
- return new DataWritableRecordConverter(readContext.getRequestedSchema(), metadata);
+ return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema,
+ metadata);
+ }
+
+ public MessageType getTableSchema() {
+ return tableSchema;
}
}
Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java?rev=1673861&r1=1673860&r2=1673861&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java Wed Apr 15 16:42:47 2015
@@ -252,10 +252,10 @@ public class ParquetRecordReaderWrapper
final List<BlockMetaData> blocks = parquetMetadata.getBlocks();
final FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
- final ReadContext readContext = new DataWritableReadSupport().init(new InitContext(jobConf,
+ DataWritableReadSupport readSupport = new DataWritableReadSupport();
+ final ReadContext readContext = readSupport.init(new InitContext(jobConf,
null, fileMetaData.getSchema()));
- schemaSize = MessageTypeParser.parseMessageType(readContext.getReadSupportMetadata()
- .get(DataWritableReadSupport.HIVE_TABLE_AS_PARQUET_SCHEMA)).getFieldCount();
+ schemaSize = readSupport.getTableSchema().getFieldCount();
final List<BlockMetaData> splitGroup = new ArrayList<BlockMetaData>();
final long splitStart = ((FileSplit) oldSplit).getStart();
final long splitLength = ((FileSplit) oldSplit).getLength();