You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2015/04/15 18:42:48 UTC

svn commit: r1673861 - in /hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet: convert/DataWritableRecordConverter.java read/DataWritableReadSupport.java read/ParquetRecordReaderWrapper.java

Author: spena
Date: Wed Apr 15 16:42:47 2015
New Revision: 1673861

URL: http://svn.apache.org/r1673861
Log:
HIVE-10016: Remove duplicated Hive table schema parsing in DataWritableReadSupport (Dong Chen via Sergio Pena)

Modified:
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java?rev=1673861&r1=1673860&r2=1673861&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java Wed Apr 15 16:42:47 2015
@@ -13,12 +13,11 @@
  */
 package org.apache.hadoop.hive.ql.io.parquet.convert;
 
-import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
 import org.apache.hadoop.io.ArrayWritable;
 import parquet.io.api.GroupConverter;
 import parquet.io.api.RecordMaterializer;
 import parquet.schema.GroupType;
-import parquet.schema.MessageTypeParser;
+import parquet.schema.MessageType;
 
 import java.util.Map;
 
@@ -31,9 +30,11 @@ public class DataWritableRecordConverter
 
   private final HiveStructConverter root;
 
-  public DataWritableRecordConverter(final GroupType requestedSchema, final Map<String, String> metadata) {
+  public DataWritableRecordConverter(final GroupType requestedSchema,
+                                     final MessageType tableSchema,
+                                     final Map<String, String> metadata) {
     this.root = new HiveStructConverter(requestedSchema,
-      MessageTypeParser.parseMessageType(metadata.get(DataWritableReadSupport.HIVE_TABLE_AS_PARQUET_SCHEMA)), metadata);
+      tableSchema, metadata);
   }
 
   @Override

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java?rev=1673861&r1=1673860&r2=1673861&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java Wed Apr 15 16:42:47 2015
@@ -52,6 +52,7 @@ public class DataWritableReadSupport ext
 
   public static final String HIVE_TABLE_AS_PARQUET_SCHEMA = "HIVE_TABLE_SCHEMA";
   public static final String PARQUET_COLUMN_INDEX_ACCESS = "parquet.column.index.access";
+  private MessageType tableSchema;
 
   /**
    * From a string which columns names (including hive column), return a list
@@ -205,7 +206,6 @@ public class DataWritableReadSupport ext
     if (columnNames != null) {
       List<String> columnNamesList = getColumnNames(columnNames);
 
-      MessageType tableSchema;
       if (indexAccess) {
         List<Integer> indexSequence = new ArrayList<Integer>();
 
@@ -229,6 +229,7 @@ public class DataWritableReadSupport ext
 
       return new ReadContext(requestedSchemaByUser, contextMetadata);
     } else {
+      tableSchema = fileSchema;
       contextMetadata.put(HIVE_TABLE_AS_PARQUET_SCHEMA, fileSchema.toString());
       return new ReadContext(fileSchema, contextMetadata);
     }
@@ -258,6 +259,11 @@ public class DataWritableReadSupport ext
       metadata.put(key, String.valueOf(HiveConf.getBoolVar(
         configuration, HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION)));
     }
-    return new DataWritableRecordConverter(readContext.getRequestedSchema(), metadata);
+    return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema,
+        metadata);
+  }
+
+  public MessageType getTableSchema() {
+    return tableSchema;
   }
 }

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java?rev=1673861&r1=1673860&r2=1673861&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java Wed Apr 15 16:42:47 2015
@@ -252,10 +252,10 @@ public class ParquetRecordReaderWrapper
       final List<BlockMetaData> blocks = parquetMetadata.getBlocks();
       final FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
 
-      final ReadContext readContext = new DataWritableReadSupport().init(new InitContext(jobConf,
+      DataWritableReadSupport readSupport = new DataWritableReadSupport();
+      final ReadContext readContext = readSupport.init(new InitContext(jobConf,
           null, fileMetaData.getSchema()));
-      schemaSize = MessageTypeParser.parseMessageType(readContext.getReadSupportMetadata()
-          .get(DataWritableReadSupport.HIVE_TABLE_AS_PARQUET_SCHEMA)).getFieldCount();
+      schemaSize = readSupport.getTableSchema().getFieldCount();
       final List<BlockMetaData> splitGroup = new ArrayList<BlockMetaData>();
       final long splitStart = ((FileSplit) oldSplit).getStart();
       final long splitLength = ((FileSplit) oldSplit).getLength();