You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/21 22:36:17 UTC

svn commit: r1525290 - in /hive/trunk: ql/src/java/org/apache/hadoop/hive/ql/io/avro/ serde/src/java/org/apache/hadoop/hive/serde2/avro/ serde/src/test/org/apache/hadoop/hive/serde2/avro/

Author: hashutosh
Date: Sat Sep 21 20:36:16 2013
New Revision: 1525290

URL: http://svn.apache.org/r1525290
Log:
HIVE-4732 : Reduce or eliminate the expensive Schema equals() check for AvroSerde (Mohammad Kamrul Islam via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java Sat Sep 21 20:36:16 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.avr
 
 
 import java.io.IOException;
+import java.rmi.server.UID;
 import java.util.Map;
 import java.util.Properties;
 
@@ -57,6 +58,10 @@ public class AvroGenericRecordReader imp
   final private long start;
   final private long stop;
   protected JobConf jobConf;
+  /**
+   * A unique ID for each record reader.
+   */
+  final private UID recordReaderID;
 
   public AvroGenericRecordReader(JobConf job, FileSplit split, Reporter reporter) throws IOException {
     this.jobConf = job;
@@ -78,6 +83,7 @@ public class AvroGenericRecordReader imp
     this.reader.sync(split.getStart());
     this.start = reader.tell();
     this.stop = split.getStart() + split.getLength();
+    this.recordReaderID = new UID();
   }
 
   /**
@@ -148,6 +154,7 @@ public class AvroGenericRecordReader imp
 
     GenericData.Record r = (GenericData.Record)reader.next();
     record.setRecord(r);
+    record.setRecordReaderID(recordReaderID);
 
     return true;
   }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java Sat Sep 21 20:36:16 2013
@@ -21,8 +21,10 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.rmi.server.UID;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -52,6 +54,20 @@ import org.apache.hadoop.io.Writable;
 class AvroDeserializer {
   private static final Log LOG = LogFactory.getLog(AvroDeserializer.class);
   /**
+   * Set of already seen and valid record readers IDs which doesn't need re-encoding
+   */
+  private final HashSet<UID> noEncodingNeeded = new HashSet<UID>();
+  /**
+   * Map of record reader ID and the associated re-encoder. It contains only the record readers
+   *  that record needs to be re-encoded.
+   */
+  private final HashMap<UID, SchemaReEncoder> reEncoderCache = new HashMap<UID, SchemaReEncoder>();
+  /**
+   * Flag to print the re-encoding warning message only once. Avoid excessive logging for each
+   * record encoding.
+   */
+  private static boolean warnedOnce = false;
+  /**
    * When encountering a record with an older schema than the one we're trying
    * to read, it is necessary to re-encode with a reader against the newer schema.
    * Because Hive doesn't provide a way to pass extra information to the
@@ -64,16 +80,15 @@ class AvroDeserializer {
     private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
     private final GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>();
     private BinaryDecoder binaryDecoder = null;
-    private final InstanceCache<ReaderWriterSchemaPair, GenericDatumReader<GenericRecord>> gdrCache
-        = new InstanceCache<ReaderWriterSchemaPair, GenericDatumReader<GenericRecord>>() {
-            @Override
-            protected GenericDatumReader<GenericRecord> makeInstance(ReaderWriterSchemaPair hv) {
-              return new GenericDatumReader<GenericRecord>(hv.getWriter(), hv.getReader());
-            }
-          };
 
-    public GenericRecord reencode(GenericRecord r, Schema readerSchema)
-            throws AvroSerdeException {
+    GenericDatumReader<GenericRecord> gdr = null;
+
+    public SchemaReEncoder(Schema writer, Schema reader) {
+      gdr = new GenericDatumReader<GenericRecord>(writer, reader);
+    }
+
+    public GenericRecord reencode(GenericRecord r)
+        throws AvroSerdeException {
       baos.reset();
 
       BinaryEncoder be = EncoderFactory.get().directBinaryEncoder(baos, null);
@@ -84,8 +99,6 @@ class AvroDeserializer {
 
         binaryDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bais, binaryDecoder);
 
-        ReaderWriterSchemaPair pair = new ReaderWriterSchemaPair(r.getSchema(), readerSchema);
-        GenericDatumReader<GenericRecord> gdr = gdrCache.retrieve(pair);
         return gdr.read(r, binaryDecoder);
 
       } catch (IOException e) {
@@ -95,7 +108,6 @@ class AvroDeserializer {
   }
 
   private List<Object> row;
-  private SchemaReEncoder reEncoder;
 
   /**
    * Deserialize an Avro record, recursing into its component fields and
@@ -127,14 +139,31 @@ class AvroDeserializer {
     AvroGenericRecordWritable recordWritable = (AvroGenericRecordWritable) writable;
     GenericRecord r = recordWritable.getRecord();
 
-    // Check if we're working with an evolved schema
-    if(!r.getSchema().equals(readerSchema)) {
-      LOG.warn("Received different schemas.  Have to re-encode: " +
-              r.getSchema().toString(false));
-      if(reEncoder == null) {
-        reEncoder = new SchemaReEncoder();
+   UID recordReaderId = recordWritable.getRecordReaderID();
+   //If the record reader (from which the record is originated) is already seen and valid,
+    //no need to re-encode the record.
+    if(!noEncodingNeeded.contains(recordReaderId)) {
+      SchemaReEncoder reEncoder = null;
+      //Check if the record record is already encoded once. If it does
+      //reuse the encoder.
+      if(reEncoderCache.containsKey(recordReaderId)) {
+        reEncoder = reEncoderCache.get(recordReaderId); //Reuse the re-encoder
+      } else if (!r.getSchema().equals(readerSchema)) { //Evolved schema?
+        //Create and store new encoder in the map for re-use
+        reEncoder = new SchemaReEncoder(r.getSchema(), readerSchema);
+        reEncoderCache.put(recordReaderId, reEncoder);
+      } else{
+        LOG.info("Adding new valid RRID :" +  recordReaderId);
+        noEncodingNeeded.add(recordReaderId);
+      }
+      if(reEncoder != null) {
+        if (!warnedOnce) {
+          LOG.warn("Received different schemas.  Have to re-encode: " +
+              r.getSchema().toString(false) + "\nSIZE" + reEncoderCache + " ID " + recordReaderId);
+          warnedOnce = true;
+        }
+        r = reEncoder.reencode(r);
       }
-      r = reEncoder.reencode(r, readerSchema);
     }
 
     workerBase(row, columnNames, columnTypes, r);
@@ -288,4 +317,13 @@ class AvroDeserializer {
 
     return map;
   }
+
+  public HashSet<UID> getNoEncodingNeeded() {
+    return noEncodingNeeded;
+  }
+
+  public HashMap<UID, SchemaReEncoder> getReEncoderCache() {
+    return reEncoderCache;
+  }
+
 }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java Sat Sep 21 20:36:16 2013
@@ -17,6 +17,13 @@
  */
 package org.apache.hadoop.hive.serde2.avro;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.rmi.server.UID;
+
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumReader;
@@ -28,12 +35,6 @@ import org.apache.avro.io.DecoderFactory
 import org.apache.avro.io.EncoderFactory;
 import org.apache.hadoop.io.Writable;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
 /**
  * Wrapper around an Avro GenericRecord.  Necessary because Hive's deserializer
  * will happily deserialize any object - as long as it's a writable.
@@ -41,6 +42,10 @@ import java.io.InputStream;
 public class AvroGenericRecordWritable implements Writable{
   GenericRecord record;
   private BinaryDecoder binaryDecoder;
+  /**
+   * Unique Id determine which record reader created this record
+   */
+  private UID recordReaderID;
 
   // There are two areas of exploration for optimization here.
   // 1.  We're serializing the schema with every object.  If we assume the schema
@@ -68,6 +73,7 @@ public class AvroGenericRecordWritable i
     // Write schema since we need it to pull the data out. (see point #1 above)
     String schemaString = record.getSchema().toString(false);
     out.writeUTF(schemaString);
+    recordReaderID.write(out);
 
     // Write record to byte buffer
     GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>();
@@ -80,9 +86,18 @@ public class AvroGenericRecordWritable i
   @Override
   public void readFields(DataInput in) throws IOException {
     Schema schema = Schema.parse(in.readUTF());
+    recordReaderID = UID.read(in);
     record = new GenericData.Record(schema);
     binaryDecoder = DecoderFactory.defaultFactory().createBinaryDecoder((InputStream) in, binaryDecoder);
     GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
     record = gdr.read(record, binaryDecoder);
   }
+
+  public UID getRecordReaderID() {
+    return recordReaderID;
+  }
+
+  public void setRecordReaderID(UID recordReaderID) {
+    this.recordReaderID = recordReaderID;
+  }
 }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java Sat Sep 21 20:36:16 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTru
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.rmi.server.UID;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Hashtable;
@@ -500,4 +501,64 @@ public class TestAvroDeserializer {
       assertEquals(expected, soi.getPrimitiveJavaObject(rowElement));
     }
   }
+
+  @Test
+  public void verifyCaching() throws SerDeException, IOException {
+    Schema s = Schema.parse(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+    GenericData.Record record = new GenericData.Record(s);
+    GenericData.Record innerRecord = new GenericData.Record(s.getField("aRecord").schema());
+    innerRecord.put("int1", 42);
+    innerRecord.put("boolean1", true);
+    innerRecord.put("long1", 42432234234l);
+    record.put("aRecord", innerRecord);
+    assertTrue(GENERIC_DATA.validate(s, record));
+
+    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
+    UID recordReaderID = new UID();
+    garw.setRecordReaderID(recordReaderID);
+    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
+
+    AvroDeserializer de = new AvroDeserializer();
+    ArrayList<Object> row =
+        (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
+
+    assertEquals(1, de.getNoEncodingNeeded().size());
+    assertEquals(0, de.getReEncoderCache().size());
+
+    // Read the record with the same record reader ID
+    row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
+
+    //Expecting not to change the size of internal structures
+    assertEquals(1, de.getNoEncodingNeeded().size());
+    assertEquals(0, de.getReEncoderCache().size());
+
+    //Read the record with **different** record reader ID
+    garw.setRecordReaderID(new UID()); //New record reader ID
+    row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
+
+    //Expecting to change the size of internal structures
+    assertEquals(2, de.getNoEncodingNeeded().size());
+    assertEquals(0, de.getReEncoderCache().size());
+
+  //Read the record with **different** record reader ID and **evolved** schema
+    Schema evolvedSchema = Schema.parse(s.toString());
+    evolvedSchema.getField("aRecord").schema().addProp("Testing", "meaningless");
+    garw.setRecordReaderID(recordReaderID = new UID()); //New record reader ID
+    row =
+            (ArrayList<Object>)de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, evolvedSchema);
+
+    //Expecting to change the size of internal structures
+    assertEquals(2, de.getNoEncodingNeeded().size());
+    assertEquals(1, de.getReEncoderCache().size());
+
+  //Read the record with existing record reader ID and same **evolved** schema
+    garw.setRecordReaderID(recordReaderID); //Reuse record reader ID
+    row =
+            (ArrayList<Object>)de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, evolvedSchema);
+
+    //Expecting NOT to change the size of internal structures
+    assertEquals(2, de.getNoEncodingNeeded().size());
+    assertEquals(1, de.getReEncoderCache().size());
+
+  }
 }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestGenericAvroRecordWritable.java Sat Sep 21 20:36:16 2013
@@ -17,18 +17,19 @@
  */
 package org.apache.hadoop.hive.serde2.avro;
 
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericRecord;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.rmi.server.UID;
 
-import static org.junit.Assert.assertEquals;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.junit.Test;
 
 public class TestGenericAvroRecordWritable {
   private static final String schemaJSON = "{\n" +
@@ -59,12 +60,14 @@ public class TestGenericAvroRecordWritab
     assertEquals("Doctor", gr.get("last"));
 
     AvroGenericRecordWritable garw = new AvroGenericRecordWritable(gr);
+    garw.setRecordReaderID(new UID());
 
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutputStream daos = new DataOutputStream(baos);
     garw.write(daos);
 
     AvroGenericRecordWritable garw2 = new AvroGenericRecordWritable(gr);
+    garw2.setRecordReaderID(new UID());
 
     ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
     DataInputStream dais = new DataInputStream(bais);

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaReEncoder.java Sat Sep 21 20:36:16 2013
@@ -17,15 +17,15 @@
  */
 package org.apache.hadoop.hive.serde2.avro;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 public class TestSchemaReEncoder {
   @Test
   public void schemasCanAddFields() throws SerDeException {
@@ -62,8 +62,8 @@ public class TestSchemaReEncoder {
     GenericRecord record = new GenericData.Record(originalSchema);
     record.put("text", "it is a far better thing I do, yadda, yadda");
     assertTrue(GenericData.get().validate(originalSchema, record));
-    AvroDeserializer.SchemaReEncoder schemaReEncoder = new AvroDeserializer.SchemaReEncoder();
-    GenericRecord r2 = schemaReEncoder.reencode(record, evolvedSchema);
+    AvroDeserializer.SchemaReEncoder schemaReEncoder = new AvroDeserializer.SchemaReEncoder(record.getSchema(), evolvedSchema);
+    GenericRecord r2 = schemaReEncoder.reencode(record);
 
     assertTrue(GenericData.get().validate(evolvedSchema, r2));
     assertEquals("Hi!", r2.get("new_kid").toString());
@@ -104,7 +104,8 @@ public class TestSchemaReEncoder {
     record.put("a", 19);
     assertTrue(GenericData.get().validate(originalSchema2, record));
 
-    r2 = schemaReEncoder.reencode(record,  evolvedSchema2);
+    schemaReEncoder = new AvroDeserializer.SchemaReEncoder(record.getSchema(), evolvedSchema2);
+    r2 = schemaReEncoder.reencode(record);
     assertTrue(GenericData.get().validate(evolvedSchema2, r2));
     assertEquals(42l, r2.get("b"));
   }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java?rev=1525290&r1=1525289&r2=1525290&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java Sat Sep 21 20:36:16 2013
@@ -17,13 +17,14 @@
  */
 package org.apache.hadoop.hive.serde2.avro;
 
-import org.apache.avro.generic.GenericData;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.rmi.server.UID;
+
+import org.apache.avro.generic.GenericData;
 
 class Utils {
   // Force Avro to serialize and de-serialize the record to make sure it has a
@@ -31,6 +32,7 @@ class Utils {
   public static AvroGenericRecordWritable
   serializeAndDeserializeRecord(GenericData.Record record) throws IOException {
     AvroGenericRecordWritable garw = new AvroGenericRecordWritable(record);
+    garw.setRecordReaderID(new UID());
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutputStream daos = new DataOutputStream(baos);
     garw.write(daos);
@@ -39,6 +41,7 @@ class Utils {
     DataInputStream dais = new DataInputStream(bais);
 
     AvroGenericRecordWritable garw2 = new AvroGenericRecordWritable();
+    garw2.setRecordReaderID(new UID());
     garw2.readFields(dais);
     return garw2;
   }