You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2012/11/15 00:21:31 UTC
svn commit: r1409542 - in /avro/trunk: ./
lang/java/avro/src/main/java/org/apache/avro/generic/
lang/java/avro/src/main/java/org/apache/avro/util/
lang/java/avro/src/test/java/org/apache/avro/
lang/java/avro/src/test/java/org/apache/avro/generic/ lang/...
Author: cutting
Date: Wed Nov 14 23:21:29 2012
New Revision: 1409542
URL: http://svn.apache.org/viewvc?rev=1409542&view=rev
Log:
AVRO-1199. Java: Fix SortedKeyValueFile to copy the key used to compare against next.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java
avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Nov 14 23:21:29 2012
@@ -40,6 +40,9 @@ Trunk (not yet released)
AVRO-1200. DatumWriter can write malformed data if collection is
modified during write. (tomwhite)
+ AVRO-1199. Java: Fix SortedKeyValueFile to copy the key used to
+ compare against next. Also improve GenericData#deepCopy() to be
+ generic, so that its return type matches its parameter type. (cutting)
Avro 1.7.2 (20 October 2012)
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java Wed Nov 14 23:21:29 2012
@@ -767,7 +767,7 @@ public class GenericData {
* @return a deep copy of the given value.
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
- public Object deepCopy(Schema schema, Object value) {
+ public <T> T deepCopy(Schema schema, T value) {
if (value == null) {
return null;
}
@@ -779,9 +779,9 @@ public class GenericData {
for (Object obj : arrayValue) {
arrayCopy.add(deepCopy(schema.getElementType(), obj));
}
- return arrayCopy;
+ return (T)arrayCopy;
case BOOLEAN:
- return new Boolean((Boolean)value);
+ return (T)new Boolean((Boolean)value);
case BYTES:
ByteBuffer byteBufferValue = (ByteBuffer) value;
int start = byteBufferValue.position();
@@ -789,20 +789,20 @@ public class GenericData {
byte[] bytesCopy = new byte[length];
byteBufferValue.get(bytesCopy, 0, length);
byteBufferValue.position(start);
- return ByteBuffer.wrap(bytesCopy, 0, length);
+ return (T)ByteBuffer.wrap(bytesCopy, 0, length);
case DOUBLE:
- return new Double((Double)value);
+ return (T)new Double((Double)value);
case ENUM:
// Enums are immutable; shallow copy will suffice
return value;
case FIXED:
- return createFixed(null, ((GenericFixed) value).bytes(), schema);
+ return (T)createFixed(null, ((GenericFixed) value).bytes(), schema);
case FLOAT:
- return new Float((Float)value);
+ return (T)new Float((Float)value);
case INT:
- return new Integer((Integer)value);
+ return (T)new Integer((Integer)value);
case LONG:
- return new Long((Long)value);
+ return (T)new Long((Long)value);
case MAP:
Map<CharSequence, Object> mapValue = (Map) value;
Map<CharSequence, Object> mapCopy =
@@ -811,7 +811,7 @@ public class GenericData {
mapCopy.put((CharSequence)(deepCopy(STRINGS, entry.getKey())),
deepCopy(schema.getValueType(), entry.getValue()));
}
- return mapCopy;
+ return (T)mapCopy;
case NULL:
return null;
case RECORD:
@@ -821,11 +821,11 @@ public class GenericData {
recordCopy.put(field.pos(),
deepCopy(field.schema(), recordValue.get(field.pos())));
}
- return recordCopy;
+ return (T)recordCopy;
case STRING:
// Strings are immutable
if (value instanceof String) {
- return value;
+ return (T)value;
}
// Some CharSequence subclasses are mutable, so we still need to make
@@ -833,9 +833,9 @@ public class GenericData {
else if (value instanceof Utf8) {
// Utf8 copy constructor is more efficient than converting
// to string and then back to Utf8
- return new Utf8((Utf8)value);
+ return (T)new Utf8((Utf8)value);
}
- return new Utf8(value.toString());
+ return (T)new Utf8(value.toString());
case UNION:
return deepCopy(
schema.getTypes().get(resolveUnion(schema, value)), value);
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java Wed Nov 14 23:21:29 2012
@@ -83,6 +83,14 @@ public class Utf8 implements Comparable<
return this;
}
+ /** Set to the contents of a String. */
+ public Utf8 set(String string) {
+ this.bytes = getBytesFor(string);
+ this.length = bytes.length;
+ this.string = string;
+ return this;
+ }
+
@Override
public String toString() {
if (this.length == 0) return "";
Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java Wed Nov 14 23:21:29 2012
@@ -96,11 +96,11 @@ public class FooBarSpecificRecord extend
private Builder(org.apache.avro.FooBarSpecificRecord other) {
super(org.apache.avro.FooBarSpecificRecord.SCHEMA$);
if (isValidValue(fields()[0], other.id)) {
- this.id = (java.lang.Integer) data().deepCopy(fields()[0].schema(), other.id);
+ this.id = data().deepCopy(fields()[0].schema(), other.id);
fieldSetFlags()[0] = true;
}
if (isValidValue(fields()[1], other.relatedids)) {
- this.relatedids = (java.util.List<java.lang.Integer>) data().deepCopy(fields()[1].schema(), other.relatedids);
+ this.relatedids = data().deepCopy(fields()[1].schema(), other.relatedids);
fieldSetFlags()[1] = true;
}
}
Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java Wed Nov 14 23:21:29 2012
@@ -374,8 +374,7 @@ public class TestGenericData {
GenericRecord record = new GenericData.Record(schema);
record.put(byte_field.name(), buffer);
- GenericRecord copy =
- (GenericRecord) GenericData.get().deepCopy(schema, record);
+ GenericRecord copy = GenericData.get().deepCopy(schema, record);
ByteBuffer buffer_copy = (ByteBuffer) copy.get(byte_field.name());
assertEquals(buffer, buffer_copy);
Modified: avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm (original)
+++ avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm Wed Nov 14 23:21:29 2012
@@ -159,7 +159,7 @@ public class ${this.mangle($schema.getNa
super(#if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.SCHEMA$)#end;
#foreach ($field in $schema.getFields())
if (isValidValue(fields()[$field.pos()], other.${this.mangle($field.name(), $schema.isError())})) {
- this.${this.mangle($field.name(), $schema.isError())} = (${this.javaType($field.schema())}) data().deepCopy(fields()[$field.pos()].schema(), other.${this.mangle($field.name(), $schema.isError())});
+ this.${this.mangle($field.name(), $schema.isError())} = data().deepCopy(fields()[$field.pos()].schema(), other.${this.mangle($field.name(), $schema.isError())});
fieldSetFlags()[$field.pos()] = true;
}
#end
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java Wed Nov 14 23:21:29 2012
@@ -536,7 +536,7 @@ public class SortedKeyValueFile {
throw new IllegalArgumentException("Records must be inserted in sorted key order."
+ " Attempted to insert key " + key + " after " + mPreviousKey + ".");
}
- mPreviousKey = key;
+ mPreviousKey = GenericData.get().deepCopy(mKeySchema, key);
// Construct the data record.
AvroKeyValue<K, V> dataRecord
Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java (original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java Wed Nov 14 23:21:29 2012
@@ -32,6 +32,7 @@ import org.apache.avro.hadoop.io.AvroKey
import org.apache.avro.io.DatumReader;
import org.apache.avro.file.FileReader;
import org.apache.avro.file.DataFileReader;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.Rule;
@@ -61,9 +62,11 @@ public class TestSortedKeyValueFile {
SortedKeyValueFile.Writer<CharSequence, CharSequence> writer
= new SortedKeyValueFile.Writer<CharSequence, CharSequence>(options);
+ Utf8 key = new Utf8(); // re-use key, to test copied
+
try {
- writer.append("banana", "Banana");
- writer.append("apple", "Apple"); // Ruh, roh!
+ writer.append(key.set("banana"), "Banana");
+ writer.append(key.set("apple"), "Apple"); // Ruh, roh!
} finally {
writer.close();
}
Modified: avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/output/Player.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/output/Player.java (original)
+++ avro/trunk/lang/java/tools/src/test/compiler/output/Player.java Wed Nov 14 23:21:29 2012
@@ -150,19 +150,19 @@ public class Player extends org.apache.a
private Builder(avro.examples.baseball.Player other) {
super(avro.examples.baseball.Player.SCHEMA$);
if (isValidValue(fields()[0], other.number)) {
- this.number = (java.lang.Integer) data().deepCopy(fields()[0].schema(), other.number);
+ this.number = data().deepCopy(fields()[0].schema(), other.number);
fieldSetFlags()[0] = true;
}
if (isValidValue(fields()[1], other.first_name)) {
- this.first_name = (java.lang.CharSequence) data().deepCopy(fields()[1].schema(), other.first_name);
+ this.first_name = data().deepCopy(fields()[1].schema(), other.first_name);
fieldSetFlags()[1] = true;
}
if (isValidValue(fields()[2], other.last_name)) {
- this.last_name = (java.lang.CharSequence) data().deepCopy(fields()[2].schema(), other.last_name);
+ this.last_name = data().deepCopy(fields()[2].schema(), other.last_name);
fieldSetFlags()[2] = true;
}
if (isValidValue(fields()[3], other.position)) {
- this.position = (java.util.List<avro.examples.baseball.Position>) data().deepCopy(fields()[3].schema(), other.position);
+ this.position = data().deepCopy(fields()[3].schema(), other.position);
fieldSetFlags()[3] = true;
}
}