You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by bu...@apache.org on 2016/09/01 13:48:17 UTC
[1/4] avro git commit: Revert "AVRO-1695. Fix deepCopy() to get
correct logical type conversion."
Repository: avro
Updated Branches:
refs/heads/master 9456db018 -> 9cfd33870
Revert "AVRO-1695. Fix deepCopy() to get correct logical type conversion."
This reverts commit 9456db01859dd3619fb09a6d4205ab433b357b02.
Incorrect JIRA key
Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/5d45505b
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/5d45505b
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/5d45505b
Branch: refs/heads/master
Commit: 5d45505b85fb55d4d3753b695bd3afc3f15b0c30
Parents: 9456db0
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Sep 1 08:44:21 2016 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 1 08:44:21 2016 -0500
----------------------------------------------------------------------
.../avro/src/main/java/org/apache/avro/generic/GenericData.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/avro/blob/5d45505b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
index 76eb625..9f3cc4d 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
@@ -1030,7 +1030,7 @@ public class GenericData {
LogicalType logicalType = schema.getLogicalType();
if (logicalType == null) // not a logical type -- use raw copy
return (T)deepCopyRaw(schema, value);
- Conversion conversion = getConversionByClass(value.getClass(), logicalType);
+ Conversion conversion = getConversionByClass(value.getClass());
if (conversion == null) // no conversion defined -- try raw copy
return (T)deepCopyRaw(schema, value);
// logical type with conversion: convert to raw, copy, then convert back to logical
[4/4] avro git commit: AVRO-1895. Fix deepCopy() to get correct
logical type conversion.
Posted by bu...@apache.org.
AVRO-1895. Fix deepCopy() to get correct logical type conversion.
(cherry picked from commit 9456db01859dd3619fb09a6d4205ab433b357b02)
Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/9cfd3387
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/9cfd3387
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/9cfd3387
Branch: refs/heads/master
Commit: 9cfd33870092d5c16a35b523305950215d1feabe
Parents: 973aa71
Author: Doug Cutting <cu...@apache.org>
Authored: Wed Aug 31 16:45:21 2016 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 1 08:45:37 2016 -0500
----------------------------------------------------------------------
.../avro/src/main/java/org/apache/avro/generic/GenericData.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/avro/blob/9cfd3387/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
index 9f3cc4d..76eb625 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
@@ -1030,7 +1030,7 @@ public class GenericData {
LogicalType logicalType = schema.getLogicalType();
if (logicalType == null) // not a logical type -- use raw copy
return (T)deepCopyRaw(schema, value);
- Conversion conversion = getConversionByClass(value.getClass());
+ Conversion conversion = getConversionByClass(value.getClass(), logicalType);
if (conversion == null) // no conversion defined -- try raw copy
return (T)deepCopyRaw(schema, value);
// logical type with conversion: convert to raw, copy, then convert back to logical
[3/4] avro git commit: AVRO-1895. Java: Fix GenericData#deepCopy() to
support logical types.
Posted by bu...@apache.org.
AVRO-1895. Java: Fix GenericData#deepCopy() to support logical types.
(cherry picked from commit be1639c44b16ddec8c54ecfca050364048daf896)
Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/973aa71a
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/973aa71a
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/973aa71a
Branch: refs/heads/master
Commit: 973aa71a97e99293a99da965c973c20ce6e0a84a
Parents: 07c8f32
Author: Doug Cutting <cu...@apache.org>
Authored: Wed Aug 31 16:20:47 2016 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 1 08:45:15 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../org/apache/avro/generic/GenericData.java | 35 ++++++++---
.../avro/generic/TestGenericLogicalTypes.java | 61 ++++++++++++++++++++
3 files changed, 90 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/avro/blob/973aa71a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 514b801..30d5782 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -15,6 +15,9 @@ Trunk (not yet released)
AVRO-1889: Update maven-shade-plugin to enable building in Java 8 on Mac.
(Sachin Goyal via blue)
+ AVRO-1695. Java: Fix GenericData#deepCopy() to support logical types.
+ (cutting)
+
BUG FIXES
AVRO-1741: Python3: Fix error when codec is not in the header.
http://git-wip-us.apache.org/repos/asf/avro/blob/973aa71a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
index 09f4c5a..9f3cc4d 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
@@ -34,6 +34,7 @@ import java.util.Map;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.AvroTypeException;
import org.apache.avro.Conversion;
+import org.apache.avro.Conversions;
import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
@@ -1018,15 +1019,31 @@ public class GenericData {
/**
* Makes a deep copy of a value given its schema.
+ * <P>Logical types are converted to raw types, copied, then converted back.
* @param schema the schema of the value to deep copy.
* @param value the value to deep copy.
* @return a deep copy of the given value.
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public <T> T deepCopy(Schema schema, T value) {
+ if (value == null) return null;
+ LogicalType logicalType = schema.getLogicalType();
+ if (logicalType == null) // not a logical type -- use raw copy
+ return (T)deepCopyRaw(schema, value);
+ Conversion conversion = getConversionByClass(value.getClass());
+ if (conversion == null) // no conversion defined -- try raw copy
+ return (T)deepCopyRaw(schema, value);
+ // logical type with conversion: convert to raw, copy, then convert back to logical
+ Object raw = Conversions.convertToRawType(value, schema, logicalType, conversion);
+ Object copy = deepCopyRaw(schema, raw); // copy raw
+ return (T)Conversions.convertToLogicalType(copy, schema, logicalType, conversion);
+ }
+
+ private Object deepCopyRaw(Schema schema, Object value) {
if (value == null) {
return null;
}
+
switch (schema.getType()) {
case ARRAY:
List<Object> arrayValue = (List) value;
@@ -1035,7 +1052,7 @@ public class GenericData {
for (Object obj : arrayValue) {
arrayCopy.add(deepCopy(schema.getElementType(), obj));
}
- return (T)arrayCopy;
+ return arrayCopy;
case BOOLEAN:
return value; // immutable
case BYTES:
@@ -1045,13 +1062,13 @@ public class GenericData {
byte[] bytesCopy = new byte[length];
byteBufferValue.get(bytesCopy, 0, length);
byteBufferValue.position(start);
- return (T)ByteBuffer.wrap(bytesCopy, 0, length);
+ return ByteBuffer.wrap(bytesCopy, 0, length);
case DOUBLE:
return value; // immutable
case ENUM:
- return (T)createEnum(value.toString(), schema);
+ return createEnum(value.toString(), schema);
case FIXED:
- return (T)createFixed(null, ((GenericFixed) value).bytes(), schema);
+ return createFixed(null, ((GenericFixed) value).bytes(), schema);
case FLOAT:
return value; // immutable
case INT:
@@ -1066,7 +1083,7 @@ public class GenericData {
mapCopy.put((CharSequence)(deepCopy(STRINGS, entry.getKey())),
deepCopy(schema.getValueType(), entry.getValue()));
}
- return (T)mapCopy;
+ return mapCopy;
case NULL:
return null;
case RECORD:
@@ -1080,11 +1097,11 @@ public class GenericData {
getField(value, name, pos, oldState));
setField(newRecord, name, pos, newValue, newState);
}
- return (T)newRecord;
+ return newRecord;
case STRING:
// Strings are immutable
if (value instanceof String) {
- return (T)value;
+ return value;
}
// Some CharSequence subclasses are mutable, so we still need to make
@@ -1092,9 +1109,9 @@ public class GenericData {
else if (value instanceof Utf8) {
// Utf8 copy constructor is more efficient than converting
// to string and then back to Utf8
- return (T)new Utf8((Utf8)value);
+ return new Utf8((Utf8)value);
}
- return (T)new Utf8(value.toString());
+ return new Utf8(value.toString());
case UNION:
return deepCopy(
schema.getTypes().get(resolveUnion(schema, value)), value);
http://git-wip-us.apache.org/repos/asf/avro/blob/973aa71a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
index 3a4b1e1..dc3c6f6 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
@@ -232,4 +232,65 @@ public class TestGenericLogicalTypes {
return file;
}
+
+ @Test
+ public void testCopyUuid() {
+ testCopy(LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)),
+ UUID.randomUUID(),
+ GENERIC);
+ }
+
+ @Test
+ public void testCopyUuidRaw() {
+ testCopy(LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)),
+ UUID.randomUUID().toString(), // use raw type
+ GenericData.get()); // with no conversions
+ }
+
+ @Test
+ public void testCopyDecimal() {
+ testCopy(LogicalTypes.decimal(9, 2).addToSchema(Schema.create(Schema.Type.BYTES)),
+ new BigDecimal("-34.34"),
+ GENERIC);
+ }
+
+ @Test
+ public void testCopyDecimalRaw() {
+ testCopy(LogicalTypes.decimal(9, 2).addToSchema(Schema.create(Schema.Type.BYTES)),
+ ByteBuffer.wrap(new BigDecimal("-34.34").unscaledValue().toByteArray()),
+ GenericData.get()); // no conversions
+ }
+
+ private void testCopy(Schema schema, Object value, GenericData model) {
+ // test direct copy of instance
+ checkCopy(value, model.deepCopy(schema, value), false);
+
+ // test nested in a record
+ Schema recordSchema = Schema.createRecord("X", "", "test", false);
+ List<Schema.Field> fields = new ArrayList<Schema.Field>();
+ fields.add(new Schema.Field("x", schema, "", null));
+ recordSchema.setFields(fields);
+
+ GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema);
+ builder.set("x", value);
+ GenericData.Record record = builder.build();
+ checkCopy(record, model.deepCopy(recordSchema, record), true);
+
+ // test nested in array
+ Schema arraySchema = Schema.createArray(schema);
+ ArrayList array = new ArrayList(Arrays.asList(value));
+ checkCopy(array, model.deepCopy(arraySchema, array), true);
+
+ // test record nested in array
+ Schema recordArraySchema = Schema.createArray(recordSchema);
+ ArrayList recordArray = new ArrayList(Arrays.asList(record));
+ checkCopy(recordArray, model.deepCopy(recordArraySchema, recordArray), true);
+ }
+
+ private void checkCopy(Object original, Object copy, boolean notSame) {
+ if (notSame)
+ Assert.assertNotSame(original, copy);
+ Assert.assertEquals(original, copy);
+ }
+
}
[2/4] avro git commit: Revert "AVRO-1695. Java: Fix
GenericData#deepCopy() to support logical types."
Posted by bu...@apache.org.
Revert "AVRO-1695. Java: Fix GenericData#deepCopy() to support logical types."
This reverts commit be1639c44b16ddec8c54ecfca050364048daf896.
Incorrect JIRA key
Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/07c8f328
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/07c8f328
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/07c8f328
Branch: refs/heads/master
Commit: 07c8f328603a4cacca087133ec3fb45337867aaf
Parents: 5d45505
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Sep 1 08:44:41 2016 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 1 08:44:41 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 -
.../org/apache/avro/generic/GenericData.java | 35 +++--------
.../avro/generic/TestGenericLogicalTypes.java | 61 --------------------
3 files changed, 9 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/avro/blob/07c8f328/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 30d5782..514b801 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -15,9 +15,6 @@ Trunk (not yet released)
AVRO-1889: Update maven-shade-plugin to enable building in Java 8 on Mac.
(Sachin Goyal via blue)
- AVRO-1695. Java: Fix GenericData#deepCopy() to support logical types.
- (cutting)
-
BUG FIXES
AVRO-1741: Python3: Fix error when codec is not in the header.
http://git-wip-us.apache.org/repos/asf/avro/blob/07c8f328/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
index 9f3cc4d..09f4c5a 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
@@ -34,7 +34,6 @@ import java.util.Map;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.AvroTypeException;
import org.apache.avro.Conversion;
-import org.apache.avro.Conversions;
import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
@@ -1019,31 +1018,15 @@ public class GenericData {
/**
* Makes a deep copy of a value given its schema.
- * <P>Logical types are converted to raw types, copied, then converted back.
* @param schema the schema of the value to deep copy.
* @param value the value to deep copy.
* @return a deep copy of the given value.
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public <T> T deepCopy(Schema schema, T value) {
- if (value == null) return null;
- LogicalType logicalType = schema.getLogicalType();
- if (logicalType == null) // not a logical type -- use raw copy
- return (T)deepCopyRaw(schema, value);
- Conversion conversion = getConversionByClass(value.getClass());
- if (conversion == null) // no conversion defined -- try raw copy
- return (T)deepCopyRaw(schema, value);
- // logical type with conversion: convert to raw, copy, then convert back to logical
- Object raw = Conversions.convertToRawType(value, schema, logicalType, conversion);
- Object copy = deepCopyRaw(schema, raw); // copy raw
- return (T)Conversions.convertToLogicalType(copy, schema, logicalType, conversion);
- }
-
- private Object deepCopyRaw(Schema schema, Object value) {
if (value == null) {
return null;
}
-
switch (schema.getType()) {
case ARRAY:
List<Object> arrayValue = (List) value;
@@ -1052,7 +1035,7 @@ public class GenericData {
for (Object obj : arrayValue) {
arrayCopy.add(deepCopy(schema.getElementType(), obj));
}
- return arrayCopy;
+ return (T)arrayCopy;
case BOOLEAN:
return value; // immutable
case BYTES:
@@ -1062,13 +1045,13 @@ public class GenericData {
byte[] bytesCopy = new byte[length];
byteBufferValue.get(bytesCopy, 0, length);
byteBufferValue.position(start);
- return ByteBuffer.wrap(bytesCopy, 0, length);
+ return (T)ByteBuffer.wrap(bytesCopy, 0, length);
case DOUBLE:
return value; // immutable
case ENUM:
- return createEnum(value.toString(), schema);
+ return (T)createEnum(value.toString(), schema);
case FIXED:
- return createFixed(null, ((GenericFixed) value).bytes(), schema);
+ return (T)createFixed(null, ((GenericFixed) value).bytes(), schema);
case FLOAT:
return value; // immutable
case INT:
@@ -1083,7 +1066,7 @@ public class GenericData {
mapCopy.put((CharSequence)(deepCopy(STRINGS, entry.getKey())),
deepCopy(schema.getValueType(), entry.getValue()));
}
- return mapCopy;
+ return (T)mapCopy;
case NULL:
return null;
case RECORD:
@@ -1097,11 +1080,11 @@ public class GenericData {
getField(value, name, pos, oldState));
setField(newRecord, name, pos, newValue, newState);
}
- return newRecord;
+ return (T)newRecord;
case STRING:
// Strings are immutable
if (value instanceof String) {
- return value;
+ return (T)value;
}
// Some CharSequence subclasses are mutable, so we still need to make
@@ -1109,9 +1092,9 @@ public class GenericData {
else if (value instanceof Utf8) {
// Utf8 copy constructor is more efficient than converting
// to string and then back to Utf8
- return new Utf8((Utf8)value);
+ return (T)new Utf8((Utf8)value);
}
- return new Utf8(value.toString());
+ return (T)new Utf8(value.toString());
case UNION:
return deepCopy(
schema.getTypes().get(resolveUnion(schema, value)), value);
http://git-wip-us.apache.org/repos/asf/avro/blob/07c8f328/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
index dc3c6f6..3a4b1e1 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericLogicalTypes.java
@@ -232,65 +232,4 @@ public class TestGenericLogicalTypes {
return file;
}
-
- @Test
- public void testCopyUuid() {
- testCopy(LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)),
- UUID.randomUUID(),
- GENERIC);
- }
-
- @Test
- public void testCopyUuidRaw() {
- testCopy(LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)),
- UUID.randomUUID().toString(), // use raw type
- GenericData.get()); // with no conversions
- }
-
- @Test
- public void testCopyDecimal() {
- testCopy(LogicalTypes.decimal(9, 2).addToSchema(Schema.create(Schema.Type.BYTES)),
- new BigDecimal("-34.34"),
- GENERIC);
- }
-
- @Test
- public void testCopyDecimalRaw() {
- testCopy(LogicalTypes.decimal(9, 2).addToSchema(Schema.create(Schema.Type.BYTES)),
- ByteBuffer.wrap(new BigDecimal("-34.34").unscaledValue().toByteArray()),
- GenericData.get()); // no conversions
- }
-
- private void testCopy(Schema schema, Object value, GenericData model) {
- // test direct copy of instance
- checkCopy(value, model.deepCopy(schema, value), false);
-
- // test nested in a record
- Schema recordSchema = Schema.createRecord("X", "", "test", false);
- List<Schema.Field> fields = new ArrayList<Schema.Field>();
- fields.add(new Schema.Field("x", schema, "", null));
- recordSchema.setFields(fields);
-
- GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema);
- builder.set("x", value);
- GenericData.Record record = builder.build();
- checkCopy(record, model.deepCopy(recordSchema, record), true);
-
- // test nested in array
- Schema arraySchema = Schema.createArray(schema);
- ArrayList array = new ArrayList(Arrays.asList(value));
- checkCopy(array, model.deepCopy(arraySchema, array), true);
-
- // test record nested in array
- Schema recordArraySchema = Schema.createArray(recordSchema);
- ArrayList recordArray = new ArrayList(Arrays.asList(record));
- checkCopy(recordArray, model.deepCopy(recordArraySchema, recordArray), true);
- }
-
- private void checkCopy(Object original, Object copy, boolean notSame) {
- if (notSame)
- Assert.assertNotSame(original, copy);
- Assert.assertEquals(original, copy);
- }
-
}