You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by gr...@apache.org on 2012/08/12 21:45:27 UTC
git commit: [CRUNCH-43] - Remove tuple re-use for avro serialization
Updated Branches:
refs/heads/master 03dd4b52b -> 0666b0f89
[CRUNCH-43] - Remove tuple re-use for avro serialization
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/0666b0f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/0666b0f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/0666b0f8
Branch: refs/heads/master
Commit: 0666b0f89e9eb5f78bbcf561e823b1d343aa434f
Parents: 03dd4b5
Author: Gabriel Reid <ga...@gmail.com>
Authored: Sat Aug 11 22:42:57 2012 +0200
Committer: Gabriel Reid <ga...@gmail.com>
Committed: Sun Aug 12 21:41:30 2012 +0200
----------------------------------------------------------------------
.../java/org/apache/crunch/types/avro/Avros.java | 21 ++++++++------
.../org/apache/crunch/types/avro/AvrosTest.java | 19 +++++++++++++
2 files changed, 31 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/0666b0f8/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java b/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java
index 1fd7baa..00a297c 100644
--- a/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java
+++ b/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java
@@ -458,9 +458,8 @@ public class Avros {
private final List<AvroType> avroTypes;
private final String jsonSchema;
private final boolean isReflect;
-
- private transient GenericRecord record;
-
+ private transient Schema schema;
+
public TupleToGenericRecord(Schema schema, PType<?>... ptypes) {
this.fns = Lists.newArrayList();
this.avroTypes = Lists.newArrayList();
@@ -493,19 +492,23 @@ public class Avros {
@Override
public void initialize() {
- Schema schema = new Schema.Parser().parse(jsonSchema);
- if (isReflect) {
- this.record = new ReflectGenericRecord(schema);
- } else {
- this.record = new GenericData.Record(schema);
- }
+ this.schema = new Schema.Parser().parse(jsonSchema);
for (MapFn fn : fns) {
fn.setContext(getContext());
}
}
+
+ private GenericRecord createRecord(){
+ if (isReflect) {
+ return new ReflectGenericRecord(schema);
+ } else {
+ return new GenericData.Record(schema);
+ }
+ }
@Override
public GenericRecord map(Tuple input) {
+ GenericRecord record = createRecord();
for (int i = 0; i < input.size(); i++) {
Object v = input.get(i);
if (v == null) {
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/0666b0f8/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
----------------------------------------------------------------------
diff --git a/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java b/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
index 082ddec..aec19f2 100644
--- a/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
+++ b/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.crunch.types.avro;
+import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -37,6 +38,7 @@ import org.apache.crunch.Tuple3;
import org.apache.crunch.Tuple4;
import org.apache.crunch.TupleN;
import org.apache.crunch.test.Person;
+import org.apache.crunch.test.StringWrapper;
import org.apache.crunch.types.PTableType;
import org.apache.crunch.types.PType;
import org.apache.hadoop.io.IntWritable;
@@ -291,5 +293,22 @@ public class AvrosTest {
assertEquals(pair, doubleMappedPair);
}
+
+ @Test
+ public void testPairOutputMapFn_VerifyNoObjectReuse(){
+ StringWrapper stringWrapper = new StringWrapper("Test");
+
+ Pair<Integer,StringWrapper> pair = Pair.of(1, stringWrapper);
+
+ AvroType<Pair<Integer, StringWrapper>> pairType = Avros.pairs(Avros.ints(), Avros.reflects(StringWrapper.class));
+
+ pairType.getOutputMapFn().initialize();
+
+ Object outputMappedValueA = pairType.getOutputMapFn().map(pair);
+ Object outputMappedValueB = pairType.getOutputMapFn().map(pair);
+
+ assertEquals(outputMappedValueA, outputMappedValueB);
+ assertNotSame(outputMappedValueA, outputMappedValueB);
+ }
}