You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2010/07/21 23:14:31 UTC
svn commit: r966422 [2/2] - in /avro/trunk: ./
lang/java/src/java/org/apache/avro/generic/
lang/java/src/java/org/apache/avro/mapred/
lang/java/src/java/org/apache/avro/mapred/tether/
lang/java/src/java/org/apache/avro/reflect/ lang/java/src/java/org/a...
Modified: avro/trunk/lang/java/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java?rev=966422&r1=966421&r2=966422&view=diff
==============================================================================
--- avro/trunk/lang/java/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java (original)
+++ avro/trunk/lang/java/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java Wed Jul 21 21:14:29 2010
@@ -22,32 +22,30 @@ import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.avro.util.Utf8;
-import org.apache.avro.mapred.WordCount;
+import org.apache.avro.mapred.Pair;
/** Example Java tethered mapreduce executable. Implements map and reduce
* functions for word count. */
-public class WordCountTask extends TetherTask<Utf8,WordCount,WordCount> {
+public class WordCountTask
+ extends TetherTask<Utf8,Pair<Utf8,Long>,Pair<Utf8,Long>> {
- @Override public void map(Utf8 text, Collector<WordCount> collector)
+ @Override public void map(Utf8 text, Collector<Pair<Utf8,Long>> collector)
throws IOException {
StringTokenizer tokens = new StringTokenizer(text.toString());
- while (tokens.hasMoreTokens()) {
- WordCount wc = new WordCount();
- wc.word = new Utf8(tokens.nextToken());
- wc.count = 1;
- collector.collect(wc);
- }
+ while (tokens.hasMoreTokens())
+ collector.collect(new Pair<Utf8,Long>(new Utf8(tokens.nextToken()),1L));
}
- private int sum;
+ private long sum;
- @Override public void reduce(WordCount wc, Collector<WordCount> c) {
- sum += wc.count;
+ @Override public void reduce(Pair<Utf8,Long> wc,
+ Collector<Pair<Utf8,Long>> c) {
+ sum += wc.value();
}
- @Override public void reduceFlush(WordCount wc, Collector<WordCount> c)
+ @Override public void reduceFlush(Pair<Utf8,Long> wc, Collector<Pair<Utf8,Long>> c)
throws IOException {
- wc.count = sum;
+ wc.value(sum);
c.collect(wc);
sum = 0;
}
Modified: avro/trunk/lang/java/src/test/java/org/apache/avro/specific/TestSpecificData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/src/test/java/org/apache/avro/specific/TestSpecificData.java?rev=966422&r1=966421&r2=966422&view=diff
==============================================================================
--- avro/trunk/lang/java/src/test/java/org/apache/avro/specific/TestSpecificData.java (original)
+++ avro/trunk/lang/java/src/test/java/org/apache/avro/specific/TestSpecificData.java Wed Jul 21 21:14:29 2010
@@ -17,11 +17,21 @@
*/
package org.apache.avro.specific;
+import java.util.List;
+import java.util.ArrayList;
+
import static org.junit.Assert.*;
import org.junit.Test;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.util.Utf8;
+
+import org.apache.avro.TestSchema;
import org.apache.avro.test.TestRecord;
+import org.apache.avro.test.MD5;
+import org.apache.avro.test.Kind;
public class TestSpecificData {
@@ -38,4 +48,32 @@ public class TestSpecificData {
new TestRecord().toString();
}
+ @Test
+ /** Test nesting of specific data within generic. */
+ public void testSpecificWithinGeneric() throws Exception {
+ // define a record with a field that's a generated TestRecord
+ Schema schema = Schema.createRecord("Foo", "", "x.y.z", false);
+ List<Schema.Field> fields = new ArrayList<Schema.Field>();
+ fields.add(new Schema.Field("f", TestRecord.SCHEMA$, "", null));
+ schema.setFields(fields);
+
+ // create a generic instance of this record
+ TestRecord nested = new TestRecord();
+ nested.name = new Utf8("foo");
+ nested.kind = Kind.BAR;
+ nested.hash = new MD5();
+ System.arraycopy(new byte[]{0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5}, 0,
+ nested.hash.bytes(), 0, 16);
+ GenericData.Record record = new GenericData.Record(schema);
+ record.put("f", nested);
+
+ // test that this instance can be written & re-read
+ TestSchema.checkBinary(schema, record,
+ new SpecificDatumWriter<Object>(),
+ new SpecificDatumReader<Object>());
+
+ }
+
+
+
}
Added: avro/trunk/share/test/data/weather-sorted.avro
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/data/weather-sorted.avro?rev=966422&view=auto
==============================================================================
Binary file - no diff available.
Propchange: avro/trunk/share/test/data/weather-sorted.avro
------------------------------------------------------------------------------
svn:executable = *
Propchange: avro/trunk/share/test/data/weather-sorted.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: avro/trunk/share/test/data/weather.avro
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/data/weather.avro?rev=966422&view=auto
==============================================================================
Binary file - no diff available.
Propchange: avro/trunk/share/test/data/weather.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: avro/trunk/share/test/data/weather.json
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/data/weather.json?rev=966422&view=auto
==============================================================================
--- avro/trunk/share/test/data/weather.json (added)
+++ avro/trunk/share/test/data/weather.json Wed Jul 21 21:14:29 2010
@@ -0,0 +1,5 @@
+{"station":"011990-99999","time":-619524000000,"temp":0}
+{"station":"011990-99999","time":-619506000000,"temp":22}
+{"station":"011990-99999","time":-619484400000,"temp":-11}
+{"station":"012650-99999","time":-655531200000,"temp":111}
+{"station":"012650-99999","time":-655509600000,"temp":78}
Added: avro/trunk/share/test/schemas/weather.avsc
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/schemas/weather.avsc?rev=966422&view=auto
==============================================================================
--- avro/trunk/share/test/schemas/weather.avsc (added)
+++ avro/trunk/share/test/schemas/weather.avsc Wed Jul 21 21:14:29 2010
@@ -0,0 +1,8 @@
+{"type": "record", "name": "test.Weather",
+ "doc": "A weather reading.",
+ "fields": [
+ {"name": "station", "type": "string", "order": "ignore"},
+ {"name": "time", "type": "long"},
+ {"name": "temp", "type": "int"}
+ ]
+}