You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2012/07/07 23:49:07 UTC
[19/28] Rename com.cloudera.crunch -> org.apache.crunch in the Java
core
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/lib/join/MapsideJoinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/lib/join/MapsideJoinTest.java b/src/test/java/com/cloudera/crunch/lib/join/MapsideJoinTest.java
deleted file mode 100644
index 97e0c63..0000000
--- a/src/test/java/com/cloudera/crunch/lib/join/MapsideJoinTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-package com.cloudera.crunch.lib.join;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import org.junit.Test;
-
-import com.cloudera.crunch.FilterFn;
-import com.cloudera.crunch.MapFn;
-import com.cloudera.crunch.PTable;
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.Pipeline;
-import com.cloudera.crunch.impl.mem.MemPipeline;
-import com.cloudera.crunch.impl.mr.MRPipeline;
-import com.cloudera.crunch.impl.mr.run.CrunchRuntimeException;
-import com.cloudera.crunch.test.FileHelper;
-import com.cloudera.crunch.types.writable.Writables;
-import com.google.common.collect.Lists;
-
-public class MapsideJoinTest {
-
- private static class LineSplitter extends MapFn<String, Pair<Integer, String>> {
-
- @Override
- public Pair<Integer, String> map(String input) {
- String[] fields = input.split("\\|");
- return Pair.of(Integer.parseInt(fields[0]), fields[1]);
- }
-
- }
-
- private static class NegativeFilter extends FilterFn<Pair<Integer, String>> {
-
- @Override
- public boolean accept(Pair<Integer, String> input) {
- return false;
- }
-
- }
-
- @Test(expected = CrunchRuntimeException.class)
- public void testNonMapReducePipeline() {
- runMapsideJoin(MemPipeline.getInstance());
- }
-
- @Test
- public void testMapsideJoin_RightSideIsEmpty() throws IOException {
- MRPipeline pipeline = new MRPipeline(MapsideJoinTest.class);
- PTable<Integer, String> customerTable = readTable(pipeline, "customers.txt");
- PTable<Integer, String> orderTable = readTable(pipeline, "orders.txt");
-
- PTable<Integer, String> filteredOrderTable = orderTable.parallelDo(new NegativeFilter(),
- orderTable.getPTableType());
-
- PTable<Integer, Pair<String, String>> joined = MapsideJoin.join(customerTable,
- filteredOrderTable);
-
- List<Pair<Integer, Pair<String, String>>> materializedJoin = Lists.newArrayList(joined
- .materialize());
-
- assertTrue(materializedJoin.isEmpty());
-
- }
-
- @Test
- public void testMapsideJoin() throws IOException {
- runMapsideJoin(new MRPipeline(MapsideJoinTest.class));
- }
-
- private void runMapsideJoin(Pipeline pipeline) {
- PTable<Integer, String> customerTable = readTable(pipeline, "customers.txt");
- PTable<Integer, String> orderTable = readTable(pipeline, "orders.txt");
-
- PTable<Integer, Pair<String, String>> joined = MapsideJoin.join(customerTable, orderTable);
-
- List<Pair<Integer, Pair<String, String>>> expectedJoinResult = Lists.newArrayList();
- expectedJoinResult.add(Pair.of(111, Pair.of("John Doe", "Corn flakes")));
- expectedJoinResult.add(Pair.of(222, Pair.of("Jane Doe", "Toilet paper")));
- expectedJoinResult.add(Pair.of(222, Pair.of("Jane Doe", "Toilet plunger")));
- expectedJoinResult.add(Pair.of(333, Pair.of("Someone Else", "Toilet brush")));
-
- List<Pair<Integer, Pair<String, String>>> joinedResultList = Lists.newArrayList(joined
- .materialize());
- Collections.sort(joinedResultList);
-
- assertEquals(expectedJoinResult, joinedResultList);
- }
-
- private static PTable<Integer, String> readTable(Pipeline pipeline, String filename) {
- try {
- return pipeline.readTextFile(FileHelper.createTempCopyOf(filename)).parallelDo("asTable",
- new LineSplitter(), Writables.tableOf(Writables.ints(), Writables.strings()));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/lib/join/MultiAvroSchemaJoinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/lib/join/MultiAvroSchemaJoinTest.java b/src/test/java/com/cloudera/crunch/lib/join/MultiAvroSchemaJoinTest.java
deleted file mode 100644
index be17bf6..0000000
--- a/src/test/java/com/cloudera/crunch/lib/join/MultiAvroSchemaJoinTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.lib.join;
-
-import static com.cloudera.crunch.types.avro.Avros.records;
-import static com.cloudera.crunch.types.avro.Avros.strings;
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.specific.SpecificRecord;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.cloudera.crunch.MapFn;
-import com.cloudera.crunch.PCollection;
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.Pipeline;
-import com.cloudera.crunch.impl.mr.MRPipeline;
-import com.cloudera.crunch.io.From;
-import com.cloudera.crunch.test.Employee;
-import com.cloudera.crunch.test.Person;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-public class MultiAvroSchemaJoinTest {
-
- private File personFile;
- private File employeeFile;
-
- @Before
- public void setUp() throws Exception {
- this.personFile = File.createTempFile("person", ".avro");
- this.employeeFile = File.createTempFile("employee", ".avro");
-
- DatumWriter<Person> pdw = new SpecificDatumWriter<Person>();
- DataFileWriter<Person> pfw = new DataFileWriter<Person>(pdw);
- pfw.create(Person.SCHEMA$, personFile);
- Person p1 = new Person();
- p1.setName("Josh");
- p1.setAge(19);
- p1.setSiblingnames(ImmutableList.<CharSequence>of("Kate", "Mike"));
- pfw.append(p1);
- Person p2 = new Person();
- p2.setName("Kate");
- p2.setAge(17);
- p2.setSiblingnames(ImmutableList.<CharSequence>of("Josh", "Mike"));
- pfw.append(p2);
- Person p3 = new Person();
- p3.setName("Mike");
- p3.setAge(12);
- p3.setSiblingnames(ImmutableList.<CharSequence>of("Josh", "Kate"));
- pfw.append(p3);
- pfw.close();
-
- DatumWriter<Employee> edw = new SpecificDatumWriter<Employee>();
- DataFileWriter<Employee> efw = new DataFileWriter<Employee>(edw);
- efw.create(Employee.SCHEMA$, employeeFile);
- Employee e1 = new Employee();
- e1.setName("Kate");
- e1.setSalary(100000);
- e1.setDepartment("Marketing");
- efw.append(e1);
- efw.close();
- }
-
- @After
- public void tearDown() throws Exception {
- personFile.delete();
- employeeFile.delete();
- }
-
- public static class NameFn<K extends SpecificRecord> extends MapFn<K, String> {
- @Override
- public String map(K input) {
- Schema s = input.getSchema();
- Schema.Field f = s.getField("name");
- return input.get(f.pos()).toString();
- }
- }
-
- @Test
- public void testJoin() throws Exception {
- Pipeline p = new MRPipeline(MultiAvroSchemaJoinTest.class);
- PCollection<Person> people = p.read(From.avroFile(personFile.getAbsolutePath(), records(Person.class)));
- PCollection<Employee> employees = p.read(From.avroFile(employeeFile.getAbsolutePath(), records(Employee.class)));
-
- Iterable<Pair<Person, Employee>> result = people.by(new NameFn<Person>(), strings())
- .join(employees.by(new NameFn<Employee>(), strings())).values().materialize();
- List<Pair<Person, Employee>> v = Lists.newArrayList(result);
- assertEquals(1, v.size());
- assertEquals("Kate", v.get(0).first().getName().toString());
- assertEquals("Kate", v.get(0).second().getName().toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/lib/join/RightOuterJoinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/lib/join/RightOuterJoinTest.java b/src/test/java/com/cloudera/crunch/lib/join/RightOuterJoinTest.java
deleted file mode 100644
index 1daed57..0000000
--- a/src/test/java/com/cloudera/crunch/lib/join/RightOuterJoinTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.lib.join;
-
-import static org.junit.Assert.assertTrue;
-
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.types.PTypeFamily;
-
-public class RightOuterJoinTest extends JoinTester {
- @Override
- public void assertPassed(Iterable<Pair<String, Long>> lines) {
- boolean passed1 = false;
- boolean passed2 = true;
- boolean passed3 = false;
- for (Pair<String, Long> line : lines) {
- if ("wretched".equals(line.first()) && 24 == line.second()) {
- passed1 = true;
- }
- if ("againe".equals(line.first())) {
- passed2 = false;
- }
- if ("Montparnasse.".equals(line.first()) && 2 == line.second()) {
- passed3 = true;
- }
- }
- assertTrue(passed1);
- assertTrue(passed2);
- assertTrue(passed3);
- }
-
- @Override
- protected JoinFn<String, Long, Long> getJoinFn(PTypeFamily typeFamily) {
- return new RightOuterJoinFn<String, Long, Long>(typeFamily.longs());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/test/CountersTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/test/CountersTest.java b/src/test/java/com/cloudera/crunch/test/CountersTest.java
deleted file mode 100644
index 635cad7..0000000
--- a/src/test/java/com/cloudera/crunch/test/CountersTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.test;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.After;
-import org.junit.Test;
-
-import com.cloudera.crunch.DoFn;
-import com.cloudera.crunch.Emitter;
-
-/**
- * A test to verify using counters inside of a unit test works. :)
- */
-public class CountersTest {
-
- public enum CT { ONE, TWO, THREE };
-
- @After
- public void after() {
- TestCounters.clearCounters();
- }
-
- public static class CTFn extends DoFn<String, String> {
- @Override
- public void process(String input, Emitter<String> emitter) {
- getCounter(CT.ONE).increment(1);
- getCounter(CT.TWO).increment(4);
- getCounter(CT.THREE).increment(7);
- }
- }
-
- @Test public void test() {
- CTFn fn = new CTFn();
- fn.process("foo", null);
- fn.process("bar", null);
- assertEquals(2L, TestCounters.getCounter(CT.ONE).getValue());
- assertEquals(8L, TestCounters.getCounter(CT.TWO).getValue());
- assertEquals(14L, TestCounters.getCounter(CT.THREE).getValue());
- }
-
- @Test public void secondTest() {
- CTFn fn = new CTFn();
- fn.process("foo", null);
- fn.process("bar", null);
- assertEquals(2L, TestCounters.getCounter(CT.ONE).getValue());
- assertEquals(8L, TestCounters.getCounter(CT.TWO).getValue());
- assertEquals(14L, TestCounters.getCounter(CT.THREE).getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/test/Employee.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/test/Employee.java b/src/test/java/com/cloudera/crunch/test/Employee.java
deleted file mode 100644
index 40dac2f..0000000
--- a/src/test/java/com/cloudera/crunch/test/Employee.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Autogenerated by Avro
- *
- * DO NOT EDIT DIRECTLY
- */
-package com.cloudera.crunch.test;
-@SuppressWarnings("all")
-public class Employee extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Employee\",\"namespace\":\"com.cloudera.crunch.test\",\"fields\":[{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"salary\",\"type\":\"int\"},{\"name\":\"department\",\"type\":[\"string\",\"null\"]}]}");
- @Deprecated public java.lang.CharSequence name;
- @Deprecated public int salary;
- @Deprecated public java.lang.CharSequence department;
- public org.apache.avro.Schema getSchema() { return SCHEMA$; }
- // Used by DatumWriter. Applications should not call.
- public java.lang.Object get(int field$) {
- switch (field$) {
- case 0: return name;
- case 1: return salary;
- case 2: return department;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
- // Used by DatumReader. Applications should not call.
- @SuppressWarnings(value="unchecked")
- public void put(int field$, java.lang.Object value$) {
- switch (field$) {
- case 0: name = (java.lang.CharSequence)value$; break;
- case 1: salary = (java.lang.Integer)value$; break;
- case 2: department = (java.lang.CharSequence)value$; break;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
-
- /**
- * Gets the value of the 'name' field.
- */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /**
- * Sets the value of the 'name' field.
- * @param value the value to set.
- */
- public void setName(java.lang.CharSequence value) {
- this.name = value;
- }
-
- /**
- * Gets the value of the 'salary' field.
- */
- public java.lang.Integer getSalary() {
- return salary;
- }
-
- /**
- * Sets the value of the 'salary' field.
- * @param value the value to set.
- */
- public void setSalary(java.lang.Integer value) {
- this.salary = value;
- }
-
- /**
- * Gets the value of the 'department' field.
- */
- public java.lang.CharSequence getDepartment() {
- return department;
- }
-
- /**
- * Sets the value of the 'department' field.
- * @param value the value to set.
- */
- public void setDepartment(java.lang.CharSequence value) {
- this.department = value;
- }
-
- /** Creates a new Employee RecordBuilder */
- public static com.cloudera.crunch.test.Employee.Builder newBuilder() {
- return new com.cloudera.crunch.test.Employee.Builder();
- }
-
- /** Creates a new Employee RecordBuilder by copying an existing Builder */
- public static com.cloudera.crunch.test.Employee.Builder newBuilder(com.cloudera.crunch.test.Employee.Builder other) {
- return new com.cloudera.crunch.test.Employee.Builder(other);
- }
-
- /** Creates a new Employee RecordBuilder by copying an existing Employee instance */
- public static com.cloudera.crunch.test.Employee.Builder newBuilder(com.cloudera.crunch.test.Employee other) {
- return new com.cloudera.crunch.test.Employee.Builder(other);
- }
-
- /**
- * RecordBuilder for Employee instances.
- */
- public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<Employee>
- implements org.apache.avro.data.RecordBuilder<Employee> {
-
- private java.lang.CharSequence name;
- private int salary;
- private java.lang.CharSequence department;
-
- /** Creates a new Builder */
- private Builder() {
- super(com.cloudera.crunch.test.Employee.SCHEMA$);
- }
-
- /** Creates a Builder by copying an existing Builder */
- private Builder(com.cloudera.crunch.test.Employee.Builder other) {
- super(other);
- }
-
- /** Creates a Builder by copying an existing Employee instance */
- private Builder(com.cloudera.crunch.test.Employee other) {
- super(com.cloudera.crunch.test.Employee.SCHEMA$);
- if (isValidValue(fields()[0], other.name)) {
- this.name = (java.lang.CharSequence) data().deepCopy(fields()[0].schema(), other.name);
- fieldSetFlags()[0] = true;
- }
- if (isValidValue(fields()[1], other.salary)) {
- this.salary = (java.lang.Integer) data().deepCopy(fields()[1].schema(), other.salary);
- fieldSetFlags()[1] = true;
- }
- if (isValidValue(fields()[2], other.department)) {
- this.department = (java.lang.CharSequence) data().deepCopy(fields()[2].schema(), other.department);
- fieldSetFlags()[2] = true;
- }
- }
-
- /** Gets the value of the 'name' field */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /** Sets the value of the 'name' field */
- public com.cloudera.crunch.test.Employee.Builder setName(java.lang.CharSequence value) {
- validate(fields()[0], value);
- this.name = value;
- fieldSetFlags()[0] = true;
- return this;
- }
-
- /** Checks whether the 'name' field has been set */
- public boolean hasName() {
- return fieldSetFlags()[0];
- }
-
- /** Clears the value of the 'name' field */
- public com.cloudera.crunch.test.Employee.Builder clearName() {
- name = null;
- fieldSetFlags()[0] = false;
- return this;
- }
-
- /** Gets the value of the 'salary' field */
- public java.lang.Integer getSalary() {
- return salary;
- }
-
- /** Sets the value of the 'salary' field */
- public com.cloudera.crunch.test.Employee.Builder setSalary(int value) {
- validate(fields()[1], value);
- this.salary = value;
- fieldSetFlags()[1] = true;
- return this;
- }
-
- /** Checks whether the 'salary' field has been set */
- public boolean hasSalary() {
- return fieldSetFlags()[1];
- }
-
- /** Clears the value of the 'salary' field */
- public com.cloudera.crunch.test.Employee.Builder clearSalary() {
- fieldSetFlags()[1] = false;
- return this;
- }
-
- /** Gets the value of the 'department' field */
- public java.lang.CharSequence getDepartment() {
- return department;
- }
-
- /** Sets the value of the 'department' field */
- public com.cloudera.crunch.test.Employee.Builder setDepartment(java.lang.CharSequence value) {
- validate(fields()[2], value);
- this.department = value;
- fieldSetFlags()[2] = true;
- return this;
- }
-
- /** Checks whether the 'department' field has been set */
- public boolean hasDepartment() {
- return fieldSetFlags()[2];
- }
-
- /** Clears the value of the 'department' field */
- public com.cloudera.crunch.test.Employee.Builder clearDepartment() {
- department = null;
- fieldSetFlags()[2] = false;
- return this;
- }
-
- @Override
- public Employee build() {
- try {
- Employee record = new Employee();
- record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
- record.salary = fieldSetFlags()[1] ? this.salary : (java.lang.Integer) defaultValue(fields()[1]);
- record.department = fieldSetFlags()[2] ? this.department : (java.lang.CharSequence) defaultValue(fields()[2]);
- return record;
- } catch (Exception e) {
- throw new org.apache.avro.AvroRuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/test/Person.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/test/Person.java b/src/test/java/com/cloudera/crunch/test/Person.java
deleted file mode 100644
index 0d8e073..0000000
--- a/src/test/java/com/cloudera/crunch/test/Person.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Autogenerated by Avro
- *
- * DO NOT EDIT DIRECTLY
- */
-package com.cloudera.crunch.test;
-@SuppressWarnings("all")
-public class Person extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.cloudera.crunch.test\",\"fields\":[{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"age\",\"type\":\"int\"},{\"name\":\"siblingnames\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}");
- @Deprecated public java.lang.CharSequence name;
- @Deprecated public int age;
- @Deprecated public java.util.List<java.lang.CharSequence> siblingnames;
- public org.apache.avro.Schema getSchema() { return SCHEMA$; }
- // Used by DatumWriter. Applications should not call.
- public java.lang.Object get(int field$) {
- switch (field$) {
- case 0: return name;
- case 1: return age;
- case 2: return siblingnames;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
- // Used by DatumReader. Applications should not call.
- @SuppressWarnings(value="unchecked")
- public void put(int field$, java.lang.Object value$) {
- switch (field$) {
- case 0: name = (java.lang.CharSequence)value$; break;
- case 1: age = (java.lang.Integer)value$; break;
- case 2: siblingnames = (java.util.List<java.lang.CharSequence>)value$; break;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
-
- /**
- * Gets the value of the 'name' field.
- */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /**
- * Sets the value of the 'name' field.
- * @param value the value to set.
- */
- public void setName(java.lang.CharSequence value) {
- this.name = value;
- }
-
- /**
- * Gets the value of the 'age' field.
- */
- public java.lang.Integer getAge() {
- return age;
- }
-
- /**
- * Sets the value of the 'age' field.
- * @param value the value to set.
- */
- public void setAge(java.lang.Integer value) {
- this.age = value;
- }
-
- /**
- * Gets the value of the 'siblingnames' field.
- */
- public java.util.List<java.lang.CharSequence> getSiblingnames() {
- return siblingnames;
- }
-
- /**
- * Sets the value of the 'siblingnames' field.
- * @param value the value to set.
- */
- public void setSiblingnames(java.util.List<java.lang.CharSequence> value) {
- this.siblingnames = value;
- }
-
- /** Creates a new Person RecordBuilder */
- public static com.cloudera.crunch.test.Person.Builder newBuilder() {
- return new com.cloudera.crunch.test.Person.Builder();
- }
-
- /** Creates a new Person RecordBuilder by copying an existing Builder */
- public static com.cloudera.crunch.test.Person.Builder newBuilder(com.cloudera.crunch.test.Person.Builder other) {
- return new com.cloudera.crunch.test.Person.Builder(other);
- }
-
- /** Creates a new Person RecordBuilder by copying an existing Person instance */
- public static com.cloudera.crunch.test.Person.Builder newBuilder(com.cloudera.crunch.test.Person other) {
- return new com.cloudera.crunch.test.Person.Builder(other);
- }
-
- /**
- * RecordBuilder for Person instances.
- */
- public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<Person>
- implements org.apache.avro.data.RecordBuilder<Person> {
-
- private java.lang.CharSequence name;
- private int age;
- private java.util.List<java.lang.CharSequence> siblingnames;
-
- /** Creates a new Builder */
- private Builder() {
- super(com.cloudera.crunch.test.Person.SCHEMA$);
- }
-
- /** Creates a Builder by copying an existing Builder */
- private Builder(com.cloudera.crunch.test.Person.Builder other) {
- super(other);
- }
-
- /** Creates a Builder by copying an existing Person instance */
- private Builder(com.cloudera.crunch.test.Person other) {
- super(com.cloudera.crunch.test.Person.SCHEMA$);
- if (isValidValue(fields()[0], other.name)) {
- this.name = (java.lang.CharSequence) data().deepCopy(fields()[0].schema(), other.name);
- fieldSetFlags()[0] = true;
- }
- if (isValidValue(fields()[1], other.age)) {
- this.age = (java.lang.Integer) data().deepCopy(fields()[1].schema(), other.age);
- fieldSetFlags()[1] = true;
- }
- if (isValidValue(fields()[2], other.siblingnames)) {
- this.siblingnames = (java.util.List<java.lang.CharSequence>) data().deepCopy(fields()[2].schema(), other.siblingnames);
- fieldSetFlags()[2] = true;
- }
- }
-
- /** Gets the value of the 'name' field */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /** Sets the value of the 'name' field */
- public com.cloudera.crunch.test.Person.Builder setName(java.lang.CharSequence value) {
- validate(fields()[0], value);
- this.name = value;
- fieldSetFlags()[0] = true;
- return this;
- }
-
- /** Checks whether the 'name' field has been set */
- public boolean hasName() {
- return fieldSetFlags()[0];
- }
-
- /** Clears the value of the 'name' field */
- public com.cloudera.crunch.test.Person.Builder clearName() {
- name = null;
- fieldSetFlags()[0] = false;
- return this;
- }
-
- /** Gets the value of the 'age' field */
- public java.lang.Integer getAge() {
- return age;
- }
-
- /** Sets the value of the 'age' field */
- public com.cloudera.crunch.test.Person.Builder setAge(int value) {
- validate(fields()[1], value);
- this.age = value;
- fieldSetFlags()[1] = true;
- return this;
- }
-
- /** Checks whether the 'age' field has been set */
- public boolean hasAge() {
- return fieldSetFlags()[1];
- }
-
- /** Clears the value of the 'age' field */
- public com.cloudera.crunch.test.Person.Builder clearAge() {
- fieldSetFlags()[1] = false;
- return this;
- }
-
- /** Gets the value of the 'siblingnames' field */
- public java.util.List<java.lang.CharSequence> getSiblingnames() {
- return siblingnames;
- }
-
- /** Sets the value of the 'siblingnames' field */
- public com.cloudera.crunch.test.Person.Builder setSiblingnames(java.util.List<java.lang.CharSequence> value) {
- validate(fields()[2], value);
- this.siblingnames = value;
- fieldSetFlags()[2] = true;
- return this;
- }
-
- /** Checks whether the 'siblingnames' field has been set */
- public boolean hasSiblingnames() {
- return fieldSetFlags()[2];
- }
-
- /** Clears the value of the 'siblingnames' field */
- public com.cloudera.crunch.test.Person.Builder clearSiblingnames() {
- siblingnames = null;
- fieldSetFlags()[2] = false;
- return this;
- }
-
- @Override
- public Person build() {
- try {
- Person record = new Person();
- record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
- record.age = fieldSetFlags()[1] ? this.age : (java.lang.Integer) defaultValue(fields()[1]);
- record.siblingnames = fieldSetFlags()[2] ? this.siblingnames : (java.util.List<java.lang.CharSequence>) defaultValue(fields()[2]);
- return record;
- } catch (Exception e) {
- throw new org.apache.avro.AvroRuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/PTypeUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/PTypeUtilsTest.java b/src/test/java/com/cloudera/crunch/types/PTypeUtilsTest.java
deleted file mode 100644
index dcc9ef6..0000000
--- a/src/test/java/com/cloudera/crunch/types/PTypeUtilsTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.types;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import java.util.Collection;
-
-import org.apache.avro.Schema;
-import org.apache.avro.util.Utf8;
-import org.apache.hadoop.io.Text;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.cloudera.crunch.Tuple3;
-import com.cloudera.crunch.TupleN;
-import com.cloudera.crunch.types.avro.AvroType;
-import com.cloudera.crunch.types.avro.AvroTypeFamily;
-import com.cloudera.crunch.types.avro.Avros;
-import com.cloudera.crunch.types.writable.WritableTypeFamily;
-import com.cloudera.crunch.types.writable.Writables;
-
-public class PTypeUtilsTest {
- @Test
- public void testPrimitives() {
- assertEquals(Avros.strings(), AvroTypeFamily.getInstance().as(Writables.strings()));
- Assert.assertEquals(Writables.doubles(), WritableTypeFamily.getInstance().as(Avros.doubles()));
- }
-
- @Test
- public void testTuple3() {
- PType<Tuple3<String, Float, Integer>> t = Writables.triples(Writables.strings(),
- Writables.floats(), Writables.ints());
- PType<Tuple3<String, Float, Integer>> at = AvroTypeFamily.getInstance().as(t);
- assertEquals(Avros.strings(), at.getSubTypes().get(0));
- assertEquals(Avros.floats(), at.getSubTypes().get(1));
- assertEquals(Avros.ints(), at.getSubTypes().get(2));
- }
-
- @Test
- public void testTupleN() {
- PType<TupleN> t = Avros.tuples(Avros.strings(),
- Avros.floats(), Avros.ints());
- PType<TupleN> wt = WritableTypeFamily.getInstance().as(t);
- assertEquals(Writables.strings(), wt.getSubTypes().get(0));
- assertEquals(Writables.floats(), wt.getSubTypes().get(1));
- assertEquals(Writables.ints(), wt.getSubTypes().get(2));
- }
-
- @Test
- public void testWritableCollections() {
- PType<Collection<String>> t = Avros.collections(Avros.strings());
- t = WritableTypeFamily.getInstance().as(t);
- assertEquals(Writables.strings(), t.getSubTypes().get(0));
- }
-
- @Test
- public void testAvroCollections() {
- PType<Collection<Double>> t = Writables.collections(Writables.doubles());
- t = AvroTypeFamily.getInstance().as(t);
- assertEquals(Avros.doubles(), t.getSubTypes().get(0));
- }
-
- @Test
- public void testAvroRegistered() {
- AvroType<Utf8> at = new AvroType<Utf8>(Utf8.class, Schema.create(Schema.Type.STRING));
- Avros.register(Utf8.class, at);
- assertEquals(at, Avros.records(Utf8.class));
- }
-
- @Test
- public void testWritableBuiltin() {
- assertNotNull(Writables.records(Text.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/avro/AvroDeepCopierTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/avro/AvroDeepCopierTest.java b/src/test/java/com/cloudera/crunch/types/avro/AvroDeepCopierTest.java
deleted file mode 100644
index cce37cd..0000000
--- a/src/test/java/com/cloudera/crunch/types/avro/AvroDeepCopierTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package com.cloudera.crunch.types.avro;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-
-import org.apache.avro.generic.GenericData.Record;
-import org.junit.Test;
-
-import com.cloudera.crunch.test.Person;
-import com.cloudera.crunch.types.avro.AvroDeepCopier.AvroSpecificDeepCopier;
-import com.google.common.collect.Lists;
-
-public class AvroDeepCopierTest {
-
- @Test
- public void testDeepCopySpecific() {
- Person person = new Person();
- person.setName("John Doe");
- person.setAge(42);
- person.setSiblingnames(Lists.<CharSequence> newArrayList());
-
- Person deepCopyPerson = new AvroSpecificDeepCopier<Person>(Person.class, Person.SCHEMA$)
- .deepCopy(person);
-
- assertEquals(person, deepCopyPerson);
- assertNotSame(person, deepCopyPerson);
- }
-
- @Test
- public void testDeepCopyGeneric() {
- Record record = new Record(Person.SCHEMA$);
- record.put("name", "John Doe");
- record.put("age", 42);
- record.put("siblingnames", Lists.newArrayList());
-
- Record deepCopyRecord = new AvroDeepCopier.AvroGenericDeepCopier(Person.SCHEMA$)
- .deepCopy(record);
-
- assertEquals(record, deepCopyRecord);
- assertNotSame(record, deepCopyRecord);
- }
-
- @Test
- public void testDeepCopyReflect() {
- Person person = new Person();
- person.setName("John Doe");
- person.setAge(42);
- person.setSiblingnames(Lists.<CharSequence> newArrayList());
-
- Person deepCopyPerson = new AvroDeepCopier.AvroReflectDeepCopier<Person>(Person.class,
- Person.SCHEMA$).deepCopy(person);
-
- assertEquals(person, deepCopyPerson);
- assertNotSame(person, deepCopyPerson);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/avro/AvroGroupedTableTypeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/avro/AvroGroupedTableTypeTest.java b/src/test/java/com/cloudera/crunch/types/avro/AvroGroupedTableTypeTest.java
deleted file mode 100644
index 134dd9d..0000000
--- a/src/test/java/com/cloudera/crunch/types/avro/AvroGroupedTableTypeTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package com.cloudera.crunch.types.avro;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import java.util.List;
-
-import org.junit.Test;
-
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.test.Person;
-import com.cloudera.crunch.types.PGroupedTableType;
-import com.google.common.collect.Lists;
-
-public class AvroGroupedTableTypeTest {
-
- @Test
- public void testGetDetachedValue() {
- Integer integerValue = 42;
- Person person = new Person();
- person.setName("John Doe");
- person.setAge(42);
- person.setSiblingnames(Lists.<CharSequence> newArrayList());
-
- Iterable<Person> inputPersonIterable = Lists.newArrayList(person);
- Pair<Integer, Iterable<Person>> pair = Pair.of(integerValue, inputPersonIterable);
-
- PGroupedTableType<Integer, Person> groupedTableType = Avros.tableOf(Avros.ints(),
- Avros.reflects(Person.class)).getGroupedTableType();
-
- Pair<Integer, Iterable<Person>> detachedPair = groupedTableType.getDetachedValue(pair);
-
- assertSame(integerValue, detachedPair.first());
- List<Person> personList = Lists.newArrayList(detachedPair.second());
- assertEquals(inputPersonIterable, personList);
- assertNotSame(person, personList.get(0));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/avro/AvroTableTypeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/avro/AvroTableTypeTest.java b/src/test/java/com/cloudera/crunch/types/avro/AvroTableTypeTest.java
deleted file mode 100644
index 867ee6f..0000000
--- a/src/test/java/com/cloudera/crunch/types/avro/AvroTableTypeTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package com.cloudera.crunch.types.avro;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import org.junit.Test;
-
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.test.Person;
-import com.google.common.collect.Lists;
-
-public class AvroTableTypeTest {
-
- @Test
- public void testGetDetachedValue() {
- Integer integerValue = 42;
- Person person = new Person();
- person.setName("John Doe");
- person.setAge(42);
- person.setSiblingnames(Lists.<CharSequence> newArrayList());
-
- Pair<Integer, Person> pair = Pair.of(integerValue, person);
-
- AvroTableType<Integer, Person> tableType = Avros.tableOf(Avros.ints(),
- Avros.reflects(Person.class));
-
- Pair<Integer, Person> detachedPair = tableType.getDetachedValue(pair);
-
- assertSame(integerValue, detachedPair.first());
- assertEquals(person, detachedPair.second());
- assertNotSame(person, detachedPair.second());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/avro/AvroTypeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/avro/AvroTypeTest.java b/src/test/java/com/cloudera/crunch/types/avro/AvroTypeTest.java
deleted file mode 100644
index 9992ae3..0000000
--- a/src/test/java/com/cloudera/crunch/types/avro/AvroTypeTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package com.cloudera.crunch.types.avro;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericData.Record;
-import org.junit.Test;
-
-import com.cloudera.crunch.test.Person;
-import com.google.common.collect.Lists;
-
-public class AvroTypeTest {
-
- @Test
- public void testIsSpecific_SpecificData() {
- assertTrue(Avros.records(Person.class).isSpecific());
- }
-
- @Test
- public void testIsGeneric_SpecificData() {
- assertFalse(Avros.records(Person.class).isGeneric());
- }
-
- @Test
- public void testIsSpecific_GenericData() {
- assertFalse(Avros.generics(Person.SCHEMA$).isSpecific());
- }
-
- @Test
- public void testIsGeneric_GenericData() {
- assertTrue(Avros.generics(Person.SCHEMA$).isGeneric());
- }
-
- @Test
- public void testIsSpecific_NonAvroClass() {
- assertFalse(Avros.ints().isSpecific());
- }
-
- @Test
- public void testIsGeneric_NonAvroClass() {
- assertFalse(Avros.ints().isGeneric());
- }
-
- @Test
- public void testIsSpecific_SpecificAvroTable() {
- assertTrue(Avros.tableOf(Avros.strings(), Avros.records(Person.class))
- .isSpecific());
- }
-
- @Test
- public void testIsGeneric_SpecificAvroTable() {
- assertFalse(Avros.tableOf(Avros.strings(), Avros.records(Person.class))
- .isGeneric());
- }
-
- @Test
- public void testIsSpecific_GenericAvroTable() {
- assertFalse(Avros.tableOf(Avros.strings(),
- Avros.generics(Person.SCHEMA$)).isSpecific());
- }
-
- @Test
- public void testIsGeneric_GenericAvroTable() {
- assertTrue(Avros.tableOf(Avros.strings(),
- Avros.generics(Person.SCHEMA$)).isGeneric());
- }
-
- @Test
- public void testGetDetachedValue_AlreadyMappedAvroType() {
- Integer value = 42;
- Integer detachedValue = Avros.ints().getDetachedValue(value);
- assertSame(value, detachedValue);
- }
-
- @Test
- public void testGetDetachedValue_GenericAvroType() {
- AvroType<Record> genericType = Avros.generics(Person.SCHEMA$);
- GenericData.Record record = new GenericData.Record(Person.SCHEMA$);
- record.put("name", "name value");
- record.put("age", 42);
- record.put("siblingnames", Lists.newArrayList());
-
- Record detachedRecord = genericType.getDetachedValue(record);
- assertEquals(record, detachedRecord);
- assertNotSame(record, detachedRecord);
- }
-
- @Test
- public void testGetDetachedValue_SpecificAvroType() {
- AvroType<Person> specificType = Avros.records(Person.class);
- Person person = new Person();
- person.setName("name value");
- person.setAge(42);
- person.setSiblingnames(Lists.<CharSequence> newArrayList());
-
- Person detachedPerson = specificType.getDetachedValue(person);
- assertEquals(person, detachedPerson);
- assertNotSame(person, detachedPerson);
- }
-
- @Test
- public void testGetDetachedValue_ReflectAvroType() {
- AvroType<Person> reflectType = Avros.reflects(Person.class);
- Person person = new Person();
- person.setName("name value");
- person.setAge(42);
- person.setSiblingnames(Lists.<CharSequence> newArrayList());
-
- Person detachedPerson = reflectType.getDetachedValue(person);
- assertEquals(person, detachedPerson);
- assertNotSame(person, detachedPerson);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/avro/AvrosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/avro/AvrosTest.java b/src/test/java/com/cloudera/crunch/types/avro/AvrosTest.java
deleted file mode 100644
index 8bdd084..0000000
--- a/src/test/java/com/cloudera/crunch/types/avro/AvrosTest.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.types.avro;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Type;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.util.Utf8;
-import org.apache.hadoop.io.LongWritable;
-import org.junit.Test;
-
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.Tuple3;
-import com.cloudera.crunch.Tuple4;
-import com.cloudera.crunch.TupleN;
-import com.cloudera.crunch.test.Person;
-import com.cloudera.crunch.types.PTableType;
-import com.cloudera.crunch.types.PType;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-/**
- * TODO test Avros.register and Avros.containers
- */
-public class AvrosTest {
-
- @Test
- public void testNulls() throws Exception {
- Void n = null;
- testInputOutputFn(Avros.nulls(), n, n);
- }
-
- @Test
- public void testStrings() throws Exception {
- String s = "abc";
- Utf8 w = new Utf8(s);
- testInputOutputFn(Avros.strings(), s, w);
- }
-
- @Test
- public void testInts() throws Exception {
- int j = 55;
- testInputOutputFn(Avros.ints(), j, j);
- }
- @Test
- public void testLongs() throws Exception {
- long j = Long.MAX_VALUE;
- testInputOutputFn(Avros.longs(), j, j);
- }
- @Test
- public void testFloats() throws Exception {
- float j = Float.MIN_VALUE;
- testInputOutputFn(Avros.floats(), j, j);
- }
- @Test
- public void testDoubles() throws Exception {
- double j = Double.MIN_VALUE;
- testInputOutputFn(Avros.doubles(), j, j);
- }
-
- @Test
- public void testBooleans() throws Exception {
- boolean j = true;
- testInputOutputFn(Avros.booleans(), j, j);
- }
-
- @Test
- public void testBytes() throws Exception {
- byte[] bytes = new byte[] { 17, 26, -98 };
- ByteBuffer bb = ByteBuffer.wrap(bytes);
- testInputOutputFn(Avros.bytes(), bb, bb);
- }
-
- @Test
- public void testCollections() throws Exception {
- Collection<String> j = Lists.newArrayList();
- j.add("a");
- j.add("b");
- Schema collectionSchema = Schema.createArray(
- Schema.createUnion(ImmutableList.of(
- Avros.strings().getSchema(), Schema.create(Type.NULL))));
- GenericData.Array<Utf8> w = new GenericData.Array<Utf8>(2, collectionSchema);
- w.add(new Utf8("a"));
- w.add(new Utf8("b"));
- testInputOutputFn(Avros.collections(Avros.strings()), j, w);
- }
-
- @Test
- public void testNestedTables() throws Exception {
- PTableType<Long, Long> pll = Avros.tableOf(Avros.longs(), Avros.longs());
- String schema = Avros.tableOf(pll, Avros.strings()).getSchema().toString();
- assertNotNull(schema);
- }
-
- @Test
- public void testPairs() throws Exception {
- AvroType<Pair<String, String>> at = Avros.pairs(Avros.strings(), Avros.strings());
- Pair<String, String> j = Pair.of("a", "b");
- GenericData.Record w = new GenericData.Record(at.getSchema());
- w.put(0, new Utf8("a"));
- w.put(1, new Utf8("b"));
- testInputOutputFn(at, j, w);
- }
-
- @Test
- public void testPairEquals() throws Exception {
- AvroType<Pair<Long, ByteBuffer>> at1 = Avros.pairs(Avros.longs(), Avros.bytes());
- AvroType<Pair<Long, ByteBuffer>> at2 = Avros.pairs(Avros.longs(), Avros.bytes());
- assertEquals(at1, at2);
- assertEquals(at1.hashCode(), at2.hashCode());
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testTriples() throws Exception {
- AvroType at = Avros.triples(Avros.strings(), Avros.strings(), Avros.strings());
- Tuple3 j = Tuple3.of("a", "b", "c");
- GenericData.Record w = new GenericData.Record(at.getSchema());
- w.put(0, new Utf8("a"));
- w.put(1, new Utf8("b"));
- w.put(2, new Utf8("c"));
- testInputOutputFn(at, j, w);
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testQuads() throws Exception {
- AvroType at = Avros.quads(Avros.strings(), Avros.strings(), Avros.strings(), Avros.strings());
- Tuple4 j = Tuple4.of("a", "b", "c", "d");
- GenericData.Record w = new GenericData.Record(at.getSchema());
- w.put(0, new Utf8("a"));
- w.put(1, new Utf8("b"));
- w.put(2, new Utf8("c"));
- w.put(3, new Utf8("d"));
- testInputOutputFn(at, j, w);
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testTupleN() throws Exception {
- AvroType at = Avros.tuples(Avros.strings(), Avros.strings(), Avros.strings(), Avros.strings(),
- Avros.strings());
- TupleN j = new TupleN("a", "b", "c", "d", "e");
- GenericData.Record w = new GenericData.Record(at.getSchema());
- w.put(0, new Utf8("a"));
- w.put(1, new Utf8("b"));
- w.put(2, new Utf8("c"));
- w.put(3, new Utf8("d"));
- w.put(4, new Utf8("e"));
- testInputOutputFn(at, j, w);
-
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testWritables() throws Exception {
- AvroType at = Avros.writables(LongWritable.class);
- LongWritable lw = new LongWritable(1729L);
- assertEquals(lw, at.getInputMapFn().map(at.getOutputMapFn().map(lw)));
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testTableOf() throws Exception {
- AvroType at = Avros.tableOf(Avros.strings(), Avros.strings());
- Pair<String, String> j = Pair.of("a", "b");
- org.apache.avro.mapred.Pair w = new org.apache.avro.mapred.Pair(at.getSchema());
- w.put(0, new Utf8("a"));
- w.put(1, new Utf8("b"));
- // TODO update this after resolving the o.a.a.m.Pair.equals issue
- initialize(at);
- assertEquals(j, at.getInputMapFn().map(w));
- org.apache.avro.mapred.Pair converted =
- (org.apache.avro.mapred.Pair) at.getOutputMapFn().map(j);
- assertEquals(w.key(), converted.key());
- assertEquals(w.value(), converted.value());
- }
-
- private static void initialize(PType ptype) {
- ptype.getInputMapFn().initialize();
- ptype.getOutputMapFn().initialize();
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- protected static void testInputOutputFn(PType ptype, Object java, Object avro) {
- initialize(ptype);
- assertEquals(java, ptype.getInputMapFn().map(avro));
- assertEquals(avro, ptype.getOutputMapFn().map(java));
- }
-
- @Test
- public void testIsPrimitive_True() {
- assertTrue(Avros.isPrimitive(Avros.ints()));
- }
-
- @Test
- public void testIsPrimitive_False() {
- assertFalse(Avros.isPrimitive(Avros.reflects(Person.class)));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/writable/WritableGroupedTableTypeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/writable/WritableGroupedTableTypeTest.java b/src/test/java/com/cloudera/crunch/types/writable/WritableGroupedTableTypeTest.java
deleted file mode 100644
index 2357393..0000000
--- a/src/test/java/com/cloudera/crunch/types/writable/WritableGroupedTableTypeTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package com.cloudera.crunch.types.writable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import java.util.List;
-
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.types.PGroupedTableType;
-import com.google.common.collect.Lists;
-
-public class WritableGroupedTableTypeTest {
-
- @Test
- public void testGetDetachedValue() {
- Integer integerValue = 42;
- Text textValue = new Text("forty-two");
- Iterable<Text> inputTextIterable = Lists.newArrayList(textValue);
- Pair<Integer, Iterable<Text>> pair = Pair.of(integerValue, inputTextIterable);
-
- PGroupedTableType<Integer, Text> groupedTableType = Writables.tableOf(Writables.ints(), Writables.writables(Text.class))
- .getGroupedTableType();
-
- Pair<Integer, Iterable<Text>> detachedPair = groupedTableType.getDetachedValue(pair);
-
- assertSame(integerValue, detachedPair.first());
- List<Text> textList = Lists.newArrayList(detachedPair.second());
- assertEquals(inputTextIterable, textList);
- assertNotSame(textValue, textList.get(0));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/writable/WritableTableTypeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/writable/WritableTableTypeTest.java b/src/test/java/com/cloudera/crunch/types/writable/WritableTableTypeTest.java
deleted file mode 100644
index 96015f6..0000000
--- a/src/test/java/com/cloudera/crunch/types/writable/WritableTableTypeTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.cloudera.crunch.types.writable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import com.cloudera.crunch.Pair;
-
-public class WritableTableTypeTest {
-
- @Test
- public void testGetDetachedValue() {
- Integer integerValue = 42;
- Text textValue = new Text("forty-two");
- Pair<Integer, Text> pair = Pair.of(integerValue, textValue);
-
- WritableTableType<Integer, Text> tableType = Writables.tableOf(Writables.ints(),
- Writables.writables(Text.class));
-
- Pair<Integer, Text> detachedPair = tableType.getDetachedValue(pair);
-
- assertSame(integerValue, detachedPair.first());
- assertEquals(textValue, detachedPair.second());
- assertNotSame(textValue, detachedPair.second());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/writable/WritableTypeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/writable/WritableTypeTest.java b/src/test/java/com/cloudera/crunch/types/writable/WritableTypeTest.java
deleted file mode 100644
index 8dff574..0000000
--- a/src/test/java/com/cloudera/crunch/types/writable/WritableTypeTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package com.cloudera.crunch.types.writable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-public class WritableTypeTest {
-
- @Test
- public void testGetDetachedValue_AlreadyMappedWritable() {
- WritableType<String, Text> stringType = Writables.strings();
- String value = "test";
- assertSame(value, stringType.getDetachedValue(value));
- }
-
- @Test
- public void testGetDetachedValue_CustomWritable() {
- WritableType<Text, Text> textWritableType = Writables.writables(Text.class);
- Text value = new Text("test");
-
- Text detachedValue = textWritableType.getDetachedValue(value);
- assertEquals(value, detachedValue);
- assertNotSame(value, detachedValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/types/writable/WritablesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/types/writable/WritablesTest.java b/src/test/java/com/cloudera/crunch/types/writable/WritablesTest.java
deleted file mode 100644
index 2be2c0b..0000000
--- a/src/test/java/com/cloudera/crunch/types/writable/WritablesTest.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.types.writable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.junit.Test;
-
-import com.cloudera.crunch.Pair;
-import com.cloudera.crunch.Tuple3;
-import com.cloudera.crunch.Tuple4;
-import com.cloudera.crunch.TupleN;
-import com.cloudera.crunch.types.PTableType;
-import com.cloudera.crunch.types.PType;
-import com.google.common.collect.Lists;
-
-public class WritablesTest {
-
- @Test
- public void testNulls() throws Exception {
- Void n = null;
- NullWritable nw = NullWritable.get();
- testInputOutputFn(Writables.nulls(), n, nw);
- }
-
- @Test
- public void testStrings() throws Exception {
- String s = "abc";
- Text text = new Text(s);
- testInputOutputFn(Writables.strings(), s, text);
- }
-
- @Test
- public void testInts() throws Exception {
- int j = 55;
- IntWritable w = new IntWritable(j);
- testInputOutputFn(Writables.ints(), j, w);
- }
-
- @Test
- public void testLongs() throws Exception {
- long j = 55;
- LongWritable w = new LongWritable(j);
- testInputOutputFn(Writables.longs(), j, w);
- }
-
- @Test
- public void testFloats() throws Exception {
- float j = 55.5f;
- FloatWritable w = new FloatWritable(j);
- testInputOutputFn(Writables.floats(), j, w);
- }
-
- @Test
- public void testDoubles() throws Exception {
- double j = 55.5d;
- DoubleWritable w = new DoubleWritable(j);
- testInputOutputFn(Writables.doubles(), j, w);
- }
-
- @Test
- public void testBoolean() throws Exception {
- boolean j = false;
- BooleanWritable w = new BooleanWritable(j);
- testInputOutputFn(Writables.booleans(), j, w);
- }
-
- @Test
- public void testBytes() throws Exception {
- byte[] bytes = new byte[] { 17, 26, -98 };
- BytesWritable bw = new BytesWritable(bytes);
- ByteBuffer bb = ByteBuffer.wrap(bytes);
- testInputOutputFn(Writables.bytes(), bb, bw);
- }
-
- @Test
- public void testCollections() throws Exception {
- String s = "abc";
- Collection<String> j = Lists.newArrayList();
- j.add(s);
- GenericArrayWritable<Text> w = new GenericArrayWritable<Text>(Text.class);
- w.set(new Text[]{
- new Text(s)
- });
- testInputOutputFn(Writables.collections(Writables.strings()), j, w);
- }
-
- @Test
- public void testPairs() throws Exception {
- Pair<String, String> j = Pair.of("a", "b");
- TupleWritable w = new TupleWritable(new Text[] {
- new Text("a"),
- new Text("b"),
- });
- w.setWritten(0);
- w.setWritten(1);
- testInputOutputFn(Writables.pairs(Writables.strings(), Writables.strings()), j, w);
- }
-
- @Test
- public void testNestedTables() throws Exception {
- PTableType<Long, Long> pll = Writables.tableOf(Writables.longs(), Writables.longs());
- PTableType<Pair<Long, Long>, String> nest = Writables.tableOf(pll, Writables.strings());
- assertNotNull(nest);
- }
-
- @Test
- public void testPairEquals() throws Exception {
- PType<Pair<Long, ByteBuffer>> t1 = Writables.pairs(Writables.longs(), Writables.bytes());
- PType<Pair<Long, ByteBuffer>> t2 = Writables.pairs(Writables.longs(), Writables.bytes());
- assertEquals(t1, t2);
- assertEquals(t1.hashCode(), t2.hashCode());
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testTriples() throws Exception {
- Tuple3 j = Tuple3.of("a", "b", "c");
- TupleWritable w = new TupleWritable(new Text[] {
- new Text("a"),
- new Text("b"),
- new Text("c"),
- });
- w.setWritten(0);
- w.setWritten(1);
- w.setWritten(2);
- WritableType<?, ?> wt = Writables.triples(Writables.strings(),
- Writables.strings(), Writables.strings());
- testInputOutputFn(wt, j, w);
- }
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testQuads() throws Exception {
- Tuple4 j = Tuple4.of("a", "b", "c", "d");
- TupleWritable w = new TupleWritable(new Text[] {
- new Text("a"),
- new Text("b"),
- new Text("c"),
- new Text("d"),
- });
- w.setWritten(0);
- w.setWritten(1);
- w.setWritten(2);
- w.setWritten(3);
- WritableType<?, ?> wt = Writables.quads(Writables.strings(), Writables.strings(),
- Writables.strings(), Writables.strings());
- testInputOutputFn(wt, j, w);
- }
-
- @Test
- public void testTupleN() throws Exception {
- TupleN j = new TupleN("a", "b", "c", "d", "e");
- TupleWritable w = new TupleWritable(new Text[] {
- new Text("a"),
- new Text("b"),
- new Text("c"),
- new Text("d"),
- new Text("e"),
- });
- w.setWritten(0);
- w.setWritten(1);
- w.setWritten(2);
- w.setWritten(3);
- w.setWritten(4);
- WritableType<?, ?> wt = Writables.tuples(Writables.strings(), Writables.strings(),
- Writables.strings(), Writables.strings(), Writables.strings());
- testInputOutputFn(wt, j, w);
- }
-
- protected static class TestWritable implements Writable {
- String left;
- int right;
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(left);
- out.writeInt(right);
- }
- @Override
- public void readFields(DataInput in) throws IOException {
- left = in.readUTF();
- right = in.readInt();
- }
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TestWritable other = (TestWritable) obj;
- if (left == null) {
- if (other.left != null)
- return false;
- } else if (!left.equals(other.left))
- return false;
- if (right != other.right)
- return false;
- return true;
- }
-
- }
- @Test
- public void testRecords() throws Exception {
- TestWritable j = new TestWritable();
- j.left = "a";
- j.right = 1;
- TestWritable w = new TestWritable();
- w.left = "a";
- w.right = 1;
- WritableType<?, ?> wt = Writables.records(TestWritable.class);
- testInputOutputFn(wt, j, w);
- }
-
- @Test
- public void testTableOf() throws Exception {
- Pair<String, String> j = Pair.of("a", "b");
- Pair<Text, Text> w = Pair.of(new Text("a"), new Text("b"));
- WritableTableType<String, String> wtt = Writables.tableOf(Writables.strings(), Writables.strings());
- testInputOutputFn(wtt, j, w);
- }
-
- @Test
- public void testRegister() throws Exception {
- WritableType<TestWritable, TestWritable> wt = Writables.writables(TestWritable.class);
- Writables.register(TestWritable.class, wt);
- assertSame(Writables.records(TestWritable.class), wt);
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- protected static void testInputOutputFn(PType ptype, Object java, Object writable) {
- ptype.getInputMapFn().initialize();
- ptype.getOutputMapFn().initialize();
- assertEquals(java, ptype.getInputMapFn().map(writable));
- assertEquals(writable, ptype.getOutputMapFn().map(java));
- }
-
- @Test
- public void testDeepCopy() {
- Text text = new Text("Test");
- Text copiedText = Writables.deepCopy(text, Text.class);
- assertEquals(text, copiedText);
- assertNotSame(text, copiedText);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/com/cloudera/crunch/util/DistCacheTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/cloudera/crunch/util/DistCacheTest.java b/src/test/java/com/cloudera/crunch/util/DistCacheTest.java
deleted file mode 100644
index 7f0368b..0000000
--- a/src/test/java/com/cloudera/crunch/util/DistCacheTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
- *
- * Cloudera, Inc. licenses this file to you under the Apache License,
- * Version 2.0 (the "License"). You may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied. See the License for
- * the specific language governing permissions and limitations under the
- * License.
- */
-package com.cloudera.crunch.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class DistCacheTest {
-
- // A temporary folder used to hold files created for the test.
- @Rule
- public TemporaryFolder testFolder = new TemporaryFolder();
-
- // A configuration and lists of paths to use in tests.
- private Configuration testConf;
- private String[] testFilePaths;
- private String[] testFileQualifiedPaths;
-
- /**
- * Setup resources for tests. These include:
- * 1. A Hadoop configuration.
- * 2. A directory of temporary files that includes 3 .jar files and 1 other file.
- * 3. Arrays containing the canonical paths and qualified paths to the test files.
- */
- @Before
- public void setup() throws IOException {
- // Create a configuration for tests.
- testConf = new Configuration();
-
- // Create the test files and add their paths to the list of test file paths.
- testFilePaths = new String[3];
- testFilePaths[0] = testFolder.newFile("jar1.jar").getCanonicalPath();
- testFilePaths[1] = testFolder.newFile("jar2.jar").getCanonicalPath();
- testFilePaths[2] = testFolder.newFile("jar3.jar").getCanonicalPath();
- testFolder.newFile("notJar.other");
-
- // Populate a list of qualified paths from the test file paths.
- testFileQualifiedPaths = new String[3];
- for (int i = 0; i < testFilePaths.length; i++) {
- testFileQualifiedPaths[i] = "file:" + testFilePaths[i];
- }
- }
-
- /**
- * Tests adding jars one-by-one to a job's configuration.
- *
- * @throws IOException If there is a problem adding the jars.
- */
- @Test
- public void testAddJar() throws IOException {
- // Add each valid jar path to the distributed cache configuration, and verify each was
- // added correctly in turn.
- for (int i = 0; i < testFilePaths.length; i++) {
- DistCache.addJarToDistributedCache(testConf, testFilePaths[i]);
- assertEquals("tmpjars configuration var does not contain expected value.",
- StringUtils.join(testFileQualifiedPaths, ",", 0, i + 1), testConf.get("tmpjars"));
- }
- }
-
- /**
- * Tests that attempting to add the path to a jar that does not exist to the configuration
- * throws an exception.
- *
- * @throws IOException If the added jar path does not exist. This exception is expected.
- */
- @Test(expected = IOException.class)
- public void testAddJarThatDoesntExist() throws IOException {
- DistCache.addJarToDistributedCache(testConf, "/garbage/doesntexist.jar");
- }
-
- /**
- * Tests that adding a directory of jars to the configuration works as expected. .jar files
- * under the added directory should be added to the configuration,
- * and all other files should be skipped.
- *
- * @throws IOException If there is a problem adding the jar directory to the configuration.
- */
- @Test
- public void testAddJarDirectory() throws IOException {
- DistCache.addJarDirToDistributedCache(testConf, testFolder.getRoot().getCanonicalPath());
- // Throw the added jar paths in a set to detect duplicates.
- String[] splitJarPaths = StringUtils.split(testConf.get("tmpjars"), ",");
- Set<String> addedJarPaths = new HashSet<String>();
- for (String path: splitJarPaths) {
- addedJarPaths.add(path);
- }
- assertEquals("Incorrect number of jar paths added.", testFilePaths.length,
- addedJarPaths.size());
-
- // Ensure all expected paths were added.
- for (int i = 0; i < testFileQualifiedPaths.length; i++) {
- assertTrue("Expected jar path missing from jar paths added to tmpjars: " +
- testFileQualifiedPaths[i], addedJarPaths.contains(testFileQualifiedPaths[i]));
- }
- }
-
- /**
- * Tests that adding a jar directory that does not exist to the configuration throws an
- * exception.
- *
- * @throws IOException If the added jar directory does not exist. This exception is expected.
- */
- @Test(expected = IOException.class)
- public void testAddJarDirectoryThatDoesntExist() throws IOException {
- DistCache.addJarDirToDistributedCache(testConf, "/garbage/doesntexist");
- }
-
- /**
- * Tests that adding a jar directory that is not a directory to the configuration throws an
- * exception.
- *
- * @throws IOException If the added jar directory is not a directory. This exception is expected.
- */
- @Test(expected = IOException.class)
- public void testAddJarDirectoryNotDirectory() throws IOException {
- DistCache.addJarDirToDistributedCache(testConf, testFilePaths[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/org/apache/crunch/CollectionsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/crunch/CollectionsTest.java b/src/test/java/org/apache/crunch/CollectionsTest.java
new file mode 100644
index 0000000..896014a
--- /dev/null
+++ b/src/test/java/org/apache/crunch/CollectionsTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.junit.Test;
+
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.types.avro.AvroTypeFamily;
+import org.apache.crunch.types.writable.WritableTypeFamily;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+public class CollectionsTest {
+
+ public static class AggregateStringListFn implements CombineFn.Aggregator<Collection<String>> {
+ private final Collection<String> rtn = Lists.newArrayList();
+
+ @Override
+ public void reset() {
+ rtn.clear();
+ }
+
+ @Override
+ public void update(Collection<String> values) {
+ rtn.addAll(values);
+ }
+
+ @Override
+ public Iterable<Collection<String>> results() {
+ return ImmutableList.of(rtn);
+ }
+ }
+
+ public static PTable<String, Collection<String>> listOfCharcters(PCollection<String> lines, PTypeFamily typeFamily) {
+
+ return lines.parallelDo(new DoFn<String, Pair<String, Collection<String>>>() {
+ @Override
+ public void process(String line, Emitter<Pair<String, Collection<String>>> emitter) {
+ for (String word : line.split("\\s+")) {
+ Collection<String> characters = Lists.newArrayList();
+ for(char c : word.toCharArray()) {
+ characters.add(String.valueOf(c));
+ }
+ emitter.emit(Pair.of(word, characters));
+ }
+ }
+ }, typeFamily.tableOf(typeFamily.strings(), typeFamily.collections(typeFamily.strings())))
+ .groupByKey()
+ .combineValues(CombineFn.<String, Collection<String>>aggregator(new AggregateStringListFn()));
+ }
+
+ @Test
+ public void testWritables() throws IOException {
+ run(new MRPipeline(CollectionsTest.class), WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testAvro() throws IOException {
+ run(new MRPipeline(CollectionsTest.class), AvroTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testInMemoryWritables() throws IOException {
+ run(MemPipeline.getInstance(), WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testInMemoryAvro() throws IOException {
+ run(MemPipeline.getInstance(), AvroTypeFamily.getInstance());
+ }
+
+ public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
+ String shakesInputPath = FileHelper.createTempCopyOf("shakes.txt");
+
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+ Iterable<Pair<String, Collection<String>>> lines = listOfCharcters(shakespeare, typeFamily).materialize();
+
+ boolean passed = false;
+ for (Pair<String, Collection<String>> line : lines) {
+ if(line.first().startsWith("yellow")) {
+ passed = true;
+ break;
+ }
+ }
+ pipeline.done();
+ assertTrue(passed);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/org/apache/crunch/CombineFnTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/crunch/CombineFnTest.java b/src/test/java/org/apache/crunch/CombineFnTest.java
new file mode 100644
index 0000000..e015498
--- /dev/null
+++ b/src/test/java/org/apache/crunch/CombineFnTest.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.apache.crunch.CombineFn.MAX_BIGINTS;
+import static org.apache.crunch.CombineFn.MAX_DOUBLES;
+import static org.apache.crunch.CombineFn.MAX_FLOATS;
+import static org.apache.crunch.CombineFn.MAX_INTS;
+import static org.apache.crunch.CombineFn.MAX_LONGS;
+import static org.apache.crunch.CombineFn.MIN_BIGINTS;
+import static org.apache.crunch.CombineFn.MIN_DOUBLES;
+import static org.apache.crunch.CombineFn.MIN_FLOATS;
+import static org.apache.crunch.CombineFn.MIN_INTS;
+import static org.apache.crunch.CombineFn.MIN_LONGS;
+import static org.apache.crunch.CombineFn.SUM_BIGINTS;
+import static org.apache.crunch.CombineFn.SUM_DOUBLES;
+import static org.apache.crunch.CombineFn.SUM_FLOATS;
+import static org.apache.crunch.CombineFn.SUM_INTS;
+import static org.apache.crunch.CombineFn.SUM_LONGS;
+import static org.junit.Assert.assertEquals;
+
+import java.math.BigInteger;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.crunch.CombineFn.Aggregator;
+import org.apache.crunch.CombineFn.AggregatorFactory;
+import org.apache.crunch.CombineFn.FirstNAggregator;
+import org.apache.crunch.CombineFn.LastNAggregator;
+import org.apache.crunch.CombineFn.MaxNAggregator;
+import org.apache.crunch.CombineFn.MinNAggregator;
+import org.apache.crunch.CombineFn.PairAggregator;
+import org.apache.crunch.CombineFn.QuadAggregator;
+import org.apache.crunch.CombineFn.TripAggregator;
+import org.apache.crunch.CombineFn.TupleNAggregator;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+public class CombineFnTest {
+
+ private <T> Iterable<T> applyAggregator(AggregatorFactory<T> a, Iterable<T> values) {
+ return applyAggregator(a.create(), values);
+ }
+
+ private <T> Iterable<T> applyAggregator(Aggregator<T> a, Iterable<T> values) {
+ a.reset();
+ for (T value : values) {
+ a.update(value);
+ }
+ return a.results();
+ }
+
+ @Test
+ public void testSums() {
+ assertEquals(ImmutableList.of(1775L),
+ applyAggregator(SUM_LONGS, ImmutableList.of(29L, 17L, 1729L)));
+
+ assertEquals(ImmutableList.of(1765L),
+ applyAggregator(SUM_LONGS, ImmutableList.of(29L, 7L, 1729L)));
+
+ assertEquals(ImmutableList.of(1775),
+ applyAggregator(SUM_INTS, ImmutableList.of(29, 17, 1729)));
+
+ assertEquals(ImmutableList.of(1775.0f),
+ applyAggregator(SUM_FLOATS, ImmutableList.of(29f, 17f, 1729f)));
+
+ assertEquals(ImmutableList.of(1775.0),
+ applyAggregator(SUM_DOUBLES, ImmutableList.of(29.0, 17.0, 1729.0)));
+
+ assertEquals(ImmutableList.of(new BigInteger("1775")),
+ applyAggregator(SUM_BIGINTS,
+ ImmutableList.of(new BigInteger("29"), new BigInteger("17"), new BigInteger("1729"))));
+ }
+
+ @Test
+ public void testMax() {
+ assertEquals(ImmutableList.of(1729L),
+ applyAggregator(MAX_LONGS, ImmutableList.of(29L, 17L, 1729L)));
+
+ assertEquals(ImmutableList.of(1729),
+ applyAggregator(MAX_INTS, ImmutableList.of(29, 17, 1729)));
+
+ assertEquals(ImmutableList.of(1729.0f),
+ applyAggregator(MAX_FLOATS, ImmutableList.of(29f, 17f, 1729f)));
+
+ assertEquals(ImmutableList.of(1729.0),
+ applyAggregator(MAX_DOUBLES, ImmutableList.of(29.0, 17.0, 1729.0)));
+
+ assertEquals(ImmutableList.of(1745.0f),
+ applyAggregator(MAX_FLOATS, ImmutableList.of(29f, 1745f, 17f, 1729f)));
+
+ assertEquals(ImmutableList.of(new BigInteger("1729")),
+ applyAggregator(MAX_BIGINTS,
+ ImmutableList.of(new BigInteger("29"), new BigInteger("17"), new BigInteger("1729"))));
+ }
+
+ @Test
+ public void testMin() {
+ assertEquals(ImmutableList.of(17L),
+ applyAggregator(MIN_LONGS, ImmutableList.of(29L, 17L, 1729L)));
+
+ assertEquals(ImmutableList.of(17),
+ applyAggregator(MIN_INTS, ImmutableList.of(29, 17, 1729)));
+
+ assertEquals(ImmutableList.of(17.0f),
+ applyAggregator(MIN_FLOATS, ImmutableList.of(29f, 17f, 1729f)));
+
+ assertEquals(ImmutableList.of(17.0),
+ applyAggregator(MIN_DOUBLES, ImmutableList.of(29.0, 17.0, 1729.0)));
+
+ assertEquals(ImmutableList.of(29),
+ applyAggregator(MIN_INTS, ImmutableList.of(29, 170, 1729)));
+
+ assertEquals(ImmutableList.of(new BigInteger("17")),
+ applyAggregator(MIN_BIGINTS,
+ ImmutableList.of(new BigInteger("29"), new BigInteger("17"), new BigInteger("1729"))));
+ }
+
+ @Test
+ public void testMaxN() {
+ assertEquals(ImmutableList.of(98, 1009), applyAggregator(new MaxNAggregator<Integer>(2),
+ ImmutableList.of(17, 34, 98, 29, 1009)));
+ }
+
+ @Test
+ public void testMinN() {
+ assertEquals(ImmutableList.of(17, 29), applyAggregator(new MinNAggregator<Integer>(2),
+ ImmutableList.of(17, 34, 98, 29, 1009)));
+ }
+
+ @Test
+ public void testFirstN() {
+ assertEquals(ImmutableList.of(17, 34), applyAggregator(new FirstNAggregator<Integer>(2),
+ ImmutableList.of(17, 34, 98, 29, 1009)));
+ }
+
+ @Test
+ public void testLastN() {
+ assertEquals(ImmutableList.of(29, 1009), applyAggregator(new LastNAggregator<Integer>(2),
+ ImmutableList.of(17, 34, 98, 29, 1009)));
+ }
+
+ @Test
+ public void testPairs() {
+ List<Pair<Long, Double>> input = ImmutableList.of(Pair.of(1720L, 17.29), Pair.of(9L, -3.14));
+ Aggregator<Pair<Long, Double>> a = new PairAggregator<Long, Double>(
+ SUM_LONGS.create(), MIN_DOUBLES.create());
+ assertEquals(Pair.of(1729L, -3.14), Iterables.getOnlyElement(applyAggregator(a, input)));
+ }
+
+ @Test
+ public void testPairsTwoLongs() {
+ List<Pair<Long, Long>> input = ImmutableList.of(Pair.of(1720L, 1L), Pair.of(9L, 19L));
+ Aggregator<Pair<Long, Long>> a = new PairAggregator<Long, Long>(
+ SUM_LONGS.create(), SUM_LONGS.create());
+ assertEquals(Pair.of(1729L, 20L), Iterables.getOnlyElement(applyAggregator(a, input)));
+ }
+
+ @Test
+ public void testTrips() {
+ List<Tuple3<Float, Double, Double>> input = ImmutableList.of(
+ Tuple3.of(17.29f, 12.2, 0.1), Tuple3.of(3.0f, 1.2, 3.14), Tuple3.of(-1.0f, 14.5, -0.98));
+ Aggregator<Tuple3<Float, Double, Double>> a = new TripAggregator<Float, Double, Double>(
+ MAX_FLOATS.create(), MAX_DOUBLES.create(), MIN_DOUBLES.create());
+ assertEquals(Tuple3.of(17.29f, 14.5, -0.98),
+ Iterables.getOnlyElement(applyAggregator(a, input)));
+ }
+
+ @Test
+ public void testQuads() {
+ List<Tuple4<Float, Double, Double, Integer>> input = ImmutableList.of(
+ Tuple4.of(17.29f, 12.2, 0.1, 1), Tuple4.of(3.0f, 1.2, 3.14, 2),
+ Tuple4.of(-1.0f, 14.5, -0.98, 3));
+ Aggregator<Tuple4<Float, Double, Double, Integer>> a =
+ new QuadAggregator<Float, Double, Double, Integer>(MAX_FLOATS.create(),
+ MAX_DOUBLES.create(), MIN_DOUBLES.create(), SUM_INTS.create());
+ assertEquals(Tuple4.of(17.29f, 14.5, -0.98, 6),
+ Iterables.getOnlyElement(applyAggregator(a, input)));
+ }
+
+ @Test
+ public void testTupleN() {
+ List<TupleN> input = ImmutableList.of(new TupleN(1, 3.0, 1, 2.0, 4L),
+ new TupleN(4, 17.0, 1, 9.7, 12L));
+ Aggregator<TupleN> a = new TupleNAggregator(MIN_INTS.create(), SUM_DOUBLES.create(),
+ MAX_INTS.create(), MIN_DOUBLES.create(), MAX_LONGS.create());
+ assertEquals(new TupleN(1, 20.0, 1, 2.0, 12L),
+ Iterables.getOnlyElement(applyAggregator(a, input)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/org/apache/crunch/FilterFnTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/crunch/FilterFnTest.java b/src/test/java/org/apache/crunch/FilterFnTest.java
new file mode 100644
index 0000000..a173bc5
--- /dev/null
+++ b/src/test/java/org/apache/crunch/FilterFnTest.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class FilterFnTest {
+
+ private static final FilterFn<String> TRUE = new FilterFn<String>() {
+ @Override
+ public boolean accept(String input) {
+ return true;
+ }
+ };
+
+ private static final FilterFn<String> FALSE = new FilterFn<String>() {
+ @Override
+ public boolean accept(String input) {
+ return false;
+ }
+ };
+
+ @Test
+ public void testAnd() {
+ assertTrue(FilterFn.and(TRUE).accept("foo"));
+ assertTrue(FilterFn.and(TRUE, TRUE).accept("foo"));
+ assertFalse(FilterFn.and(TRUE, FALSE).accept("foo"));
+ assertFalse(FilterFn.and(FALSE, FALSE, FALSE).accept("foo"));
+ }
+
+ @Test
+ public void testOr() {
+ assertFalse(FilterFn.or(FALSE).accept("foo"));
+ assertTrue(FilterFn.or(FALSE, TRUE).accept("foo"));
+ assertTrue(FilterFn.or(TRUE, FALSE, TRUE).accept("foo"));
+ assertFalse(FilterFn.or(FALSE, FALSE, FALSE).accept("foo"));
+ }
+
+ @Test
+ public void testNot() {
+ assertFalse(FilterFn.not(TRUE).accept("foo"));
+ assertTrue(FilterFn.not(FALSE).accept("foo"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/org/apache/crunch/MapsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/crunch/MapsTest.java b/src/test/java/org/apache/crunch/MapsTest.java
new file mode 100644
index 0000000..5c9eb57
--- /dev/null
+++ b/src/test/java/org/apache/crunch/MapsTest.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.types.avro.AvroTypeFamily;
+import org.apache.crunch.types.writable.WritableTypeFamily;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+public class MapsTest {
+
+ @Test
+ public void testWritables() throws Exception {
+ run(WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testAvros() throws Exception {
+ run(AvroTypeFamily.getInstance());
+ }
+
+ public static void run(PTypeFamily typeFamily) throws Exception {
+ Pipeline pipeline = new MRPipeline(MapsTest.class);
+ String shakesInputPath = FileHelper.createTempCopyOf("shakes.txt");
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+ Iterable<Pair<String, Map<String, Long>>> output = shakespeare.parallelDo(
+ new DoFn<String, Pair<String, Map<String, Long>>>() {
+ @Override
+ public void process(String input,
+ Emitter<Pair<String, Map<String, Long>>> emitter) {
+ String last = null;
+ for (String word : input.toLowerCase().split("\\W+")) {
+ if (!word.isEmpty()) {
+ String firstChar = word.substring(0, 1);
+ if (last != null) {
+ Map<String, Long> cc = ImmutableMap.of(firstChar, 1L);
+ emitter.emit(Pair.of(last, cc));
+ }
+ last = firstChar;
+ }
+ }
+ }
+ }, typeFamily.tableOf(typeFamily.strings(), typeFamily.maps(typeFamily.longs())))
+ .groupByKey()
+ .combineValues(new CombineFn<String, Map<String, Long>>() {
+ @Override
+ public void process(Pair<String, Iterable<Map<String, Long>>> input,
+ Emitter<Pair<String, Map<String, Long>>> emitter) {
+ Map<String, Long> agg = Maps.newHashMap();
+ for (Map<String, Long> in : input.second()) {
+ for (Map.Entry<String, Long> e : in.entrySet()) {
+ if (!agg.containsKey(e.getKey())) {
+ agg.put(e.getKey(), e.getValue());
+ } else {
+ agg.put(e.getKey(), e.getValue() + agg.get(e.getKey()));
+ }
+ }
+ }
+ emitter.emit(Pair.of(input.first(), agg));
+ }
+ }).materialize();
+ boolean passed = false;
+ for (Pair<String, Map<String, Long>> v : output) {
+ if (v.first() == "k" && v.second().get("n") == 8L) {
+ passed = true;
+ break;
+ }
+ }
+ pipeline.done();
+ }
+}