You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2013/04/23 22:41:45 UTC
[43/43] git commit: CRUNCH-196: crunch -> crunch-core rename to fix
build issues
CRUNCH-196: crunch -> crunch-core rename to fix build issues
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/890e0086
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/890e0086
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/890e0086
Branch: refs/heads/master
Commit: 890e0086a12df5006a23cfdd86f3703f929cb147
Parents: cbc7c2f
Author: Josh Wills <jw...@cloudera.com>
Authored: Tue Apr 23 13:37:16 2013 -0700
Committer: Josh Wills <jw...@cloudera.com>
Committed: Tue Apr 23 13:39:00 2013 -0700
----------------------------------------------------------------------
crunch-contrib/pom.xml | 2 +-
crunch-core/pom.xml | 182 +
.../it/java/org/apache/crunch/CancelJobsIT.java | 84 +
.../src/it/java/org/apache/crunch/CleanTextIT.java | 82 +
.../org/apache/crunch/CollectionPObjectIT.java | 98 +
.../it/java/org/apache/crunch/CollectionsIT.java | 117 +
.../org/apache/crunch/CollectionsLengthIT.java | 70 +
.../org/apache/crunch/DeepCopyCustomTuplesIT.java | 79 +
.../src/it/java/org/apache/crunch/EnumPairIT.java | 59 +
.../org/apache/crunch/FirstElementPObjectIT.java | 61 +
.../apache/crunch/IterableReuseProtectionIT.java | 89 +
.../it/java/org/apache/crunch/MRPipelineIT.java | 78 +
.../it/java/org/apache/crunch/MapPObjectIT.java | 101 +
.../src/it/java/org/apache/crunch/MapsIT.java | 101 +
.../it/java/org/apache/crunch/MaterializeIT.java | 139 +
.../java/org/apache/crunch/MaterializeToMapIT.java | 81 +
.../java/org/apache/crunch/MultipleOutputIT.java | 175 +
.../org/apache/crunch/PCollectionGetSizeIT.java | 151 +
.../src/it/java/org/apache/crunch/PObjectsIT.java | 99 +
.../java/org/apache/crunch/PTableKeyValueIT.java | 103 +
.../src/it/java/org/apache/crunch/PageRankIT.java | 168 +
.../org/apache/crunch/StageResultsCountersIT.java | 135 +
.../it/java/org/apache/crunch/TermFrequencyIT.java | 135 +
.../src/it/java/org/apache/crunch/TextPairIT.java | 72 +
.../src/it/java/org/apache/crunch/TfIdfIT.java | 224 +
.../org/apache/crunch/TupleNClassCastBugIT.java | 95 +
.../org/apache/crunch/UnionFromSameSourceIT.java | 132 +
.../src/it/java/org/apache/crunch/UnionIT.java | 136 +
.../it/java/org/apache/crunch/UnionResultsIT.java | 80 +
.../src/it/java/org/apache/crunch/WordCountIT.java | 171 +
.../java/org/apache/crunch/fn/AggregatorsIT.java | 83 +
.../crunch/impl/mem/MemPipelineFileWritingIT.java | 58 +
.../crunch/impl/mr/collect/UnionCollectionIT.java | 154 +
.../apache/crunch/io/CompositePathIterableIT.java | 84 +
.../it/java/org/apache/crunch/io/NLineInputIT.java | 72 +
.../java/org/apache/crunch/io/TextFileTableIT.java | 56 +
.../crunch/io/avro/AvroFileSourceTargetIT.java | 140 +
.../org/apache/crunch/io/avro/AvroPipelineIT.java | 95 +
.../org/apache/crunch/io/avro/AvroReflectIT.java | 109 +
.../org/apache/crunch/io/avro/AvroWritableIT.java | 89 +
.../it/java/org/apache/crunch/lib/AggregateIT.java | 231 +
.../java/org/apache/crunch/lib/AvroTypeSortIT.java | 145 +
.../it/java/org/apache/crunch/lib/CogroupIT.java | 112 +
.../org/apache/crunch/lib/SecondarySortIT.java | 65 +
.../src/it/java/org/apache/crunch/lib/SetIT.java | 114 +
.../java/org/apache/crunch/lib/SortByValueIT.java | 84 +
.../src/it/java/org/apache/crunch/lib/SortIT.java | 327 +
.../apache/crunch/lib/SpecificAvroGroupByIT.java | 119 +
.../apache/crunch/lib/join/FullOuterJoinIT.java | 51 +
.../org/apache/crunch/lib/join/InnerJoinIT.java | 51 +
.../org/apache/crunch/lib/join/JoinTester.java | 108 +
.../apache/crunch/lib/join/LeftOuterJoinIT.java | 51 +
.../org/apache/crunch/lib/join/MapsideJoinIT.java | 158 +
.../crunch/lib/join/MultiAvroSchemaJoinIT.java | 121 +
.../apache/crunch/lib/join/RightOuterJoinIT.java | 51 +
.../org/apache/crunch/test/TemporaryPaths.java | 40 +
.../src/it/java/org/apache/crunch/test/Tests.java | 124 +
crunch-core/src/it/resources/customers.txt | 4 +
crunch-core/src/it/resources/docs.txt | 6 +
crunch-core/src/it/resources/letters.txt | 2 +
crunch-core/src/it/resources/log4j.properties | 29 +
crunch-core/src/it/resources/maugham.txt |29112 +++++++++++++++
crunch-core/src/it/resources/orders.txt | 4 +
.../org/apache/crunch/UnionITData/src1.txt | 5 +
.../org/apache/crunch/UnionITData/src2.txt | 3 +
.../apache/crunch/fn/AggregatorsITData/ints.txt | 5 +
.../org/apache/crunch/lib/CogroupITData/src1.txt | 4 +
.../org/apache/crunch/lib/CogroupITData/src2.txt | 4 +
.../src/it/resources/secondary_sort_input.txt | 7 +
crunch-core/src/it/resources/set1.txt | 4 +
crunch-core/src/it/resources/set2.txt | 3 +
crunch-core/src/it/resources/shakes.txt | 3667 ++
crunch-core/src/it/resources/sort_by_value.txt | 5 +
crunch-core/src/it/resources/urls.txt | 11 +
.../main/java/org/apache/crunch/Aggregator.java | 86 +
.../src/main/java/org/apache/crunch/CombineFn.java | 1211 +
.../org/apache/crunch/CrunchRuntimeException.java | 54 +
.../src/main/java/org/apache/crunch/DoFn.java | 162 +
.../src/main/java/org/apache/crunch/Emitter.java | 37 +
.../src/main/java/org/apache/crunch/FilterFn.java | 244 +
.../java/org/apache/crunch/GroupingOptions.java | 167 +
.../src/main/java/org/apache/crunch/MapFn.java | 41 +
.../main/java/org/apache/crunch/PCollection.java | 245 +
.../main/java/org/apache/crunch/PGroupedTable.java | 53 +
.../src/main/java/org/apache/crunch/PObject.java | 36 +
.../src/main/java/org/apache/crunch/PTable.java | 181 +
.../src/main/java/org/apache/crunch/Pair.java | 105 +
.../java/org/apache/crunch/ParallelDoOptions.java | 62 +
.../src/main/java/org/apache/crunch/Pipeline.java | 138 +
.../java/org/apache/crunch/PipelineExecution.java | 54 +
.../java/org/apache/crunch/PipelineResult.java | 76 +
.../src/main/java/org/apache/crunch/Source.java | 52 +
.../main/java/org/apache/crunch/SourceTarget.java | 26 +
.../main/java/org/apache/crunch/TableSource.java | 28 +
.../java/org/apache/crunch/TableSourceTarget.java | 25 +
.../src/main/java/org/apache/crunch/Target.java | 83 +
.../src/main/java/org/apache/crunch/Tuple.java | 36 +
.../src/main/java/org/apache/crunch/Tuple3.java | 96 +
.../src/main/java/org/apache/crunch/Tuple4.java | 105 +
.../src/main/java/org/apache/crunch/TupleN.java | 73 +
.../java/org/apache/crunch/fn/Aggregators.java | 1111 +
.../java/org/apache/crunch/fn/CompositeMapFn.java | 71 +
.../java/org/apache/crunch/fn/ExtractKeyFn.java | 50 +
.../main/java/org/apache/crunch/fn/FilterFns.java | 112 +
.../main/java/org/apache/crunch/fn/IdentityFn.java | 39 +
.../main/java/org/apache/crunch/fn/MapKeysFn.java | 32 +
.../java/org/apache/crunch/fn/MapValuesFn.java | 32 +
.../main/java/org/apache/crunch/fn/PairMapFn.java | 65 +
.../java/org/apache/crunch/fn/package-info.java | 22 +
.../mapreduce/TaskAttemptContextFactory.java | 70 +
.../lib/jobcontrol/CrunchControlledJob.java | 325 +
.../mapreduce/lib/jobcontrol/CrunchJobControl.java | 211 +
.../org/apache/crunch/impl/SingleUseIterable.java | 49 +
.../org/apache/crunch/impl/mem/MemPipeline.java | 275 +
.../crunch/impl/mem/collect/MemCollection.java | 295 +
.../crunch/impl/mem/collect/MemGroupedTable.java | 113 +
.../apache/crunch/impl/mem/collect/MemTable.java | 177 +
.../apache/crunch/impl/mem/collect/Shuffler.java | 149 +
.../crunch/impl/mem/emit/InMemoryEmitter.java | 57 +
.../org/apache/crunch/impl/mem/package-info.java | 22 +
.../java/org/apache/crunch/impl/mr/MRPipeline.java | 396 +
.../crunch/impl/mr/collect/DoCollectionImpl.java | 74 +
.../apache/crunch/impl/mr/collect/DoTableImpl.java | 84 +
.../crunch/impl/mr/collect/InputCollection.java | 85 +
.../apache/crunch/impl/mr/collect/InputTable.java | 86 +
.../crunch/impl/mr/collect/PCollectionImpl.java | 295 +
.../crunch/impl/mr/collect/PGroupedTableImpl.java | 144 +
.../apache/crunch/impl/mr/collect/PTableBase.java | 169 +
.../crunch/impl/mr/collect/UnionCollection.java | 80 +
.../apache/crunch/impl/mr/collect/UnionTable.java | 92 +
.../crunch/impl/mr/emit/IntermediateEmitter.java | 64 +
.../crunch/impl/mr/emit/MultipleOutputEmitter.java | 56 +
.../apache/crunch/impl/mr/emit/OutputEmitter.java | 52 +
.../impl/mr/exec/CappedExponentialCounter.java | 40 +
.../apache/crunch/impl/mr/exec/CrunchJobHooks.java | 153 +
.../org/apache/crunch/impl/mr/exec/MRExecutor.java | 198 +
.../org/apache/crunch/impl/mr/package-info.java | 22 +
.../org/apache/crunch/impl/mr/plan/DoNode.java | 163 +
.../apache/crunch/impl/mr/plan/DotfileWriter.java | 238 +
.../java/org/apache/crunch/impl/mr/plan/Edge.java | 125 +
.../java/org/apache/crunch/impl/mr/plan/Graph.java | 133 +
.../apache/crunch/impl/mr/plan/GraphBuilder.java | 92 +
.../apache/crunch/impl/mr/plan/JobNameBuilder.java | 79 +
.../apache/crunch/impl/mr/plan/JobPrototype.java | 245 +
.../crunch/impl/mr/plan/MSCROutputHandler.java | 77 +
.../apache/crunch/impl/mr/plan/MSCRPlanner.java | 378 +
.../org/apache/crunch/impl/mr/plan/NodePath.java | 124 +
.../crunch/impl/mr/plan/PlanningParameters.java | 38 +
.../org/apache/crunch/impl/mr/plan/Vertex.java | 126 +
.../apache/crunch/impl/mr/run/CrunchCombiner.java | 27 +
.../crunch/impl/mr/run/CrunchInputFormat.java | 78 +
.../crunch/impl/mr/run/CrunchInputSplit.java | 116 +
.../apache/crunch/impl/mr/run/CrunchMapper.java | 73 +
.../crunch/impl/mr/run/CrunchRecordReader.java | 75 +
.../apache/crunch/impl/mr/run/CrunchReducer.java | 73 +
.../crunch/impl/mr/run/CrunchTaskContext.java | 86 +
.../org/apache/crunch/impl/mr/run/NodeContext.java | 35 +
.../java/org/apache/crunch/impl/mr/run/RTNode.java | 124 +
.../crunch/impl/mr/run/RuntimeParameters.java | 38 +
.../src/main/java/org/apache/crunch/io/At.java | 257 +
.../apache/crunch/io/CompositePathIterable.java | 102 +
.../java/org/apache/crunch/io/CrunchInputs.java | 71 +
.../java/org/apache/crunch/io/CrunchOutputs.java | 184 +
.../org/apache/crunch/io/FileNamingScheme.java | 58 +
.../org/apache/crunch/io/FileReaderFactory.java | 27 +
.../java/org/apache/crunch/io/FormatBundle.java | 121 +
.../src/main/java/org/apache/crunch/io/From.java | 324 +
.../java/org/apache/crunch/io/MapReduceTarget.java | 27 +
.../java/org/apache/crunch/io/OutputHandler.java | 25 +
.../main/java/org/apache/crunch/io/PathTarget.java | 36 +
.../java/org/apache/crunch/io/PathTargetImpl.java | 64 +
.../java/org/apache/crunch/io/ReadableSource.java | 41 +
.../org/apache/crunch/io/ReadableSourceTarget.java | 30 +
.../crunch/io/SequentialFileNamingScheme.java | 51 +
.../org/apache/crunch/io/SourceTargetHelper.java | 48 +
.../src/main/java/org/apache/crunch/io/To.java | 153 +
.../crunch/io/avro/AvroFileReaderFactory.java | 96 +
.../org/apache/crunch/io/avro/AvroFileSource.java | 58 +
.../crunch/io/avro/AvroFileSourceTarget.java | 39 +
.../org/apache/crunch/io/avro/AvroFileTarget.java | 91 +
.../apache/crunch/io/impl/AutoClosingIterator.java | 62 +
.../org/apache/crunch/io/impl/FileSourceImpl.java | 104 +
.../apache/crunch/io/impl/FileTableSourceImpl.java | 41 +
.../org/apache/crunch/io/impl/FileTargetImpl.java | 162 +
.../io/impl/ReadableSourcePathTargetImpl.java | 39 +
.../crunch/io/impl/ReadableSourceTargetImpl.java | 37 +
.../crunch/io/impl/SourcePathTargetImpl.java | 50 +
.../apache/crunch/io/impl/SourceTargetImpl.java | 89 +
.../crunch/io/impl/TableSourcePathTargetImpl.java | 41 +
.../crunch/io/impl/TableSourceTargetImpl.java | 35 +
.../java/org/apache/crunch/io/package-info.java | 22 +
.../org/apache/crunch/io/seq/SeqFileHelper.java | 35 +
.../apache/crunch/io/seq/SeqFileReaderFactory.java | 112 +
.../org/apache/crunch/io/seq/SeqFileSource.java | 47 +
.../apache/crunch/io/seq/SeqFileSourceTarget.java | 44 +
.../apache/crunch/io/seq/SeqFileTableSource.java | 57 +
.../crunch/io/seq/SeqFileTableSourceTarget.java | 54 +
.../org/apache/crunch/io/seq/SeqFileTarget.java | 55 +
.../crunch/io/text/BZip2TextInputFormat.java | 235 +
.../apache/crunch/io/text/CBZip2InputStream.java | 980 +
.../java/org/apache/crunch/io/text/LineParser.java | 125 +
.../org/apache/crunch/io/text/NLineFileSource.java | 77 +
.../crunch/io/text/TextFileReaderFactory.java | 83 +
.../org/apache/crunch/io/text/TextFileSource.java | 73 +
.../crunch/io/text/TextFileSourceTarget.java | 44 +
.../apache/crunch/io/text/TextFileTableSource.java | 81 +
.../crunch/io/text/TextFileTableSourceTarget.java | 63 +
.../org/apache/crunch/io/text/TextFileTarget.java | 109 +
.../main/java/org/apache/crunch/lib/Aggregate.java | 272 +
.../main/java/org/apache/crunch/lib/Cartesian.java | 216 +
.../main/java/org/apache/crunch/lib/Cogroup.java | 106 +
.../main/java/org/apache/crunch/lib/Distinct.java | 126 +
.../src/main/java/org/apache/crunch/lib/Join.java | 181 +
.../main/java/org/apache/crunch/lib/PTables.java | 117 +
.../main/java/org/apache/crunch/lib/Sample.java | 217 +
.../java/org/apache/crunch/lib/SampleUtils.java | 168 +
.../java/org/apache/crunch/lib/SecondarySort.java | 118 +
.../src/main/java/org/apache/crunch/lib/Set.java | 118 +
.../src/main/java/org/apache/crunch/lib/Sort.java | 294 +
.../apache/crunch/lib/join/FullOuterJoinFn.java | 102 +
.../org/apache/crunch/lib/join/InnerJoinFn.java | 78 +
.../java/org/apache/crunch/lib/join/JoinFn.java | 81 +
.../java/org/apache/crunch/lib/join/JoinUtils.java | 126 +
.../apache/crunch/lib/join/LeftOuterJoinFn.java | 98 +
.../org/apache/crunch/lib/join/MapsideJoin.java | 164 +
.../apache/crunch/lib/join/RightOuterJoinFn.java | 83 +
.../org/apache/crunch/lib/join/package-info.java | 22 +
.../java/org/apache/crunch/lib/package-info.java | 22 +
.../org/apache/crunch/lib/sort/Comparators.java | 187 +
.../java/org/apache/crunch/lib/sort/SortFns.java | 210 +
.../crunch/lib/sort/TotalOrderPartitioner.java | 145 +
.../crunch/materialize/MaterializableIterable.java | 81 +
.../crunch/materialize/MaterializableMap.java | 50 +
.../materialize/pobject/CollectionPObject.java | 55 +
.../materialize/pobject/FirstElementPObject.java | 50 +
.../crunch/materialize/pobject/MapPObject.java | 62 +
.../crunch/materialize/pobject/PObjectImpl.java | 85 +
.../main/java/org/apache/crunch/package-info.java | 25 +
.../apache/crunch/types/CollectionDeepCopier.java | 57 +
.../java/org/apache/crunch/types/Converter.java | 41 +
.../java/org/apache/crunch/types/DeepCopier.java | 60 +
.../org/apache/crunch/types/MapDeepCopier.java | 54 +
.../org/apache/crunch/types/PGroupedTableType.java | 141 +
.../java/org/apache/crunch/types/PTableType.java | 44 +
.../main/java/org/apache/crunch/types/PType.java | 86 +
.../java/org/apache/crunch/types/PTypeFamily.java | 77 +
.../java/org/apache/crunch/types/PTypeUtils.java | 66 +
.../main/java/org/apache/crunch/types/PTypes.java | 252 +
.../main/java/org/apache/crunch/types/Protos.java | 173 +
.../org/apache/crunch/types/TupleDeepCopier.java | 65 +
.../java/org/apache/crunch/types/TupleFactory.java | 134 +
.../apache/crunch/types/avro/AvroCapabilities.java | 106 +
.../apache/crunch/types/avro/AvroDeepCopier.java | 209 +
.../crunch/types/avro/AvroGroupedTableType.java | 114 +
.../apache/crunch/types/avro/AvroInputFormat.java | 41 +
.../apache/crunch/types/avro/AvroKeyConverter.java | 65 +
.../apache/crunch/types/avro/AvroOutputFormat.java | 87 +
.../crunch/types/avro/AvroPairConverter.java | 108 +
.../apache/crunch/types/avro/AvroRecordReader.java | 114 +
.../apache/crunch/types/avro/AvroTableType.java | 151 +
.../crunch/types/avro/AvroTextOutputFormat.java | 60 +
.../org/apache/crunch/types/avro/AvroType.java | 199 +
.../apache/crunch/types/avro/AvroTypeFamily.java | 164 +
.../crunch/types/avro/AvroUtf8InputFormat.java | 98 +
.../java/org/apache/crunch/types/avro/Avros.java | 709 +
.../crunch/types/avro/ReflectDataFactory.java | 41 +
.../crunch/types/avro/SafeAvroSerialization.java | 145 +
.../org/apache/crunch/types/avro/package-info.java | 22 +
.../java/org/apache/crunch/types/package-info.java | 22 +
.../types/writable/GenericArrayWritable.java | 135 +
.../crunch/types/writable/TextMapWritable.java | 88 +
.../crunch/types/writable/TupleWritable.java | 224 +
.../crunch/types/writable/WritableDeepCopier.java | 70 +
.../types/writable/WritableGroupedTableType.java | 85 +
.../types/writable/WritablePairConverter.java | 62 +
.../crunch/types/writable/WritableTableType.java | 130 +
.../apache/crunch/types/writable/WritableType.java | 133 +
.../crunch/types/writable/WritableTypeFamily.java | 147 +
.../types/writable/WritableValueConverter.java | 60 +
.../apache/crunch/types/writable/Writables.java | 588 +
.../apache/crunch/types/writable/package-info.java | 22 +
.../java/org/apache/crunch/util/CrunchTool.java | 118 +
.../java/org/apache/crunch/util/DistCache.java | 231 +
.../org/apache/crunch/util/PartitionUtils.java | 34 +
.../main/java/org/apache/crunch/util/Tuples.java | 150 +
.../java/org/apache/crunch/util/package-info.java | 22 +
crunch-core/src/main/resources/log4j.properties | 24 +
crunch-core/src/site/site.xml | 34 +
crunch-core/src/test/avro/employee.avsc | 26 +
crunch-core/src/test/avro/person.avsc | 26 +
.../src/test/java/org/apache/crunch/AndFnTest.java | 77 +
.../test/java/org/apache/crunch/CombineFnTest.java | 222 +
.../src/test/java/org/apache/crunch/NotFnTest.java | 72 +
.../src/test/java/org/apache/crunch/OrFnTest.java | 78 +
.../src/test/java/org/apache/crunch/PairTest.java | 66 +
.../src/test/java/org/apache/crunch/TupleTest.java | 139 +
.../test/java/org/apache/crunch/WriteModeTest.java | 103 +
.../java/org/apache/crunch/fn/AggregatorsTest.java | 239 +
.../org/apache/crunch/fn/ExtractKeyFnTest.java | 44 +
.../java/org/apache/crunch/fn/FilterFnTest.java | 85 +
.../java/org/apache/crunch/fn/MapKeysTest.java | 51 +
.../java/org/apache/crunch/fn/MapValuesTest.java | 50 +
.../java/org/apache/crunch/fn/PairMapTest.java | 52 +
.../org/apache/crunch/fn/StoreLastEmitter.java | 41 +
.../apache/crunch/impl/SingleUseIterableTest.java | 54 +
.../org/apache/crunch/impl/mr/MRPipelineTest.java | 86 +
.../impl/mr/collect/DoCollectionImplTest.java | 112 +
.../crunch/impl/mr/collect/DoTableImplTest.java | 86 +
.../impl/mr/emit/IntermediateEmitterTest.java | 83 +
.../impl/mr/exec/CappedExponentialCounterTest.java | 42 +
.../crunch/impl/mr/exec/CrunchJobHooksTest.java | 42 +
.../crunch/impl/mr/plan/DotfileWriterTest.java | 132 +
.../crunch/impl/mr/plan/JobNameBuilderTest.java | 41 +
.../crunch/io/SequentialFileNamingSchemeTest.java | 84 +
.../apache/crunch/io/SourceTargetHelperTest.java | 59 +
.../crunch/io/avro/AvroFileReaderFactoryTest.java | 184 +
.../apache/crunch/io/avro/AvroFileSourceTest.java | 91 +
.../lib/AvroIndexedRecordPartitionerTest.java | 98 +
.../java/org/apache/crunch/lib/CartesianTest.java | 77 +
.../java/org/apache/crunch/lib/DistinctTest.java | 52 +
.../java/org/apache/crunch/lib/SampleTest.java | 71 +
.../org/apache/crunch/lib/SecondarySortTest.java | 53 +
.../crunch/lib/TupleWritablePartitionerTest.java | 68 +
.../lib/join/BrokenLeftAndOuterJoinTest.java | 90 +
.../crunch/lib/join/FullOuterJoinFnTest.java | 48 +
.../apache/crunch/lib/join/InnerJoinFnTest.java | 42 +
.../org/apache/crunch/lib/join/JoinFnTestBase.java | 82 +
.../apache/crunch/lib/join/LeftOuterJoinTest.java | 46 +
.../crunch/lib/join/RightOuterJoinFnTest.java | 46 +
.../java/org/apache/crunch/test/CountersTest.java | 70 +
.../java/org/apache/crunch/test/StringWrapper.java | 102 +
.../crunch/types/CollectionDeepCopierTest.java | 61 +
.../org/apache/crunch/types/MapDeepCopierTest.java | 63 +
.../org/apache/crunch/types/PTypeUtilsTest.java | 89 +
.../java/org/apache/crunch/types/PTypesTest.java | 34 +
.../apache/crunch/types/TupleDeepCopierTest.java | 77 +
.../org/apache/crunch/types/TupleFactoryTest.java | 69 +
.../crunch/types/avro/AvroDeepCopierTest.java | 107 +
.../types/avro/AvroGroupedTableTypeTest.java | 60 +
.../crunch/types/avro/AvroTableTypeTest.java | 72 +
.../org/apache/crunch/types/avro/AvroTypeTest.java | 279 +
.../org/apache/crunch/types/avro/AvrosTest.java | 325 +
.../types/writable/GenericArrayWritableTest.java | 70 +
.../types/writable/WritableDeepCopierTest.java | 54 +
.../writable/WritableGroupedTableTypeTest.java | 56 +
.../types/writable/WritableTableTypeTest.java | 47 +
.../crunch/types/writable/WritableTypeTest.java | 97 +
.../crunch/types/writable/WritablesTest.java | 256 +
.../java/org/apache/crunch/util/DistCacheTest.java | 156 +
crunch-dist/pom.xml | 2 +-
crunch-examples/pom.xml | 2 +-
crunch-hbase/pom.xml | 2 +-
crunch-scrunch/pom.xml | 2 +-
crunch/pom.xml | 182 -
.../it/java/org/apache/crunch/CancelJobsIT.java | 84 -
.../src/it/java/org/apache/crunch/CleanTextIT.java | 82 -
.../org/apache/crunch/CollectionPObjectIT.java | 98 -
.../it/java/org/apache/crunch/CollectionsIT.java | 117 -
.../org/apache/crunch/CollectionsLengthIT.java | 70 -
.../org/apache/crunch/DeepCopyCustomTuplesIT.java | 79 -
.../src/it/java/org/apache/crunch/EnumPairIT.java | 59 -
.../org/apache/crunch/FirstElementPObjectIT.java | 61 -
.../apache/crunch/IterableReuseProtectionIT.java | 89 -
.../it/java/org/apache/crunch/MRPipelineIT.java | 78 -
.../it/java/org/apache/crunch/MapPObjectIT.java | 101 -
crunch/src/it/java/org/apache/crunch/MapsIT.java | 101 -
.../it/java/org/apache/crunch/MaterializeIT.java | 139 -
.../java/org/apache/crunch/MaterializeToMapIT.java | 81 -
.../java/org/apache/crunch/MultipleOutputIT.java | 175 -
.../org/apache/crunch/PCollectionGetSizeIT.java | 151 -
.../src/it/java/org/apache/crunch/PObjectsIT.java | 99 -
.../java/org/apache/crunch/PTableKeyValueIT.java | 103 -
.../src/it/java/org/apache/crunch/PageRankIT.java | 168 -
.../org/apache/crunch/StageResultsCountersIT.java | 135 -
.../it/java/org/apache/crunch/TermFrequencyIT.java | 135 -
.../src/it/java/org/apache/crunch/TextPairIT.java | 72 -
crunch/src/it/java/org/apache/crunch/TfIdfIT.java | 224 -
.../org/apache/crunch/TupleNClassCastBugIT.java | 95 -
.../org/apache/crunch/UnionFromSameSourceIT.java | 132 -
crunch/src/it/java/org/apache/crunch/UnionIT.java | 136 -
.../it/java/org/apache/crunch/UnionResultsIT.java | 80 -
.../src/it/java/org/apache/crunch/WordCountIT.java | 171 -
.../java/org/apache/crunch/fn/AggregatorsIT.java | 83 -
.../crunch/impl/mem/MemPipelineFileWritingIT.java | 58 -
.../crunch/impl/mr/collect/UnionCollectionIT.java | 154 -
.../apache/crunch/io/CompositePathIterableIT.java | 84 -
.../it/java/org/apache/crunch/io/NLineInputIT.java | 72 -
.../java/org/apache/crunch/io/TextFileTableIT.java | 56 -
.../crunch/io/avro/AvroFileSourceTargetIT.java | 140 -
.../org/apache/crunch/io/avro/AvroPipelineIT.java | 95 -
.../org/apache/crunch/io/avro/AvroReflectIT.java | 109 -
.../org/apache/crunch/io/avro/AvroWritableIT.java | 89 -
.../it/java/org/apache/crunch/lib/AggregateIT.java | 231 -
.../java/org/apache/crunch/lib/AvroTypeSortIT.java | 145 -
.../it/java/org/apache/crunch/lib/CogroupIT.java | 112 -
.../org/apache/crunch/lib/SecondarySortIT.java | 65 -
.../src/it/java/org/apache/crunch/lib/SetIT.java | 114 -
.../java/org/apache/crunch/lib/SortByValueIT.java | 84 -
.../src/it/java/org/apache/crunch/lib/SortIT.java | 327 -
.../apache/crunch/lib/SpecificAvroGroupByIT.java | 119 -
.../apache/crunch/lib/join/FullOuterJoinIT.java | 51 -
.../org/apache/crunch/lib/join/InnerJoinIT.java | 51 -
.../org/apache/crunch/lib/join/JoinTester.java | 108 -
.../apache/crunch/lib/join/LeftOuterJoinIT.java | 51 -
.../org/apache/crunch/lib/join/MapsideJoinIT.java | 158 -
.../crunch/lib/join/MultiAvroSchemaJoinIT.java | 121 -
.../apache/crunch/lib/join/RightOuterJoinIT.java | 51 -
.../org/apache/crunch/test/TemporaryPaths.java | 40 -
.../src/it/java/org/apache/crunch/test/Tests.java | 124 -
crunch/src/it/resources/customers.txt | 4 -
crunch/src/it/resources/docs.txt | 6 -
crunch/src/it/resources/letters.txt | 2 -
crunch/src/it/resources/log4j.properties | 29 -
crunch/src/it/resources/maugham.txt |29112 ---------------
crunch/src/it/resources/orders.txt | 4 -
.../org/apache/crunch/UnionITData/src1.txt | 5 -
.../org/apache/crunch/UnionITData/src2.txt | 3 -
.../apache/crunch/fn/AggregatorsITData/ints.txt | 5 -
.../org/apache/crunch/lib/CogroupITData/src1.txt | 4 -
.../org/apache/crunch/lib/CogroupITData/src2.txt | 4 -
crunch/src/it/resources/secondary_sort_input.txt | 7 -
crunch/src/it/resources/set1.txt | 4 -
crunch/src/it/resources/set2.txt | 3 -
crunch/src/it/resources/shakes.txt | 3667 --
crunch/src/it/resources/sort_by_value.txt | 5 -
crunch/src/it/resources/urls.txt | 11 -
.../main/java/org/apache/crunch/Aggregator.java | 86 -
.../src/main/java/org/apache/crunch/CombineFn.java | 1211 -
.../org/apache/crunch/CrunchRuntimeException.java | 54 -
crunch/src/main/java/org/apache/crunch/DoFn.java | 162 -
.../src/main/java/org/apache/crunch/Emitter.java | 37 -
.../src/main/java/org/apache/crunch/FilterFn.java | 244 -
.../java/org/apache/crunch/GroupingOptions.java | 167 -
crunch/src/main/java/org/apache/crunch/MapFn.java | 41 -
.../main/java/org/apache/crunch/PCollection.java | 245 -
.../main/java/org/apache/crunch/PGroupedTable.java | 53 -
.../src/main/java/org/apache/crunch/PObject.java | 36 -
crunch/src/main/java/org/apache/crunch/PTable.java | 181 -
crunch/src/main/java/org/apache/crunch/Pair.java | 105 -
.../java/org/apache/crunch/ParallelDoOptions.java | 62 -
.../src/main/java/org/apache/crunch/Pipeline.java | 138 -
.../java/org/apache/crunch/PipelineExecution.java | 54 -
.../java/org/apache/crunch/PipelineResult.java | 76 -
crunch/src/main/java/org/apache/crunch/Source.java | 52 -
.../main/java/org/apache/crunch/SourceTarget.java | 26 -
.../main/java/org/apache/crunch/TableSource.java | 28 -
.../java/org/apache/crunch/TableSourceTarget.java | 25 -
crunch/src/main/java/org/apache/crunch/Target.java | 83 -
crunch/src/main/java/org/apache/crunch/Tuple.java | 36 -
crunch/src/main/java/org/apache/crunch/Tuple3.java | 96 -
crunch/src/main/java/org/apache/crunch/Tuple4.java | 105 -
crunch/src/main/java/org/apache/crunch/TupleN.java | 73 -
.../java/org/apache/crunch/fn/Aggregators.java | 1111 -
.../java/org/apache/crunch/fn/CompositeMapFn.java | 71 -
.../java/org/apache/crunch/fn/ExtractKeyFn.java | 50 -
.../main/java/org/apache/crunch/fn/FilterFns.java | 112 -
.../main/java/org/apache/crunch/fn/IdentityFn.java | 39 -
.../main/java/org/apache/crunch/fn/MapKeysFn.java | 32 -
.../java/org/apache/crunch/fn/MapValuesFn.java | 32 -
.../main/java/org/apache/crunch/fn/PairMapFn.java | 65 -
.../java/org/apache/crunch/fn/package-info.java | 22 -
.../mapreduce/TaskAttemptContextFactory.java | 70 -
.../lib/jobcontrol/CrunchControlledJob.java | 325 -
.../mapreduce/lib/jobcontrol/CrunchJobControl.java | 211 -
.../org/apache/crunch/impl/SingleUseIterable.java | 49 -
.../org/apache/crunch/impl/mem/MemPipeline.java | 275 -
.../crunch/impl/mem/collect/MemCollection.java | 295 -
.../crunch/impl/mem/collect/MemGroupedTable.java | 113 -
.../apache/crunch/impl/mem/collect/MemTable.java | 177 -
.../apache/crunch/impl/mem/collect/Shuffler.java | 149 -
.../crunch/impl/mem/emit/InMemoryEmitter.java | 57 -
.../org/apache/crunch/impl/mem/package-info.java | 22 -
.../java/org/apache/crunch/impl/mr/MRPipeline.java | 396 -
.../crunch/impl/mr/collect/DoCollectionImpl.java | 74 -
.../apache/crunch/impl/mr/collect/DoTableImpl.java | 84 -
.../crunch/impl/mr/collect/InputCollection.java | 85 -
.../apache/crunch/impl/mr/collect/InputTable.java | 86 -
.../crunch/impl/mr/collect/PCollectionImpl.java | 295 -
.../crunch/impl/mr/collect/PGroupedTableImpl.java | 144 -
.../apache/crunch/impl/mr/collect/PTableBase.java | 169 -
.../crunch/impl/mr/collect/UnionCollection.java | 80 -
.../apache/crunch/impl/mr/collect/UnionTable.java | 92 -
.../crunch/impl/mr/emit/IntermediateEmitter.java | 64 -
.../crunch/impl/mr/emit/MultipleOutputEmitter.java | 56 -
.../apache/crunch/impl/mr/emit/OutputEmitter.java | 52 -
.../impl/mr/exec/CappedExponentialCounter.java | 40 -
.../apache/crunch/impl/mr/exec/CrunchJobHooks.java | 153 -
.../org/apache/crunch/impl/mr/exec/MRExecutor.java | 198 -
.../org/apache/crunch/impl/mr/package-info.java | 22 -
.../org/apache/crunch/impl/mr/plan/DoNode.java | 163 -
.../apache/crunch/impl/mr/plan/DotfileWriter.java | 238 -
.../java/org/apache/crunch/impl/mr/plan/Edge.java | 125 -
.../java/org/apache/crunch/impl/mr/plan/Graph.java | 133 -
.../apache/crunch/impl/mr/plan/GraphBuilder.java | 92 -
.../apache/crunch/impl/mr/plan/JobNameBuilder.java | 79 -
.../apache/crunch/impl/mr/plan/JobPrototype.java | 245 -
.../crunch/impl/mr/plan/MSCROutputHandler.java | 77 -
.../apache/crunch/impl/mr/plan/MSCRPlanner.java | 378 -
.../org/apache/crunch/impl/mr/plan/NodePath.java | 124 -
.../crunch/impl/mr/plan/PlanningParameters.java | 38 -
.../org/apache/crunch/impl/mr/plan/Vertex.java | 126 -
.../apache/crunch/impl/mr/run/CrunchCombiner.java | 27 -
.../crunch/impl/mr/run/CrunchInputFormat.java | 78 -
.../crunch/impl/mr/run/CrunchInputSplit.java | 116 -
.../apache/crunch/impl/mr/run/CrunchMapper.java | 73 -
.../crunch/impl/mr/run/CrunchRecordReader.java | 75 -
.../apache/crunch/impl/mr/run/CrunchReducer.java | 73 -
.../crunch/impl/mr/run/CrunchTaskContext.java | 86 -
.../org/apache/crunch/impl/mr/run/NodeContext.java | 35 -
.../java/org/apache/crunch/impl/mr/run/RTNode.java | 124 -
.../crunch/impl/mr/run/RuntimeParameters.java | 38 -
crunch/src/main/java/org/apache/crunch/io/At.java | 257 -
.../apache/crunch/io/CompositePathIterable.java | 102 -
.../java/org/apache/crunch/io/CrunchInputs.java | 71 -
.../java/org/apache/crunch/io/CrunchOutputs.java | 184 -
.../org/apache/crunch/io/FileNamingScheme.java | 58 -
.../org/apache/crunch/io/FileReaderFactory.java | 27 -
.../java/org/apache/crunch/io/FormatBundle.java | 121 -
.../src/main/java/org/apache/crunch/io/From.java | 324 -
.../java/org/apache/crunch/io/MapReduceTarget.java | 27 -
.../java/org/apache/crunch/io/OutputHandler.java | 25 -
.../main/java/org/apache/crunch/io/PathTarget.java | 36 -
.../java/org/apache/crunch/io/PathTargetImpl.java | 64 -
.../java/org/apache/crunch/io/ReadableSource.java | 41 -
.../org/apache/crunch/io/ReadableSourceTarget.java | 30 -
.../crunch/io/SequentialFileNamingScheme.java | 51 -
.../org/apache/crunch/io/SourceTargetHelper.java | 48 -
crunch/src/main/java/org/apache/crunch/io/To.java | 153 -
.../crunch/io/avro/AvroFileReaderFactory.java | 96 -
.../org/apache/crunch/io/avro/AvroFileSource.java | 58 -
.../crunch/io/avro/AvroFileSourceTarget.java | 39 -
.../org/apache/crunch/io/avro/AvroFileTarget.java | 91 -
.../apache/crunch/io/impl/AutoClosingIterator.java | 62 -
.../org/apache/crunch/io/impl/FileSourceImpl.java | 104 -
.../apache/crunch/io/impl/FileTableSourceImpl.java | 41 -
.../org/apache/crunch/io/impl/FileTargetImpl.java | 162 -
.../io/impl/ReadableSourcePathTargetImpl.java | 39 -
.../crunch/io/impl/ReadableSourceTargetImpl.java | 37 -
.../crunch/io/impl/SourcePathTargetImpl.java | 50 -
.../apache/crunch/io/impl/SourceTargetImpl.java | 89 -
.../crunch/io/impl/TableSourcePathTargetImpl.java | 41 -
.../crunch/io/impl/TableSourceTargetImpl.java | 35 -
.../java/org/apache/crunch/io/package-info.java | 22 -
.../org/apache/crunch/io/seq/SeqFileHelper.java | 35 -
.../apache/crunch/io/seq/SeqFileReaderFactory.java | 112 -
.../org/apache/crunch/io/seq/SeqFileSource.java | 47 -
.../apache/crunch/io/seq/SeqFileSourceTarget.java | 44 -
.../apache/crunch/io/seq/SeqFileTableSource.java | 57 -
.../crunch/io/seq/SeqFileTableSourceTarget.java | 54 -
.../org/apache/crunch/io/seq/SeqFileTarget.java | 55 -
.../crunch/io/text/BZip2TextInputFormat.java | 235 -
.../apache/crunch/io/text/CBZip2InputStream.java | 980 -
.../java/org/apache/crunch/io/text/LineParser.java | 125 -
.../org/apache/crunch/io/text/NLineFileSource.java | 77 -
.../crunch/io/text/TextFileReaderFactory.java | 83 -
.../org/apache/crunch/io/text/TextFileSource.java | 73 -
.../crunch/io/text/TextFileSourceTarget.java | 44 -
.../apache/crunch/io/text/TextFileTableSource.java | 81 -
.../crunch/io/text/TextFileTableSourceTarget.java | 63 -
.../org/apache/crunch/io/text/TextFileTarget.java | 109 -
.../main/java/org/apache/crunch/lib/Aggregate.java | 272 -
.../main/java/org/apache/crunch/lib/Cartesian.java | 216 -
.../main/java/org/apache/crunch/lib/Cogroup.java | 106 -
.../main/java/org/apache/crunch/lib/Distinct.java | 126 -
.../src/main/java/org/apache/crunch/lib/Join.java | 181 -
.../main/java/org/apache/crunch/lib/PTables.java | 117 -
.../main/java/org/apache/crunch/lib/Sample.java | 217 -
.../java/org/apache/crunch/lib/SampleUtils.java | 168 -
.../java/org/apache/crunch/lib/SecondarySort.java | 118 -
.../src/main/java/org/apache/crunch/lib/Set.java | 118 -
.../src/main/java/org/apache/crunch/lib/Sort.java | 294 -
.../apache/crunch/lib/join/FullOuterJoinFn.java | 102 -
.../org/apache/crunch/lib/join/InnerJoinFn.java | 78 -
.../java/org/apache/crunch/lib/join/JoinFn.java | 81 -
.../java/org/apache/crunch/lib/join/JoinUtils.java | 126 -
.../apache/crunch/lib/join/LeftOuterJoinFn.java | 98 -
.../org/apache/crunch/lib/join/MapsideJoin.java | 164 -
.../apache/crunch/lib/join/RightOuterJoinFn.java | 83 -
.../org/apache/crunch/lib/join/package-info.java | 22 -
.../java/org/apache/crunch/lib/package-info.java | 22 -
.../org/apache/crunch/lib/sort/Comparators.java | 187 -
.../java/org/apache/crunch/lib/sort/SortFns.java | 210 -
.../crunch/lib/sort/TotalOrderPartitioner.java | 145 -
.../crunch/materialize/MaterializableIterable.java | 81 -
.../crunch/materialize/MaterializableMap.java | 50 -
.../materialize/pobject/CollectionPObject.java | 55 -
.../materialize/pobject/FirstElementPObject.java | 50 -
.../crunch/materialize/pobject/MapPObject.java | 62 -
.../crunch/materialize/pobject/PObjectImpl.java | 85 -
.../main/java/org/apache/crunch/package-info.java | 25 -
.../apache/crunch/types/CollectionDeepCopier.java | 57 -
.../java/org/apache/crunch/types/Converter.java | 41 -
.../java/org/apache/crunch/types/DeepCopier.java | 60 -
.../org/apache/crunch/types/MapDeepCopier.java | 54 -
.../org/apache/crunch/types/PGroupedTableType.java | 141 -
.../java/org/apache/crunch/types/PTableType.java | 44 -
.../main/java/org/apache/crunch/types/PType.java | 86 -
.../java/org/apache/crunch/types/PTypeFamily.java | 77 -
.../java/org/apache/crunch/types/PTypeUtils.java | 66 -
.../main/java/org/apache/crunch/types/PTypes.java | 252 -
.../main/java/org/apache/crunch/types/Protos.java | 173 -
.../org/apache/crunch/types/TupleDeepCopier.java | 65 -
.../java/org/apache/crunch/types/TupleFactory.java | 134 -
.../apache/crunch/types/avro/AvroCapabilities.java | 106 -
.../apache/crunch/types/avro/AvroDeepCopier.java | 209 -
.../crunch/types/avro/AvroGroupedTableType.java | 114 -
.../apache/crunch/types/avro/AvroInputFormat.java | 41 -
.../apache/crunch/types/avro/AvroKeyConverter.java | 65 -
.../apache/crunch/types/avro/AvroOutputFormat.java | 87 -
.../crunch/types/avro/AvroPairConverter.java | 108 -
.../apache/crunch/types/avro/AvroRecordReader.java | 114 -
.../apache/crunch/types/avro/AvroTableType.java | 151 -
.../crunch/types/avro/AvroTextOutputFormat.java | 60 -
.../org/apache/crunch/types/avro/AvroType.java | 199 -
.../apache/crunch/types/avro/AvroTypeFamily.java | 164 -
.../crunch/types/avro/AvroUtf8InputFormat.java | 98 -
.../java/org/apache/crunch/types/avro/Avros.java | 709 -
.../crunch/types/avro/ReflectDataFactory.java | 41 -
.../crunch/types/avro/SafeAvroSerialization.java | 145 -
.../org/apache/crunch/types/avro/package-info.java | 22 -
.../java/org/apache/crunch/types/package-info.java | 22 -
.../types/writable/GenericArrayWritable.java | 135 -
.../crunch/types/writable/TextMapWritable.java | 88 -
.../crunch/types/writable/TupleWritable.java | 224 -
.../crunch/types/writable/WritableDeepCopier.java | 70 -
.../types/writable/WritableGroupedTableType.java | 85 -
.../types/writable/WritablePairConverter.java | 62 -
.../crunch/types/writable/WritableTableType.java | 130 -
.../apache/crunch/types/writable/WritableType.java | 133 -
.../crunch/types/writable/WritableTypeFamily.java | 147 -
.../types/writable/WritableValueConverter.java | 60 -
.../apache/crunch/types/writable/Writables.java | 588 -
.../apache/crunch/types/writable/package-info.java | 22 -
.../java/org/apache/crunch/util/CrunchTool.java | 118 -
.../java/org/apache/crunch/util/DistCache.java | 231 -
.../org/apache/crunch/util/PartitionUtils.java | 34 -
.../main/java/org/apache/crunch/util/Tuples.java | 150 -
.../java/org/apache/crunch/util/package-info.java | 22 -
crunch/src/main/resources/log4j.properties | 24 -
crunch/src/site/site.xml | 34 -
crunch/src/test/avro/employee.avsc | 26 -
crunch/src/test/avro/person.avsc | 26 -
.../src/test/java/org/apache/crunch/AndFnTest.java | 77 -
.../test/java/org/apache/crunch/CombineFnTest.java | 222 -
.../src/test/java/org/apache/crunch/NotFnTest.java | 72 -
.../src/test/java/org/apache/crunch/OrFnTest.java | 78 -
.../src/test/java/org/apache/crunch/PairTest.java | 66 -
.../src/test/java/org/apache/crunch/TupleTest.java | 139 -
.../test/java/org/apache/crunch/WriteModeTest.java | 103 -
.../java/org/apache/crunch/fn/AggregatorsTest.java | 239 -
.../org/apache/crunch/fn/ExtractKeyFnTest.java | 44 -
.../java/org/apache/crunch/fn/FilterFnTest.java | 85 -
.../java/org/apache/crunch/fn/MapKeysTest.java | 51 -
.../java/org/apache/crunch/fn/MapValuesTest.java | 50 -
.../java/org/apache/crunch/fn/PairMapTest.java | 52 -
.../org/apache/crunch/fn/StoreLastEmitter.java | 41 -
.../apache/crunch/impl/SingleUseIterableTest.java | 54 -
.../org/apache/crunch/impl/mr/MRPipelineTest.java | 86 -
.../impl/mr/collect/DoCollectionImplTest.java | 112 -
.../crunch/impl/mr/collect/DoTableImplTest.java | 86 -
.../impl/mr/emit/IntermediateEmitterTest.java | 83 -
.../impl/mr/exec/CappedExponentialCounterTest.java | 42 -
.../crunch/impl/mr/exec/CrunchJobHooksTest.java | 42 -
.../crunch/impl/mr/plan/DotfileWriterTest.java | 132 -
.../crunch/impl/mr/plan/JobNameBuilderTest.java | 41 -
.../crunch/io/SequentialFileNamingSchemeTest.java | 84 -
.../apache/crunch/io/SourceTargetHelperTest.java | 59 -
.../crunch/io/avro/AvroFileReaderFactoryTest.java | 184 -
.../apache/crunch/io/avro/AvroFileSourceTest.java | 91 -
.../lib/AvroIndexedRecordPartitionerTest.java | 98 -
.../java/org/apache/crunch/lib/CartesianTest.java | 77 -
.../java/org/apache/crunch/lib/DistinctTest.java | 52 -
.../java/org/apache/crunch/lib/SampleTest.java | 71 -
.../org/apache/crunch/lib/SecondarySortTest.java | 53 -
.../crunch/lib/TupleWritablePartitionerTest.java | 68 -
.../lib/join/BrokenLeftAndOuterJoinTest.java | 90 -
.../crunch/lib/join/FullOuterJoinFnTest.java | 48 -
.../apache/crunch/lib/join/InnerJoinFnTest.java | 42 -
.../org/apache/crunch/lib/join/JoinFnTestBase.java | 82 -
.../apache/crunch/lib/join/LeftOuterJoinTest.java | 46 -
.../crunch/lib/join/RightOuterJoinFnTest.java | 46 -
.../java/org/apache/crunch/test/CountersTest.java | 70 -
.../java/org/apache/crunch/test/StringWrapper.java | 102 -
.../crunch/types/CollectionDeepCopierTest.java | 61 -
.../org/apache/crunch/types/MapDeepCopierTest.java | 63 -
.../org/apache/crunch/types/PTypeUtilsTest.java | 89 -
.../java/org/apache/crunch/types/PTypesTest.java | 34 -
.../apache/crunch/types/TupleDeepCopierTest.java | 77 -
.../org/apache/crunch/types/TupleFactoryTest.java | 69 -
.../crunch/types/avro/AvroDeepCopierTest.java | 107 -
.../types/avro/AvroGroupedTableTypeTest.java | 60 -
.../crunch/types/avro/AvroTableTypeTest.java | 72 -
.../org/apache/crunch/types/avro/AvroTypeTest.java | 279 -
.../org/apache/crunch/types/avro/AvrosTest.java | 325 -
.../types/writable/GenericArrayWritableTest.java | 70 -
.../types/writable/WritableDeepCopierTest.java | 54 -
.../writable/WritableGroupedTableTypeTest.java | 56 -
.../types/writable/WritableTableTypeTest.java | 47 -
.../crunch/types/writable/WritableTypeTest.java | 97 -
.../crunch/types/writable/WritablesTest.java | 256 -
.../java/org/apache/crunch/util/DistCacheTest.java | 156 -
pom.xml | 4 +-
702 files changed, 70421 insertions(+), 70421 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-contrib/pom.xml b/crunch-contrib/pom.xml
index 12f9a13..e5a35c5 100644
--- a/crunch-contrib/pom.xml
+++ b/crunch-contrib/pom.xml
@@ -32,7 +32,7 @@ under the License.
<dependency>
<groupId>org.apache.crunch</groupId>
- <artifactId>crunch</artifactId>
+ <artifactId>crunch-core</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
new file mode 100644
index 0000000..d365c3d
--- /dev/null
+++ b/crunch-core/pom.xml
@@ -0,0 +1,182 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.crunch</groupId>
+ <artifactId>crunch-parent</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>crunch-core</artifactId>
+ <name>Apache Crunch Core</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-mapred</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Override the slf4j dependency from Avro, which is incompatible with
+ Hadoop's. -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Both Protobufs and Thrift are supported as
+ derived serialization types, and you can use
+ (almost) any version of them you like, Crunch
+ only relies on the stable public APIs, not the
+ structure of the files themselves.
+
+ Both dependencies are scoped as provided, in
+ order to not expand the size of the assembly jars
+ unnecessarily.
+ -->
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Used by LocalJobRunner in integration tests -->
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.crunch</groupId>
+ <artifactId>crunch-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>schemas</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>schema</goal>
+ </goals>
+ <configuration>
+ <testSourceDirectory>${project.basedir}/src/test/avro/</testSourceDirectory>
+ <testOutputDirectory>target/generated-test-sources/</testOutputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java b/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java
new file mode 100644
index 0000000..ff01a2f
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.io.To;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class CancelJobsIT {
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void testRun() throws Exception {
+ PipelineExecution pe = run();
+ pe.waitUntilDone();
+ PipelineResult pr = pe.getResult();
+ assertEquals(PipelineExecution.Status.SUCCEEDED, pe.getStatus());
+ assertEquals(2, pr.getStageResults().size());
+ }
+
+ @Test
+ public void testKill() throws Exception {
+ PipelineExecution pe = run();
+ pe.kill();
+ pe.waitUntilDone();
+ assertEquals(PipelineExecution.Status.KILLED, pe.getStatus());
+ }
+
+ @Test
+ public void testKillMultipleTimes() throws Exception {
+ PipelineExecution pe = run();
+ for (int i = 0; i < 10; i++) {
+ pe.kill();
+ }
+ pe.waitUntilDone();
+ assertEquals(PipelineExecution.Status.KILLED, pe.getStatus());
+ }
+
+ @Test
+ public void testKillAfterDone() throws Exception {
+ PipelineExecution pe = run();
+ pe.waitUntilDone();
+ assertEquals(PipelineExecution.Status.SUCCEEDED, pe.getStatus());
+ pe.kill(); // expect no-op
+ assertEquals(PipelineExecution.Status.SUCCEEDED, pe.getStatus());
+ }
+
+ public PipelineExecution run() throws IOException {
+ String shakes = tmpDir.copyResourceFileName("shakes.txt");
+ String out = tmpDir.getFileName("cancel");
+ Pipeline p = new MRPipeline(CancelJobsIT.class, tmpDir.getDefaultConfiguration());
+ PCollection<String> words = p.readTextFile(shakes);
+ p.write(words.count().top(20), To.textFile(out));
+ return p.runAsync(); // need to hack to slow down job start up if this test becomes flaky.
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java b/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java
new file mode 100644
index 0000000..2f4004e
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.io.To;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.avro.Avros;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Files;
+
+/**
+ *
+ */
+public class CleanTextIT {
+
+ private static final int LINES_IN_SHAKES = 3667;
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ static DoFn<String, String> CLEANER = new DoFn<String, String>() {
+ @Override
+ public void process(String input, Emitter<String> emitter) {
+ emitter.emit(input.toLowerCase());
+ }
+ };
+
+ static DoFn<String, String> SPLIT = new DoFn<String, String>() {
+ @Override
+ public void process(String input, Emitter<String> emitter) {
+ for (String word : input.split("\\S+")) {
+ if (!word.isEmpty()) {
+ emitter.emit(word);
+ }
+ }
+ }
+ };
+
+ @Test
+ public void testMapSideOutputs() throws Exception {
+ Pipeline pipeline = new MRPipeline(CleanTextIT.class, tmpDir.getDefaultConfiguration());
+ String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+
+ PCollection<String> cleanShakes = shakespeare.parallelDo(CLEANER, Avros.strings());
+ File cso = tmpDir.getFile("cleanShakes");
+ cleanShakes.write(To.textFile(cso.getAbsolutePath()));
+
+ File wc = tmpDir.getFile("wordCounts");
+ cleanShakes.parallelDo(SPLIT, Avros.strings()).count().write(To.textFile(wc.getAbsolutePath()));
+ pipeline.done();
+
+ File cleanFile = new File(cso, "part-m-00000");
+ List<String> lines = Files.readLines(cleanFile, Charset.defaultCharset());
+ assertEquals(LINES_IN_SHAKES, lines.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java
new file mode 100644
index 0000000..7e0c75c
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.String;
+import java.util.Collection;
+
+import org.apache.crunch.PCollection;
+import org.apache.crunch.PObject;
+import org.apache.crunch.Pipeline;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.materialize.pobject.CollectionPObject;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.junit.Rule;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class CollectionPObjectIT {
+
+ private static final int LINES_IN_SHAKES = 3667;
+
+ private static final String FIRST_SHAKESPEARE_LINE =
+ "***The Project Gutenberg's Etext of Shakespeare's First Folio***";
+
+ private static final String LAST_SHAKESPEARE_LINE =
+ "FINIS. THE TRAGEDIE OF MACBETH.";
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void testPObjectMRPipeline() throws IOException {
+ runPObject(new MRPipeline(CollectionPObjectIT.class, tmpDir.getDefaultConfiguration()));
+ }
+
+ @Test
+ public void testAsCollectionMRPipeline() throws IOException {
+ runAsCollection(new MRPipeline(CollectionPObjectIT.class, tmpDir.getDefaultConfiguration()));
+ }
+
+ @Test
+ public void testPObjectMemPipeline() throws IOException {
+ runPObject(MemPipeline.getInstance());
+ }
+
+ @Test
+ public void testAsCollectionMemPipeline() throws IOException {
+ runAsCollection(MemPipeline.getInstance());
+ }
+
+ private PCollection<String> getPCollection(Pipeline pipeline) throws IOException {
+ String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+ return shakespeare;
+ }
+
+ private void verifyLines(String[] lines) {
+ assertEquals("Not enough lines in Shakespeare.", LINES_IN_SHAKES, lines.length);
+ assertEquals("First line in Shakespeare is wrong.", FIRST_SHAKESPEARE_LINE, lines[0]);
+ assertEquals("Last line in Shakespeare is wrong.", LAST_SHAKESPEARE_LINE,
+ lines[lines.length - 1]);
+ }
+
+ public void runPObject(Pipeline pipeline) throws IOException {
+ PCollection<String> shakespeare = getPCollection(pipeline);
+ PObject<Collection<String>> linesP = new CollectionPObject<String>(shakespeare);
+ String[] lines = new String[LINES_IN_SHAKES];
+ lines = linesP.getValue().toArray(lines);
+ verifyLines(lines);
+ }
+
+ public void runAsCollection(Pipeline pipeline) throws IOException {
+ PCollection<String> shakespeare = getPCollection(pipeline);
+ String[] lines = new String[LINES_IN_SHAKES];
+ lines = shakespeare.asCollection().getValue().toArray(lines);
+ verifyLines(lines);
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java
new file mode 100644
index 0000000..17d0cae
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.crunch.fn.Aggregators.SimpleAggregator;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.types.avro.AvroTypeFamily;
+import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+public class CollectionsIT {
+
+ private static class AggregateStringListFn extends SimpleAggregator<Collection<String>> {
+ private final Collection<String> rtn = Lists.newArrayList();
+
+ @Override
+ public void reset() {
+ rtn.clear();
+ }
+
+ @Override
+ public void update(Collection<String> values) {
+ rtn.addAll(values);
+ }
+
+ @Override
+ public Iterable<Collection<String>> results() {
+ return ImmutableList.of(rtn);
+ }
+ }
+
+ private static PTable<String, Collection<String>> listOfCharcters(PCollection<String> lines, PTypeFamily typeFamily) {
+
+ return lines.parallelDo(new DoFn<String, Pair<String, Collection<String>>>() {
+ @Override
+ public void process(String line, Emitter<Pair<String, Collection<String>>> emitter) {
+ for (String word : line.split("\\s+")) {
+ Collection<String> characters = Lists.newArrayList();
+ for (char c : word.toCharArray()) {
+ characters.add(String.valueOf(c));
+ }
+ emitter.emit(Pair.of(word, characters));
+ }
+ }
+ }, typeFamily.tableOf(typeFamily.strings(), typeFamily.collections(typeFamily.strings())))
+ .groupByKey().combineValues(new AggregateStringListFn());
+ }
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void testWritables() throws IOException {
+ run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testAvro() throws IOException {
+ run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), AvroTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testInMemoryWritables() throws IOException {
+ run(MemPipeline.getInstance(), WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testInMemoryAvro() throws IOException {
+ run(MemPipeline.getInstance(), AvroTypeFamily.getInstance());
+ }
+
+ public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
+ String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+ Iterable<Pair<String, Collection<String>>> lines = listOfCharcters(shakespeare, typeFamily).materialize();
+
+ boolean passed = false;
+ for (Pair<String, Collection<String>> line : lines) {
+ if (line.first().startsWith("yellow")) {
+ passed = true;
+ break;
+ }
+ }
+ pipeline.done();
+ assertTrue(passed);
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java
new file mode 100644
index 0000000..3a38b92
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.Long;
+
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.types.avro.AvroTypeFamily;
+import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class CollectionsLengthIT {
+
+ public static final Long LINES_IN_SHAKESPEARE = 3667L;
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void testWritables() throws IOException {
+ run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testAvro() throws IOException {
+ run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), AvroTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testInMemoryWritables() throws IOException {
+ run(MemPipeline.getInstance(), WritableTypeFamily.getInstance());
+ }
+
+ @Test
+ public void testInMemoryAvro() throws IOException {
+ run(MemPipeline.getInstance(), AvroTypeFamily.getInstance());
+ }
+
+ public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
+ String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+ Long length = shakespeare.length().getValue();
+ assertEquals("Incorrect length for shakespear PCollection.", LINES_IN_SHAKESPEARE, length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java b/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java
new file mode 100644
index 0000000..f1323ca
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.apache.crunch.types.avro.Avros.*;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PType;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+/**
+ *
+ */
+public class DeepCopyCustomTuplesIT {
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ public static class PID extends Pair<Integer, String> {
+ public PID(Integer first, String second) {
+ super(first, second);
+ }
+ }
+
+ private static PType<PID> pids = tuples(PID.class, ints(), strings());
+
+ @Test
+ public void testDeepCopyCustomTuple() throws Exception {
+ Pipeline p = new MRPipeline(DeepCopyCustomTuplesIT.class, tmpDir.getDefaultConfiguration());
+ String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+ PCollection<String> shakes = p.readTextFile(shakesInputPath);
+ Iterable<String> out = shakes
+ .parallelDo(new PreProcFn(), tableOf(ints(), pairs(ints(), pids)))
+ .groupByKey()
+ .parallelDo(new PostProcFn(), strings())
+ .materialize();
+ assertEquals(65, Iterables.size(out));
+ p.done();
+ }
+
+ private static class PreProcFn extends MapFn<String, Pair<Integer, Pair<Integer, PID>>> {
+ private int counter = 0;
+ @Override
+ public Pair<Integer, Pair<Integer, PID>> map(String input) {
+ return Pair.of(counter++, Pair.of(counter++, new PID(input.length(), input)));
+ }
+ };
+
+ private static class PostProcFn extends DoFn<Pair<Integer, Iterable<Pair<Integer, PID>>>, String> {
+ @Override
+ public void process(Pair<Integer, Iterable<Pair<Integer, PID>>> input, Emitter<String> emitter) {
+ for (Pair<Integer, PID> p : input.second()) {
+ if (p.second().first() > 0 && p.second().first() < 10) {
+ emitter.emit(p.second().second());
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java b/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java
new file mode 100644
index 0000000..1d0974e
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypes;
+import org.apache.crunch.types.writable.Writables;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class EnumPairIT implements Serializable {
+ @Rule
+ public transient TemporaryPath tmpDir = TemporaryPaths.create();
+
+ static enum etypes {
+ type1,
+ }
+
+ @Test
+ public void testEnumPTypes() throws IOException {
+ String inputFile1 = tmpDir.copyResourceFileName("set1.txt");
+ Pipeline pipeline = new MRPipeline(EnumPairIT.class);
+ PCollection<String> set1 = pipeline.readTextFile(inputFile1);
+ PTable<String, etypes> data = set1.parallelDo(new DoFn<String, Pair<String, etypes>>() {
+ @Override
+ public void process(String input, Emitter<Pair<String, etypes>> emitter) {
+ emitter.emit(new Pair<String, etypes>(input, etypes.type1));
+ }
+ }, Writables.tableOf(Writables.strings(), PTypes.enums(etypes.class, set1.getTypeFamily())));
+
+ Iterable<Pair<String, etypes>> materialized = data.materialize();
+ pipeline.run();
+ for (Pair<String, etypes> pair : materialized) {
+ assertEquals(etypes.type1, pair.second());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java b/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java
new file mode 100644
index 0000000..d985e10
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.String;
+
+import org.apache.crunch.PCollection;
+import org.apache.crunch.PObject;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.materialize.pobject.FirstElementPObject;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.junit.Rule;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class FirstElementPObjectIT {
+
+ private static final String FIRST_SHAKESPEARE_LINE =
+ "***The Project Gutenberg's Etext of Shakespeare's First Folio***";
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void testMRPipeline() throws IOException {
+ run(new MRPipeline(FirstElementPObjectIT.class, tmpDir.getDefaultConfiguration()));
+ }
+
+ @Test
+ public void testInMemoryPipeline() throws IOException {
+ run(MemPipeline.getInstance());
+ }
+
+ public void run(Pipeline pipeline) throws IOException {
+ String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+ PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+ PObject<String> firstLine = new FirstElementPObject<String>(shakespeare);
+ String first = firstLine.getValue();
+ assertEquals("First line in Shakespeare is wrong.", FIRST_SHAKESPEARE_LINE, first);
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java b/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java
new file mode 100644
index 0000000..da487eb
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.crunch.fn.IdentityFn;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.writable.Writables;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Verify that calling the iterator method on a Reducer-based Iterable
+ * is forcefully disallowed.
+ */
+public class IterableReuseProtectionIT {
+
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+
+ public void checkIteratorReuse(Pipeline pipeline) throws IOException {
+ Iterable<String> values = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt"))
+ .by(IdentityFn.<String>getInstance(), Writables.strings())
+ .groupByKey()
+ .combineValues(new TestIterableReuseFn())
+ .values().materialize();
+
+ List<String> valueList = Lists.newArrayList(values);
+ Collections.sort(valueList);
+ assertEquals(Lists.newArrayList("a", "b", "c", "e"), valueList);
+ }
+
+ @Test
+ public void testIteratorReuse_MRPipeline() throws IOException {
+ checkIteratorReuse(new MRPipeline(IterableReuseProtectionIT.class, tmpDir.getDefaultConfiguration()));
+ }
+
+ @Test
+ public void testIteratorReuse_InMemoryPipeline() throws IOException {
+ checkIteratorReuse(MemPipeline.getInstance());
+ }
+
+ static class TestIterableReuseFn extends CombineFn<String, String> {
+
+ @Override
+ public void process(Pair<String, Iterable<String>> input, Emitter<Pair<String, String>> emitter) {
+ StringBuilder combinedBuilder = new StringBuilder();
+ for (String v : input.second()) {
+ combinedBuilder.append(v);
+ }
+
+ try {
+ input.second().iterator();
+ throw new RuntimeException("Second call to iterator should throw an exception");
+ } catch (IllegalStateException e) {
+ // Expected situation
+ }
+ emitter.emit(Pair.of(input.first(), combinedBuilder.toString()));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java b/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java
new file mode 100644
index 0000000..7670e88
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.crunch.fn.FilterFns;
+import org.apache.crunch.fn.IdentityFn;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.io.To;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.writable.Writables;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class MRPipelineIT implements Serializable {
+ @Rule
+ public transient TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void materializedColShouldBeWritten() throws Exception {
+ File textFile = tmpDir.copyResourceFile("shakes.txt");
+ Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
+ PCollection<String> genericCollection = pipeline.readTextFile(textFile.getAbsolutePath());
+ pipeline.run();
+ PCollection<String> filter = genericCollection.filter("Filtering data", FilterFns.<String>ACCEPT_ALL());
+ filter.materialize();
+ pipeline.run();
+ File file = tmpDir.getFile("output.txt");
+ Target outFile = To.textFile(file.getAbsolutePath());
+ PCollection<String> write = filter.write(outFile);
+ write.materialize();
+ pipeline.run();
+ }
+
+
+
+ @Test
+ public void testPGroupedTableToMultipleOutputs() throws IOException{
+ Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
+ PGroupedTable<String, String> groupedLineTable = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt")).by(IdentityFn.<String>getInstance(), Writables.strings()).groupByKey();
+
+ PTable<String, String> ungroupedTableA = groupedLineTable.ungroup();
+ PTable<String, String> ungroupedTableB = groupedLineTable.ungroup();
+
+ File outputDirA = tmpDir.getFile("output_a");
+ File outputDirB = tmpDir.getFile("output_b");
+
+ pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
+ pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
+ pipeline.done();
+
+ // Verify that output from a single PGroupedTable can be sent to multiple collections
+ assertTrue(new File(outputDirA, "part-r-00000").exists());
+ assertTrue(new File(outputDirB, "part-r-00000").exists());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java b/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java
new file mode 100644
index 0000000..c48284f
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.materialize.pobject.MapPObject;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypeFamily;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+
+public class MapPObjectIT {
+
+ static final ImmutableList<Pair<Integer, String>> kvPairs = ImmutableList.of(Pair.of(0, "a"), Pair.of(1, "b"),
+ Pair.of(2, "c"), Pair.of(3, "e"));
+
+ public void assertMatches(Map<Integer, String> m) {
+ for (Integer k : m.keySet()) {
+ assertEquals(kvPairs.get(k).second(), m.get(k));
+ }
+ }
+
+ private static class Set1Mapper extends MapFn<String, Pair<Integer, String>> {
+ @Override
+ public Pair<Integer, String> map(String input) {
+
+ int k = -1;
+ if (input.equals("a"))
+ k = 0;
+ else if (input.equals("b"))
+ k = 1;
+ else if (input.equals("c"))
+ k = 2;
+ else if (input.equals("e"))
+ k = 3;
+ return Pair.of(k, input);
+ }
+ }
+ @Rule
+ public TemporaryPath tmpDir = TemporaryPaths.create();
+
+ @Test
+ public void testMemMapPObject() {
+ PTable<Integer, String> table = MemPipeline.tableOf(kvPairs);
+ PObject<Map<Integer, String>> map = new MapPObject<Integer, String>(table);
+ assertMatches(map.getValue());
+ }
+
+ @Test
+ public void testMemAsMap() {
+ PTable<Integer, String> table = MemPipeline.tableOf(kvPairs);
+ assertMatches(table.asMap().getValue());
+ }
+
+ private PTable<Integer, String> getMRPTable() throws IOException {
+ Pipeline p = new MRPipeline(MaterializeToMapIT.class, tmpDir.getDefaultConfiguration());
+ String inputFile = tmpDir.copyResourceFileName("set1.txt");
+ PCollection<String> c = p.readTextFile(inputFile);
+ PTypeFamily tf = c.getTypeFamily();
+ PTable<Integer, String> table = c.parallelDo(new Set1Mapper(), tf.tableOf(tf.ints(),
+ tf.strings()));
+ return table;
+ }
+
+ @Test
+ public void testMRMapPObject() throws IOException {
+ PTable<Integer, String> table = getMRPTable();
+ PObject<Map<Integer, String>> map = new MapPObject<Integer, String>(table);
+ assertMatches(map.getValue());
+ }
+
+ @Test
+ public void testMRAsMap() throws IOException {
+ PTable<Integer, String> table = getMRPTable();
+ assertMatches(table.asMap().getValue());
+ }
+}