You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/07/17 21:22:38 UTC
git commit: CRUNCH-444: Generate Tuple5..Tuple22 and generalized
product Aggregators for Scrunch
Repository: crunch
Updated Branches:
refs/heads/master ded504eb1 -> 0e7e0d0b0
CRUNCH-444: Generate Tuple5..Tuple22 and generalized product Aggregators for Scrunch
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/0e7e0d0b
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/0e7e0d0b
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/0e7e0d0b
Branch: refs/heads/master
Commit: 0e7e0d0b079c2ff51dbcd483402802917ac79f8f
Parents: ded504e
Author: Josh Wills <jw...@apache.org>
Authored: Tue Jul 15 20:04:26 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Thu Jul 17 12:12:45 2014 -0700
----------------------------------------------------------------------
.../src/codegen/make_tuple_conversions.py | 51 +++++++
.../src/codegen/make_tuple_ptypefamily.py | 60 +++++++++
.../scrunch/AggregatorsIntegrationTest.scala | 41 ++++++
.../org/apache/crunch/scrunch/Aggregators.scala | 83 ++++++------
.../org/apache/crunch/scrunch/Conversions.scala | 2 +-
.../scrunch/GeneratedTupleConversions.scala | 129 ++++++++++++++++++
.../scrunch/GeneratedTuplePTypeFamily.scala | 135 +++++++++++++++++++
.../org/apache/crunch/scrunch/PTypeFamily.scala | 20 ++-
.../org/apache/crunch/scrunch/TupleNTest.scala | 32 +++++
9 files changed, 505 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/codegen/make_tuple_conversions.py
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/codegen/make_tuple_conversions.py b/crunch-scrunch/src/codegen/make_tuple_conversions.py
new file mode 100755
index 0000000..114ff1e
--- /dev/null
+++ b/crunch-scrunch/src/codegen/make_tuple_conversions.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+print """/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */"""
+print "package org.apache.crunch.scrunch\n"
+print "trait GeneratedTupleConversions {"
+for j in range(5, 23):
+ lets = letters[0: j]
+ types = ", ".join(lets)
+ pth = ", ".join(["%s: PTypeH" % x for x in lets])
+ print " implicit def tuple%d[%s] = new PTypeH[(%s)] {" % (j, pth, types)
+ print " def get(ptf: PTypeFamily) = {"
+ implicits = ", ".join(["implicitly[PTypeH[%s]].get(ptf)" % x for x in lets])
+ print " ptf.tuple%d(%s)" % (j, implicits)
+ print " }"
+ print " }\n"
+print "}"
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py
new file mode 100755
index 0000000..fc7b989
--- /dev/null
+++ b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+print """/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */"""
+print "package org.apache.crunch.scrunch\n"
+print "import org.apache.crunch.TupleN"
+print "import org.apache.crunch.types.PType\n"
+
+print "trait GeneratedTuplePTypeFamily extends BasePTypeFamily {"
+print " import GeneratedTupleHelper._\n"
+for j in range(5, 23):
+ lets = letters[0:j]
+ types = ", ".join(lets)
+ args = ", ".join(["p%d: PType[%s]" % (x, l) for (x, l) in enumerate(lets)])
+ print " def tuple%d[%s](%s) = {" % (j, types, args)
+
+ inout = ",".join(["t.get(%d).asInstanceOf[%s]" % (x, l) for (x, l) in enumerate(lets)])
+ print " val in = (t: TupleN) => (%s)" % inout
+
+ outin = ", ".join(lets)
+ outout = ", ".join(["t._%d" % (1 + x) for x in range(j)])
+ print " val out = (t: (%s)) => tupleN(%s)" % (outin, outout)
+ derout = ", ".join(["p%d" % x for x in range(j)])
+ print " derived(classOf[(%s)], in, out, ptf.tuples(%s))" % (outin, derout)
+ print " }\n"
+print "}\n"
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala
new file mode 100644
index 0000000..94a6e12
--- /dev/null
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala
@@ -0,0 +1,41 @@
+/*
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.crunch.scrunch
+
+import org.junit.Test
+
+class AggregatorsIntegrationTest extends CrunchSuite {
+ @Test def productAggregators {
+ val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration)
+ val input = tempDir.copyResourceFileName("shakes.txt")
+
+ val fcc = pipeline.read(From.textFile(input))
+ .flatMap(_.toLowerCase.split("\\s+"))
+ .filter(!_.isEmpty())
+ .map(word => (word.slice(0, 1), (1L, word.length)))
+ .groupByKey
+ .combineValues(Aggregators.product[(Long, Int)](Aggregators.sum[Long], Aggregators.max[Int]))
+ .materialize
+ assert(fcc.exists(_ == ("w", (1404, 12))))
+
+ pipeline.done
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
index 238a01f..8b0f6b0 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala
@@ -22,19 +22,19 @@ package org.apache.crunch.scrunch
import org.apache.crunch.fn.{Aggregators => JAgg}
import org.apache.crunch._
-import com.google.common.collect.ImmutableList
+import com.google.common.collect.{UnmodifiableIterator, ImmutableList}
import org.apache.hadoop.conf.Configuration
import java.lang.{Iterable => JIterable}
import scala.collection.JavaConversions
import com.twitter.algebird.Monoid
+import scala.reflect.ClassTag
+import java.util
/**
* Scrunch versions of the common Aggregator types from Crunch.
*/
object Aggregators {
- import scala.math.Ordering._
-
def sum[T: Monoid]: Aggregator[T] = sumUsing(implicitly[Monoid[T]])
def sumUsing[T](m: Monoid[T]): Aggregator[T] = new SimpleAggregator[T] {
@@ -176,14 +176,14 @@ object Aggregators {
* Apply separate aggregators to each component of a {@link Tuple2}.
*/
def pair[V1, V2](a1: Aggregator[V1], a2: Aggregator[V2]): Aggregator[(V1, V2)] = {
- return new Aggregators.PairAggregator[V1, V2](a1, a2)
+ new Aggregators.PairAggregator[V1, V2](a1, a2)
}
/**
* Apply separate aggregators to each component of a {@link Tuple3}.
*/
def trip[V1, V2, V3](a1: Aggregator[V1], a2: Aggregator[V2], a3: Aggregator[V3]): Aggregator[(V1, V2, V3)] = {
- return new Aggregators.TripAggregator[V1, V2, V3](a1, a2, a3)
+ new Aggregators.TripAggregator[V1, V2, V3](a1, a2, a3)
}
/**
@@ -191,7 +191,15 @@ object Aggregators {
*/
def quad[V1, V2, V3, V4](a1: Aggregator[V1], a2: Aggregator[V2], a3: Aggregator[V3], a4: Aggregator[V4])
: Aggregator[(V1, V2, V3, V4)] = {
- return new Aggregators.QuadAggregator[V1, V2, V3, V4](a1, a2, a3, a4)
+ new Aggregators.QuadAggregator[V1, V2, V3, V4](a1, a2, a3, a4)
+ }
+
+ /**
+ * Apply separate aggregators to each component of a {@link Product} subclass.
+ */
+ def product[T <: Product : ClassTag](aggregators: Aggregator[_]*): Aggregator[T] = {
+ new Aggregators.ProductAggregator[T](Array(aggregators : _*),
+ implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]])
}
/**
@@ -202,20 +210,22 @@ object Aggregators {
}
}
- private abstract class ProductAggregator[T <: Product](aggs: Array[Aggregator[_]]) extends Aggregator[T] {
+ private class ProductAggregator[T <: Product](val aggs: Array[Aggregator[_]], val clazz: Class[T])
+ extends Aggregator[T] {
+
def initialize(configuration: Configuration) {
for (a <- aggs) {
a.initialize(configuration)
}
}
- def reset {
+ def reset() {
for (a <- aggs) {
- a.reset
+ a.reset()
}
}
- protected def updateTuple(t: T) {
+ override def update(t: T) {
var i: Int = 0
while (i < aggs.length) {
aggs(i).asInstanceOf[Aggregator[Any]].update(t.productElement(i))
@@ -223,49 +233,40 @@ object Aggregators {
}
}
- protected def results[T](index: Int): Iterable[T] = {
- return JavaConversions.iterableAsScalaIterable(aggs(index).results.asInstanceOf[JIterable[T]])
+ override def results: JIterable[T] = {
+ val res = aggs.map(_.results()).map(JavaConversions.iterableAsScalaIterable(_).toList)
+ return new JIterable[T] {
+ override def iterator(): util.Iterator[T] = {
+ return new AggIterator[T](res, clazz)
+ }
+ }
}
}
- private class PairAggregator[A, B](val a1: Aggregator[A], val a2: Aggregator[B])
- extends ProductAggregator[(A, B)](Array(a1, a2)) {
+ private class AggIterator[T](val results: Array[List[_]], val clazz: Class[T]) extends UnmodifiableIterator[T] {
+ var offset = 0
+ val maxoffset = results.map(_.size).min
+ val construct = clazz.getConstructors.find(_.getParameterTypes.length == results.length)
- def update(value: (A, B)) {
- updateTuple(value)
- }
+ override def hasNext: Boolean = offset < maxoffset
- def results: JIterable[(A, B)] = {
- return JavaConversions.asJavaIterable(results[A](0).zip(results[B](1)))
+ override def next(): T = {
+ val refs = results.map(x => x(offset)).asInstanceOf[Array[AnyRef]]
+ offset = offset + 1
+ construct.get.newInstance(refs : _*).asInstanceOf[T]
}
}
- private class TripAggregator[A, B, C](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C])
- extends ProductAggregator[(A, B, C)](Array(a1, a2, a3)) {
- def update(value: (A, B, C)) {
- updateTuple(value)
- }
+ private class PairAggregator[A, B](val a1: Aggregator[A], val a2: Aggregator[B])
+ extends ProductAggregator[(A, B)](Array(a1, a2), classOf[(A, B)]) {
+ }
- def results: JIterable[(A, B, C)] = {
- return JavaConversions.asJavaIterable(
- results[A](0).zip(results[B](1)).zip(results[C](2))
- .map(x => (x._1._1, x._1._2, x._2)))
- }
+ private class TripAggregator[A, B, C](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C])
+ extends ProductAggregator[(A, B, C)](Array(a1, a2, a3), classOf[(A, B, C)]) {
}
private class QuadAggregator[A, B, C, D](val a1: Aggregator[A], val a2: Aggregator[B],
val a3: Aggregator[C], val a4: Aggregator[D])
- extends ProductAggregator[(A, B, C, D)](Array(a1, a2, a3, a4)) {
-
- def update(value: (A, B, C, D)) {
- updateTuple(value)
- }
-
- def results: JIterable[(A, B, C, D)] = {
- return JavaConversions.asJavaIterable(
- (results[A](0).zip(results[B](1))).zip(results[C](2).zip(results[D](3)))
- .map(x => (x._1._1, x._1._2, x._2._1, x._2._2)))
- }
+ extends ProductAggregator[(A, B, C, D)](Array(a1, a2, a3, a4), classOf[(A, B, C, D)]) {
}
-
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
index 05baebf..ffd85c6 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala
@@ -92,7 +92,7 @@ trait LowPriorityPTypeH {
}
}
-object PTypeH extends LowPriorityPTypeH {
+object PTypeH extends GeneratedTupleConversions with LowPriorityPTypeH {
implicit val longs = new PTypeH[Long] { def get(ptf: PTypeFamily) = ptf.longs }
implicit val jlongs = new PTypeH[java.lang.Long] { def get(ptf: PTypeFamily) = ptf.jlongs }
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala
new file mode 100644
index 0000000..e2218c5
--- /dev/null
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+trait GeneratedTupleConversions {
+ implicit def tuple5[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH] = new PTypeH[(A, B, C, D, E)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple5(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf))
+ }
+ }
+
+ implicit def tuple6[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH] = new PTypeH[(A, B, C, D, E, F)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple6(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf))
+ }
+ }
+
+ implicit def tuple7[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH] = new PTypeH[(A, B, C, D, E, F, G)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple7(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf))
+ }
+ }
+
+ implicit def tuple8[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple8(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf))
+ }
+ }
+
+ implicit def tuple9[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple9(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf))
+ }
+ }
+
+ implicit def tuple10[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple10(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf))
+ }
+ }
+
+ implicit def tuple11[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple11(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf))
+ }
+ }
+
+ implicit def tuple12[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple12(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf))
+ }
+ }
+
+ implicit def tuple13[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple13(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf))
+ }
+ }
+
+ implicit def tuple14[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple14(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf))
+ }
+ }
+
+ implicit def tuple15[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple15(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf))
+ }
+ }
+
+ implicit def tuple16[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple16(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf))
+ }
+ }
+
+ implicit def tuple17[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple17(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf))
+ }
+ }
+
+ implicit def tuple18[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple18(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf))
+ }
+ }
+
+ implicit def tuple19[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple19(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf))
+ }
+ }
+
+ implicit def tuple20[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple20(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf))
+ }
+ }
+
+ implicit def tuple21[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH, U: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple21(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf), implicitly[PTypeH[U]].get(ptf))
+ }
+ }
+
+ implicit def tuple22[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH, U: PTypeH, V: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)] {
+ def get(ptf: PTypeFamily) = {
+ ptf.tuple22(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf), implicitly[PTypeH[U]].get(ptf), implicitly[PTypeH[V]].get(ptf))
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala
new file mode 100644
index 0000000..34a424f
--- /dev/null
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+import org.apache.crunch.TupleN
+import org.apache.crunch.types.PType
+
+trait GeneratedTuplePTypeFamily extends BasePTypeFamily {
+ import GeneratedTupleHelper._
+
+ def tuple5[A, B, C, D, E](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E])
+ val out = (t: (A, B, C, D, E)) => tupleN(t._1, t._2, t._3, t._4, t._5)
+ derived(classOf[(A, B, C, D, E)], in, out, ptf.tuples(p0, p1, p2, p3, p4))
+ }
+
+ def tuple6[A, B, C, D, E, F](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F])
+ val out = (t: (A, B, C, D, E, F)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6)
+ derived(classOf[(A, B, C, D, E, F)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5))
+ }
+
+ def tuple7[A, B, C, D, E, F, G](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G])
+ val out = (t: (A, B, C, D, E, F, G)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7)
+ derived(classOf[(A, B, C, D, E, F, G)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6))
+ }
+
+ def tuple8[A, B, C, D, E, F, G, H](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H])
+ val out = (t: (A, B, C, D, E, F, G, H)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8)
+ derived(classOf[(A, B, C, D, E, F, G, H)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7))
+ }
+
+ def tuple9[A, B, C, D, E, F, G, H, I](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I])
+ val out = (t: (A, B, C, D, E, F, G, H, I)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9)
+ derived(classOf[(A, B, C, D, E, F, G, H, I)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8))
+ }
+
+ def tuple10[A, B, C, D, E, F, G, H, I, J](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9))
+ }
+
+ def tuple11[A, B, C, D, E, F, G, H, I, J, K](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10))
+ }
+
+ def tuple12[A, B, C, D, E, F, G, H, I, J, K, L](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11))
+ }
+
+ def tuple13[A, B, C, D, E, F, G, H, I, J, K, L, M](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12))
+ }
+
+ def tuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13))
+ }
+
+ def tuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14))
+ }
+
+ def tuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15))
+ }
+
+ def tuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16))
+ }
+
+ def tuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17))
+ }
+
+ def tuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18))
+ }
+
+ def tuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19))
+ }
+
+ def tuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T], p20: PType[U]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T],t.get(20).asInstanceOf[U])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20, t._21)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20))
+ }
+
+ def tuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T], p20: PType[U], p21: PType[V]) = {
+ val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T],t.get(20).asInstanceOf[U],t.get(21).asInstanceOf[V])
+ val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20, t._21, t._22)
+ derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21))
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
index bd03f6f..f77067a 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala
@@ -17,7 +17,7 @@
*/
package org.apache.crunch.scrunch
-import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, Union, MapFn}
+import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, TupleN, Union, MapFn}
import org.apache.crunch.types.{PType, PTypeFamily => PTF}
import org.apache.crunch.types.writable.{WritableTypeFamily, Writables => CWritables}
import org.apache.crunch.types.avro.{AvroType, AvroTypeFamily, Avros => CAvros}
@@ -47,10 +47,22 @@ class TMapFn[S, T](val f: S => T, val pt: Option[PType[S]] = None, var init: Boo
}
}
-trait PTypeFamily {
+object GeneratedTupleHelper {
+ def tupleN(args: Any*) = {
+ TupleN.of(args.map(_.asInstanceOf[AnyRef]): _*)
+ }
+}
+trait BasePTypeFamily {
def ptf: PTF
+ def derived[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = {
+ ptf.derived(cls, new TMapFn[S, T](in, Some(pt)), new TMapFn[T, S](out), pt)
+ }
+}
+
+trait PTypeFamily extends GeneratedTuplePTypeFamily {
+
def writables[T <: Writable : ClassTag]: PType[T]
def as[T](ptype: PType[T]) = ptf.as(ptype)
@@ -61,10 +73,6 @@ trait PTypeFamily {
def records[T: ClassTag] = ptf.records(implicitly[ClassTag[T]].runtimeClass)
- def derived[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = {
- ptf.derived(cls, new TMapFn[S, T](in, Some(pt)), new TMapFn[T, S](out), pt)
- }
-
def derivedImmutable[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = {
ptf.derivedImmutable(cls, new TMapFn[S, T](in), new TMapFn[T, S](out), pt)
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
new file mode 100644
index 0000000..640a72f
--- /dev/null
+++ b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala
@@ -0,0 +1,32 @@
+/*
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.crunch.scrunch
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.Test
+
+class TupleNTest extends JUnitSuite{
+ @Test def testTupleN {
+ val pc = Mem.collectionOf((1, 2, "a", 3, "b"), (4, 5, "a", 6, "c"))
+ val res = pc.map(x => (x._3, x._4)).groupByKey.combineValues(Aggregators.sum[Int]).materialize
+ org.junit.Assert.assertEquals(List(("a", 9)), res.toList)
+ }
+}