You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/05/31 23:47:53 UTC
git commit: CRUNCH-406: Have From/At code for Avro fields read PTypes
and do the internal casting themselves
Repository: crunch
Updated Branches:
refs/heads/master 567d25dd7 -> c135bba63
CRUNCH-406: Have From/At code for Avro fields read PTypes and do the internal casting themselves
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/c135bba6
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/c135bba6
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/c135bba6
Branch: refs/heads/master
Commit: c135bba631c106f93aa2cb4f973bbff4262aba0d
Parents: 567d25d
Author: Josh Wills <jw...@apache.org>
Authored: Fri May 30 12:54:47 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Sat May 31 14:43:10 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/crunch/io/At.java | 12 +++++-----
.../main/java/org/apache/crunch/io/From.java | 18 +++++++--------
.../scala/org/apache/crunch/scrunch/IO.scala | 24 ++++++++------------
3 files changed, 24 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-core/src/main/java/org/apache/crunch/io/At.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/At.java b/crunch-core/src/main/java/org/apache/crunch/io/At.java
index e60831a..9a8db2f 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/At.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/At.java
@@ -131,22 +131,22 @@ public class At {
* Creates a {@code SourceTarget<T>} instance from the Avro file(s) at the given path name.
*
* @param pathName The name of the path to the data on the filesystem
- * @param avroType The {@code AvroType} for the Avro records
+ * @param ptype The {@code PType} for the Avro records
* @return A new {@code SourceTarget<T>} instance
*/
- public static <T> SourceTarget<T> avroFile(String pathName, AvroType<T> avroType) {
- return avroFile(new Path(pathName), avroType);
+ public static <T> SourceTarget<T> avroFile(String pathName, PType<T> ptype) {
+ return avroFile(new Path(pathName), ptype);
}
/**
* Creates a {@code SourceTarget<T>} instance from the Avro file(s) at the given {@code Path}.
*
* @param path The {@code Path} to the data
- * @param avroType The {@code AvroType} for the Avro records
+ * @param ptype The {@code PType} for the Avro records
* @return A new {@code SourceTarget<T>} instance
*/
- public static <T> SourceTarget<T> avroFile(Path path, AvroType<T> avroType) {
- return new AvroFileSourceTarget<T>(path, avroType);
+ public static <T> SourceTarget<T> avroFile(Path path, PType<T> ptype) {
+ return new AvroFileSourceTarget<T>(path, (AvroType<T>) ptype);
}
/**
http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-core/src/main/java/org/apache/crunch/io/From.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/From.java b/crunch-core/src/main/java/org/apache/crunch/io/From.java
index 14793a6..d72ddf4 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/From.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/From.java
@@ -221,33 +221,33 @@ public class From {
* Creates a {@code Source<T>} instance from the Avro file(s) at the given path name.
*
* @param pathName The name of the path to the data on the filesystem
- * @param avroType The {@code AvroType} for the Avro records
+ * @param ptype The {@code AvroType} for the Avro records
* @return A new {@code Source<T>} instance
*/
- public static <T> Source<T> avroFile(String pathName, AvroType<T> avroType) {
- return avroFile(new Path(pathName), avroType);
+ public static <T> Source<T> avroFile(String pathName, PType<T> ptype) {
+ return avroFile(new Path(pathName), ptype);
}
/**
* Creates a {@code Source<T>} instance from the Avro file(s) at the given {@code Path}.
*
* @param path The {@code Path} to the data
- * @param avroType The {@code AvroType} for the Avro records
+ * @param ptype The {@code AvroType} for the Avro records
* @return A new {@code Source<T>} instance
*/
- public static <T> Source<T> avroFile(Path path, AvroType<T> avroType) {
- return new AvroFileSource<T>(path, avroType);
+ public static <T> Source<T> avroFile(Path path, PType<T> ptype) {
+ return new AvroFileSource<T>(path, (AvroType<T>) ptype);
}
/**
* Creates a {@code Source<T>} instance from the Avro file(s) at the given {@code Path}s.
*
* @param paths A list of {@code Path}s to the data
- * @param avroType The {@code AvroType} for the Avro records
+ * @param ptype The {@code PType} for the Avro records
* @return A new {@code Source<T>} instance
*/
- public static <T> Source<T> avroFile(List<Path> paths, AvroType<T> avroType) {
- return new AvroFileSource<T>(paths, avroType);
+ public static <T> Source<T> avroFile(List<Path> paths, PType<T> ptype) {
+ return new AvroFileSource<T>(paths, (AvroType<T>) ptype);
}
/**
http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
index c4af454..d9d1acd 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
@@ -20,19 +20,13 @@
package org.apache.crunch.scrunch
-import org.apache.crunch.io.{From => from, To => to, At => at, SequentialFileNamingScheme}
+import org.apache.crunch.io.{From => from, To => to, At => at}
import org.apache.hadoop.fs.Path
import org.apache.hadoop.conf.Configuration
import org.apache.crunch.types.PType
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.crunch.types.avro.AvroType
import org.apache.hadoop.io.Writable
import org.apache.avro.specific.SpecificRecord
-import org.apache.crunch.Target
-import org.apache.crunch.io.impl.FileTargetImpl
-import org.apache.crunch.io.avro.AvroFileTarget
-import org.apache.crunch.io.seq.SeqFileTarget
-import org.apache.crunch.io.text.TextFileTarget
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
object From {
@@ -80,16 +74,16 @@ object From {
from.avroFile(paths, avroClass)
}
- def avroFile[T](pathName: String, avroType: AvroType[T]) = {
- from.avroFile(pathName, avroType)
+ def avroFile[T](pathName: String, ptype: PType[T]) = {
+ from.avroFile(pathName, ptype)
}
- def avroFile[T](path: Path, avroType: AvroType[T]) = {
- from.avroFile(path, avroType)
+ def avroFile[T](path: Path, ptype: PType[T]) = {
+ from.avroFile(path, ptype)
}
- def avroFile[T](paths: List[Path], avroType: AvroType[T]) = {
- from.avroFile(paths, avroType)
+ def avroFile[T](paths: List[Path], ptype: PType[T]) = {
+ from.avroFile(paths, ptype)
}
def avroFile(pathName: String) = {
@@ -204,11 +198,11 @@ object At {
at.avroFile(path, avroClass)
}
- def avroFile[T](pathName: String, avroType: AvroType[T]) = {
+ def avroFile[T](pathName: String, avroType: PType[T]) = {
at.avroFile(pathName, avroType)
}
- def avroFile[T](path: Path, avroType: AvroType[T]) = {
+ def avroFile[T](path: Path, avroType: PType[T]) = {
at.avroFile(path, avroType)
}