You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/08/27 02:53:30 UTC
git commit: TEZ-1496. Multi MR inputs can not be configured without
accessing internal proto structures (Siddharth Seth via bikas)
Repository: tez
Updated Branches:
refs/heads/master 396b99279 -> 926faf182
TEZ-1496. Multi MR inputs can not be configured without accessing internal proto structures (Siddharth Seth via bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/926faf18
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/926faf18
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/926faf18
Branch: refs/heads/master
Commit: 926faf18217323f1715065ad3519a584ba5cdee1
Parents: 396b992
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Aug 26 17:53:20 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Aug 26 17:53:20 2014 -0700
----------------------------------------------------------------------
.../org/apache/tez/mapreduce/input/MRInput.java | 5 ++-
.../tez/mapreduce/input/MultiMRInput.java | 41 +++++++++++++++++++-
2 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/926faf18/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index 7deef7c..d541e7e 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -97,7 +97,7 @@ public class MRInput extends MRInputBase {
String inputPaths = null;
InputInitializerDescriptor customInitializerDescriptor = null;
- private MRInputConfigBuilder(Configuration conf, Class<?> inputFormat) {
+ MRInputConfigBuilder(Configuration conf, Class<?> inputFormat) {
this.conf = conf;
if (inputFormat != null) {
inputFormatProvided = true;
@@ -345,7 +345,8 @@ public class MRInput extends MRInputBase {
}
/**
- * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} for {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
+ * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder}
+ * for {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
* or {@link org.apache.hadoop.mapred.FileInputFormat} format based InputFormats.
* <p/>
* The preferred usage model is to provide all of the parameters, and use methods to configure
http://git-wip-us.apache.org/repos/asf/tez/blob/926faf18/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
index 4ab5a52..78d01c5 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
@@ -18,6 +18,8 @@
package org.apache.tez.mapreduce.input;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -27,12 +29,13 @@ import java.util.concurrent.locks.ReentrantLock;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-
import com.google.protobuf.ByteString;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.tez.mapreduce.input.base.MRInputBase;
import org.apache.tez.mapreduce.lib.MRInputUtils;
@@ -62,6 +65,42 @@ public class MultiMRInput extends MRInputBase {
private List<MRReader> readers = new LinkedList<MRReader>();
+ /**
+ * Create an {@link MultiMRInputConfigBuilder} to configure a {@link MultiMRInput}</p>
+ * The preferred usage model is to provide all of the parameters, and use methods to configure
+ * the Input.
+ * <p/>
+ * For legacy applications, which may already have a fully configured {@link
+ * org.apache.hadoop.conf.Configuration}
+ * instance, the inputFormat can be specified as null
+ * <p/>
+ * Typically, this will be used along with a custom {@link org.apache.tez.dag.api.VertexManagerPlugin}
+ * or {@link org.apache.tez.runtime.api.InputInitializer} to generate the multiple inputs to be
+ * used by each task. If this is not setup, this will work the same as {@link
+ * org.apache.tez.mapreduce.input.MRInput} </p>
+ * Grouping of splits is disabled by default.
+ *
+ * @param conf Configuration for the {@link MRInput}. This configuration instance will be
+ * modified in place
+ * @param inputFormat InputFormat derived class. This can be null. If the InputFormat specified
+ * is
+ * null, the provided configuration should be complete.
+ * @return {@link MultiMRInputConfigBuilder}
+ */
+ public static MultiMRInputConfigBuilder createConfigBuilder(Configuration conf,
+ @Nullable Class<?> inputFormat) {
+ MultiMRInputConfigBuilder configBuilder = new MultiMRInputConfigBuilder(conf, inputFormat);
+ configBuilder.setInputClassName(MultiMRInput.class.getName()).groupSplits(false);
+
+ return configBuilder;
+ }
+
+ public static class MultiMRInputConfigBuilder extends MRInput.MRInputConfigBuilder {
+ private MultiMRInputConfigBuilder(Configuration conf, Class<?> inputFormat) {
+ super(conf, inputFormat);
+ }
+ }
+
@Override
public List<Event> initialize() throws IOException {
super.initialize();