You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2014/08/16 02:47:00 UTC
git commit: TEZ-1438. Annotate add java doc for tez-runtime-library
and tez-mapreduce. (bikas via hitesh)
Repository: tez
Updated Branches:
refs/heads/master 14f8b36ac -> 4a41ccf06
TEZ-1438. Annotate add java doc for tez-runtime-library and tez-mapreduce. (bikas via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4a41ccf0
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4a41ccf0
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4a41ccf0
Branch: refs/heads/master
Commit: 4a41ccf06ffbcc8103b008f43c23442b5f4e988d
Parents: 14f8b36
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Aug 15 17:46:25 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Aug 15 17:46:25 2014 -0700
----------------------------------------------------------------------
.../hadoop/mapred/split/TezGroupedSplit.java | 8 +++++++
.../split/TezGroupedSplitsInputFormat.java | 8 +++++++
.../mapred/split/TezMapredSplitsGrouper.java | 9 ++++++++
.../hadoop/mapreduce/split/TezGroupedSplit.java | 8 +++++++
.../split/TezGroupedSplitsInputFormat.java | 8 +++++++
.../split/TezMapReduceSplitsGrouper.java | 9 ++++++++
.../org/apache/tez/common/package-info.java | 22 +++++++++++++++++++
.../apache/tez/mapreduce/client/YARNRunner.java | 2 ++
.../client/YarnTezClientProtocolProvider.java | 2 ++
.../tez/mapreduce/client/package-info.java | 22 +++++++++++++++++++
.../tez/mapreduce/combine/MRCombiner.java | 5 +++++
.../mapreduce/committer/MROutputCommitter.java | 6 +++++
.../common/MRInputAMSplitGenerator.java | 11 ++++++++++
.../common/MRInputSplitDistributor.java | 13 +++++++++++
.../org/apache/tez/mapreduce/common/Utils.java | 1 +
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 4 ++++
.../tez/mapreduce/hadoop/MRInputHelpers.java | 4 ++++
.../mapreduce/hadoop/mapred/package-info.java | 22 +++++++++++++++++++
.../hadoop/mapreduce/package-info.java | 22 +++++++++++++++++++
.../tez/mapreduce/hadoop/package-info.java | 22 +++++++++++++++++++
.../org/apache/tez/mapreduce/input/MRInput.java | 8 ++++++-
.../tez/mapreduce/input/MultiMRInput.java | 5 +++++
.../apache/tez/mapreduce/lib/package-info.java | 22 +++++++++++++++++++
.../apache/tez/mapreduce/output/MROutput.java | 18 ++++++++++++++-
.../tez/mapreduce/output/MROutputLegacy.java | 2 ++
.../tez/mapreduce/partition/MRPartitioner.java | 7 ++++++
.../apache/tez/mapreduce/processor/MRTask.java | 1 +
.../mapreduce/processor/SimpleMRProcessor.java | 10 +++++++++
.../mapreduce/processor/map/MapProcessor.java | 2 ++
.../processor/reduce/ReduceProcessor.java | 3 ++-
.../common/sort/impl/dflt/package-info.java | 22 +++++++++++++++++++
.../library/common/writers/package-info.java | 22 +++++++++++++++++++
.../conf/OnFileSortedOutputConfigurer.java | 2 +-
.../conf/OnFileUnorderedKVOutputConfigurer.java | 2 +-
...eUnorderedPartitionedKVOutputConfigurer.java | 2 +-
.../conf/ShuffledMergedInputConfigurer.java | 2 +-
.../ShuffledUnorderedKVInputConfigurer.java | 2 +-
.../input/ConcatenatedMergedKeyValueInput.java | 14 ++++++++++--
.../input/ConcatenatedMergedKeyValuesInput.java | 15 +++++++++++--
.../library/input/ShuffledMergedInput.java | 7 ++++--
.../library/input/ShuffledUnorderedKVInput.java | 6 +++++
.../library/input/SortedGroupedMergedInput.java | 3 +++
.../library/output/OnFileSortedOutput.java | 6 +++--
.../library/output/OnFileUnorderedKVOutput.java | 6 +++++
.../library/partitioner/HashPartitioner.java | 3 +++
.../library/processor/PreWarmProcessor.java | 7 +++++-
.../library/processor/SimpleProcessor.java | 23 ++++++++++++++++++++
.../shuffle/common/impl/package-info.java | 22 +++++++++++++++++++
48 files changed, 435 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java
index 3fdd6a4..0c1c327 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -31,6 +33,12 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tez.dag.api.TezUncheckedException;
+/**
+ * Implements an InputSplit that provides a generic wrapper around
+ * a group of real InputSplits
+ */
+@Public
+@Evolving
public class TezGroupedSplit implements InputSplit, Configurable {
List<InputSplit> wrappedSplits = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java
index 76da547..a1044f0 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.InputFormat;
@@ -34,6 +36,12 @@ import org.apache.tez.dag.api.TezUncheckedException;
import com.google.common.base.Preconditions;
+/**
+ * An InputFormat that provides a generic grouping around the splits
+ * of a real InputFormat
+ */
+@Public
+@Evolving
public class TezGroupedSplitsInputFormat<K, V>
implements InputFormat<K, V>, Configurable{
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java
index 0e1c889..57d6f57 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java
@@ -28,6 +28,8 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -36,6 +38,13 @@ import org.apache.tez.dag.api.TezUncheckedException;
import com.google.common.base.Preconditions;
+/**
+ * A Helper that provides grouping logic to group InputSplits
+ * using various parameters. A {@link TezGroupedSplit} is used
+ * to wrap the real InputSplits in a group.
+ */
+@Public
+@Evolving
public class TezMapredSplitsGrouper {
private static final Log LOG = LogFactory.getLog(TezMapredSplitsGrouper.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java
index 31c0971..9275f14 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -31,6 +33,12 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.tez.dag.api.TezUncheckedException;
+/**
+ * A Mapreduce InputSplit that provides a generic wrapper
+ * around a set of real InputSplits.
+ */
+@Public
+@Evolving
public class TezGroupedSplit extends InputSplit
implements Writable, Configurable {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java
index 3fd0b6e..58fcf73 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java
@@ -24,6 +24,8 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -36,6 +38,12 @@ import org.apache.tez.dag.api.TezUncheckedException;
import com.google.common.base.Preconditions;
+/**
+ * An InputFormat that provides a generic grouping around
+ * the splits of a real InputFormat
+ */
+@Public
+@Evolving
public class TezGroupedSplitsInputFormat<K, V> extends InputFormat<K, V>
implements Configurable{
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java
index bb1295b..d0f53d2 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java
@@ -28,6 +28,8 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -36,6 +38,13 @@ import org.apache.tez.dag.api.TezUncheckedException;
import com.google.common.base.Preconditions;
+/**
+ * Helper that provides a grouping of input splits based
+ * on multiple parameters. It creates {@link TezGroupedSplit}
+ * to wrap the each group of real InputSplits
+ */
+@Public
+@Evolving
public class TezMapReduceSplitsGrouper {
private static final Log LOG = LogFactory.getLog(TezMapReduceSplitsGrouper.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/common/package-info.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/common/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/common/package-info.java
new file mode 100644
index 0000000..88a5e86
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/common/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.common;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index ed94c69..a303648 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -33,6 +33,7 @@ import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -123,6 +124,7 @@ import com.google.common.annotations.VisibleForTesting;
* This class enables the current JobClient (0.22 hadoop) to run on YARN-TEZ.
*/
@SuppressWarnings({ "unchecked" })
+@LimitedPrivate("Mapreduce")
public class YARNRunner implements ClientProtocol {
private static final Log LOG = LogFactory.getLog(YARNRunner.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java
index c36dc9d..b77dec3 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java
@@ -21,11 +21,13 @@ package org.apache.tez.mapreduce.client;
import java.io.IOException;
import java.net.InetSocketAddress;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
import org.apache.tez.mapreduce.hadoop.MRConfig;
+@LimitedPrivate("Mapreduce")
public class YarnTezClientProtocolProvider extends ClientProtocolProvider {
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java
new file mode 100644
index 0000000..4366a67
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.mapreduce.client;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
index 96c03c6..d8bd628 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
@@ -55,6 +56,10 @@ import org.apache.tez.runtime.library.common.combine.Combiner;
import org.apache.tez.runtime.library.common.sort.impl.TezRawKeyValueIterator;
import org.apache.tez.runtime.library.common.sort.impl.IFile.Writer;
+/**
+ * Implements a Map Reduce compatible combiner
+ */
+@Public
@SuppressWarnings({"rawtypes", "unchecked"})
public class MRCombiner implements Combiner {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
index ba60074..8852363 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
@@ -20,6 +20,7 @@ package org.apache.tez.mapreduce.committer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.mapred.FileOutputCommitter;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
@@ -44,6 +45,11 @@ import org.apache.tez.runtime.api.OutputCommitterContext;
import java.io.IOException;
+/**
+ * Implements the {@link OutputCommitter} and provide Map Reduce compatible
+ * output commit operations for Map Reduce compatible data sinks.
+ */
+@Public
public class MROutputCommitter extends OutputCommitter {
private static final Log LOG = LogFactory.getLog(MROutputCommitter.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
index 8491d66..63dd3bf 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
@@ -25,6 +25,8 @@ import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.security.UserGroupInformation;
@@ -45,6 +47,15 @@ import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.api.events.InputInitializerEvent;
+/**
+ * Implements an {@link InputInitializer} that generates Map Reduce
+ * splits in the App Master. This may utilizes the up to date cluster
+ * information to create an optimal distribution of splits. This is the
+ * recommended {@link InputInitializer} to use when reading Map Reduce
+ * compatible data sources.
+ */
+@Public
+@Evolving
public class MRInputAMSplitGenerator extends InputInitializer {
private boolean sendSerializedEvents;
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
index 4555c43..f23042f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
@@ -23,9 +23,12 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.Vertex;
import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.mapreduce.lib.MRInputUtils;
@@ -42,6 +45,16 @@ import org.apache.tez.runtime.api.events.InputUpdatePayloadEvent;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
+/**
+ * Implements an {@link InputInitializer} that distributes Map Reduce
+ * splits created by the client to tasks in the {@link Vertex}
+ * This can be used when certain reasons (e.g. security) prevent splits
+ * from being produced in the App Master via {@link MRInputAMSplitGenerator}
+ * and splits must be produced at the client. They can still be distributed
+ * intelligently among tasks at runtime using this.
+ */
+@Public
+@Evolving
public class MRInputSplitDistributor extends InputInitializer {
private static final Log LOG = LogFactory.getLog(MRInputSplitDistributor.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
index 2024f85..42c68bd 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java
@@ -33,6 +33,7 @@ import org.apache.tez.mapreduce.hadoop.mapred.MRCounters;
import com.google.common.base.Preconditions;
+@Private
public class Utils {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 0ba6e38..cea05e7 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -25,6 +25,8 @@ import java.util.Vector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -41,6 +43,8 @@ import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
* This class contains helper methods for frameworks which migrate from MapReduce to Tez, and need
* to continue to work with existing MapReduce configurations.
*/
+@Public
+@Evolving
public class MRHelpers {
private static final Log LOG = LogFactory.getLog(MRHelpers.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
index 1a4af9c..2dbdadf 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
@@ -38,7 +38,9 @@ import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -68,6 +70,8 @@ import org.apache.tez.mapreduce.input.MRInput;
import org.apache.tez.mapreduce.input.MRInputLegacy;
import org.apache.tez.mapreduce.protos.MRRuntimeProtos;
+@Public
+@Unstable
public class MRInputHelpers {
private static final Log LOG = LogFactory.getLog(MRInputHelpers.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java
new file mode 100644
index 0000000..73874b8
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.mapreduce.hadoop.mapred;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java
new file mode 100644
index 0000000..c12a8ac
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.mapreduce.hadoop.mapreduce;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java
new file mode 100644
index 0000000..3cc21db
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.mapreduce.hadoop;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index b220ffd..8c8fbb6 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -18,6 +18,7 @@
package org.apache.tez.mapreduce.input;
import javax.annotation.Nullable;
+
import java.io.IOException;
import java.net.URI;
import java.util.List;
@@ -27,6 +28,7 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -75,7 +77,7 @@ import com.google.common.collect.Lists;
*
* This class is not meant to be extended by external projects.
*/
-
+@Public
public class MRInput extends MRInputBase {
/**
@@ -448,6 +450,10 @@ public class MRInput extends MRInputBase {
LOG.info("Initialzed MRInput: " + getContext().getSourceVertexName());
}
+ /**
+ * Returns a {@link KeyValueReader} that can be used to read
+ * Map Reduce compatible key value data
+ */
@Override
public KeyValueReader getReader() throws IOException {
Preconditions
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
index f3e414c..b360840 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
@@ -27,8 +27,11 @@ import java.util.concurrent.locks.ReentrantLock;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.mapred.JobConf;
import org.apache.tez.mapreduce.input.base.MRInputBase;
import org.apache.tez.mapreduce.lib.MRInputUtils;
@@ -42,6 +45,8 @@ import org.apache.tez.runtime.api.InputContext;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.library.api.KeyValueReader;
+@Public
+@Evolving
public class MultiMRInput extends MRInputBase {
private static final Log LOG = LogFactory.getLog(MultiMRInput.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java
new file mode 100644
index 0000000..04a9500
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.mapreduce.lib;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
index 53395ad..d51a8cc 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
@@ -19,6 +19,7 @@
package org.apache.tez.mapreduce.output;
import javax.annotation.Nullable;
+
import java.io.IOException;
import java.text.NumberFormat;
import java.util.Collections;
@@ -26,8 +27,10 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.base.Preconditions;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -59,10 +62,20 @@ import org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl;
import org.apache.tez.mapreduce.processor.MRTaskReporter;
import org.apache.tez.runtime.api.AbstractLogicalOutput;
import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.Output;
import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.library.api.KeyValueWriter;
-
+/**
+ * {@link MROutput} is an {@link Output} which allows key/values pairs
+ * to be written by a processor.
+ *
+ * It is compatible with all standard Apache Hadoop MapReduce
+ * OutputFormat implementations.
+ *
+ * This class is not meant to be extended by external projects.
+ */
+@Public
public class MROutput extends AbstractLogicalOutput {
/**
@@ -428,6 +441,9 @@ public class MROutput extends AbstractLogicalOutput {
"-" + taskNumberFormat.format(getContext().getTaskIndex());
}
+ /**
+ * Get a key value write to write Map Reduce compatible output
+ */
@Override
public KeyValueWriter getWriter() throws IOException {
return new KeyValueWriter() {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java
index 3820b65..768680e 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java
@@ -18,10 +18,12 @@
package org.apache.tez.mapreduce.output;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.tez.runtime.api.OutputContext;
+@Private
public class MROutputLegacy extends MROutput {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java
index 90db734..cf16c7c 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java
@@ -20,14 +20,21 @@ package org.apache.tez.mapreduce.partition;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.runtime.library.api.Partitioner;
import org.apache.tez.runtime.library.common.ConfigUtils;
+/**
+ * Provides an implementation of {@link Partitioner} that is compatible
+ * with Map Reduce partitioners.
+ */
@SuppressWarnings({ "rawtypes", "unchecked" })
+@Public
public class MRPartitioner implements org.apache.tez.runtime.library.api.Partitioner {
static final Log LOG = LogFactory.getLog(MRPartitioner.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index 210fa8a..2ed1ee4 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -85,6 +85,7 @@ import org.apache.tez.runtime.library.common.Constants;
import org.apache.tez.runtime.library.common.sort.impl.TezRawKeyValueIterator;
@SuppressWarnings("deprecation")
+@Private
public abstract class MRTask extends AbstractLogicalIOProcessor {
static final Log LOG = LogFactory.getLog(MRTask.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java
index 7be2407..3a13d51 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java
@@ -22,13 +22,23 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.Processor;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.library.processor.SimpleProcessor;
import com.google.common.collect.Lists;
+/**
+ * A {@link SimpleProcessor} that provides Map Reduce specific post
+ * processing by calling commit (if needed) on all {@link MROutput}s
+ * connected to this {@link Processor}.
+ */
+@Public
+@Evolving
public abstract class SimpleMRProcessor extends SimpleProcessor {
private static final Log LOG = LogFactory.getLog(SimpleMRProcessor.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
index f0010a0..f5d920f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
@@ -24,6 +24,7 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
@@ -52,6 +53,7 @@ import org.apache.tez.runtime.library.api.KeyValueWriter;
import org.apache.tez.runtime.library.output.OnFileSortedOutput;
@SuppressWarnings({ "unchecked", "rawtypes" })
+@Private
public class MapProcessor extends MRTask{
private static final Log LOG = LogFactory.getLog(MapProcessor.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index 99fbdb6..3d94a58 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.mapred.Counters.Counter;
@@ -53,7 +54,7 @@ import org.apache.tez.runtime.library.common.sort.impl.TezRawKeyValueIterator;
import org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy;
import org.apache.tez.runtime.library.output.OnFileSortedOutput;
-
+@Private
@SuppressWarnings({ "unchecked", "rawtypes" })
public class ReduceProcessor extends MRTask {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java
new file mode 100644
index 0000000..3cf4e9c
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.runtime.library.common.sort.impl.dflt;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java
new file mode 100644
index 0000000..2376c99
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.runtime.library.common.writers;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileSortedOutputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileSortedOutputConfigurer.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileSortedOutputConfigurer.java
index 94de0b1..8ace60c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileSortedOutputConfigurer.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileSortedOutputConfigurer.java
@@ -43,7 +43,7 @@ import org.apache.tez.runtime.library.output.OnFileSortedOutput;
@InterfaceAudience.Public
@InterfaceStability.Evolving
/**
- * Configure {@link org.apache.tez.runtime.library.output.OnFileSortedOutput. </p>
+ * Configure {@link org.apache.tez.runtime.library.output.OnFileSortedOutput} </p>
*
* Values will be picked up from tez-site if not specified, otherwise defaults from
* {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used.
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedKVOutputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedKVOutputConfigurer.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedKVOutputConfigurer.java
index 7bfc0cc..42634eb 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedKVOutputConfigurer.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedKVOutputConfigurer.java
@@ -42,7 +42,7 @@ import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
@InterfaceAudience.Public
@InterfaceStability.Evolving
/**
- * Configure {@link org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput. </p>
+ * Configure {@link org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput} </p>
*
* Values will be picked up from tez-site if not specified, otherwise defaults from
* {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used.
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedPartitionedKVOutputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedPartitionedKVOutputConfigurer.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedPartitionedKVOutputConfigurer.java
index 664f346..68a522c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedPartitionedKVOutputConfigurer.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OnFileUnorderedPartitionedKVOutputConfigurer.java
@@ -42,7 +42,7 @@ import org.apache.tez.runtime.library.output.OnFileUnorderedPartitionedKVOutput;
@InterfaceAudience.Public
@InterfaceStability.Evolving
/**
- * Configure {@link org.apache.tez.runtime.library.output.OnFileUnorderedPartitionedKVOutput. </p>
+ * Configure {@link org.apache.tez.runtime.library.output.OnFileUnorderedPartitionedKVOutput} </p>
*
* Values will be picked up from tez-site if not specified, otherwise defaults from
* {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used.
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledMergedInputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledMergedInputConfigurer.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledMergedInputConfigurer.java
index 4a1a8af..5175cd0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledMergedInputConfigurer.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledMergedInputConfigurer.java
@@ -43,7 +43,7 @@ import org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy;
@InterfaceAudience.Public
@InterfaceStability.Evolving
/**
- * Configure {@link org.apache.tez.runtime.library.input.ShuffledMergedInput. </p>
+ * Configure {@link org.apache.tez.runtime.library.input.ShuffledMergedInput} </p>
*
* Values will be picked up from tez-site if not specified, otherwise defaults from
* {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used.
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledUnorderedKVInputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledUnorderedKVInputConfigurer.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledUnorderedKVInputConfigurer.java
index ad57ed2..628c40b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledUnorderedKVInputConfigurer.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/ShuffledUnorderedKVInputConfigurer.java
@@ -42,7 +42,7 @@ import org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput;
@InterfaceAudience.Public
@InterfaceStability.Evolving
/**
- * Configure {@link org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput. </p>
+ * Configure {@link org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput} </p>
*
* Values will be picked up from tez-site if not specified, otherwise defaults from
* {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used.
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java
index f8d4215..e875240 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java
@@ -21,6 +21,7 @@ package org.apache.tez.runtime.library.input;
import java.io.IOException;
import java.util.List;
+import org.apache.tez.dag.api.GroupInputEdge;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.runtime.api.Input;
import org.apache.tez.runtime.api.MergedLogicalInput;
@@ -28,6 +29,11 @@ import org.apache.tez.runtime.api.Reader;
import org.apache.tez.runtime.api.MergedInputContext;
import org.apache.tez.runtime.library.api.KeyValueReader;
+/**
+ * Implements a {@link MergedLogicalInput} that merges the incoming inputs
+ * (e.g. from a {@link GroupInputEdge} and provide a unified view of the
+ * input. It concatenates all the inputs to provide a unified view
+ */
public class ConcatenatedMergedKeyValueInput extends MergedLogicalInput {
public ConcatenatedMergedKeyValueInput(MergedInputContext context,
@@ -71,9 +77,13 @@ public class ConcatenatedMergedKeyValueInput extends MergedLogicalInput {
}
}
-
+
+ /**
+ * Provides a {@link KeyValueReader} that iterates over the
+ * concatenated input data
+ */
@Override
- public Reader getReader() throws Exception {
+ public KeyValueReader getReader() throws Exception {
return new ConcatenatedMergedKeyValueReader();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java
index 3d5fed1..7a57240 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java
@@ -21,6 +21,7 @@ package org.apache.tez.runtime.library.input;
import java.io.IOException;
import java.util.List;
+import org.apache.tez.dag.api.GroupInputEdge;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.runtime.api.Input;
import org.apache.tez.runtime.api.MergedLogicalInput;
@@ -28,6 +29,12 @@ import org.apache.tez.runtime.api.Reader;
import org.apache.tez.runtime.api.MergedInputContext;
import org.apache.tez.runtime.library.api.KeyValuesReader;
+/**
+ * Implements a {@link MergedLogicalInput} that merges the incoming inputs
+ * (e.g. from a {@link GroupInputEdge} and provide a unified view of the
+ * input. It concatenates all the inputs to provide a unified view
+ */
+
public class ConcatenatedMergedKeyValuesInput extends MergedLogicalInput {
public ConcatenatedMergedKeyValuesInput(MergedInputContext context,
@@ -71,9 +78,13 @@ public class ConcatenatedMergedKeyValuesInput extends MergedLogicalInput {
}
}
-
+
+ /**
+ * Provides a {@link KeyValuesReader} that iterates over the
+ * concatenated input data
+ */
@Override
- public Reader getReader() throws Exception {
+ public KeyValuesReader getReader() throws Exception {
return new ConcatenatedMergedKeyValuesReader();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledMergedInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledMergedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledMergedInput.java
index 7bca296..bd8e472 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledMergedInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledMergedInput.java
@@ -53,9 +53,12 @@ import com.google.common.base.Preconditions;
/**
- * <code>ShuffleMergedInput</code> in a {@link AbstractLogicalInput} which shuffles
+ * {@link ShuffledMergedInput} in a {@link AbstractLogicalInput} which shuffles
* intermediate sorted data, merges them and provides key/<values> to the
- * consumer.
+ * consumer. This is typically used to bring one partition of a set of partitioned
+ * distributed data to one consumer. The shuffle operation brings all partitions
+ * to one place. These partitions are assumed to be sorted and are merged sorted to
+ * merge them into a single input view.
*
* The Copy and Merge will be triggered by the initialization - which is handled
* by the Tez framework. Input is not consumable until the Copy and Merge are
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledUnorderedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledUnorderedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledUnorderedKVInput.java
index c4fb228..6070d87 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledUnorderedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ShuffledUnorderedKVInput.java
@@ -51,6 +51,12 @@ import org.apache.tez.runtime.library.shuffle.common.impl.SimpleFetchedInputAllo
import com.google.common.base.Preconditions;
+/**
+ * {@link ShuffledUnorderedKVInput} provides unordered key value input by
+ * bringing together (shuffling) a set of distributed data and providing a
+ * unified view to that data. There are no ordering constraints applied by
+ * this input.
+ */
public class ShuffledUnorderedKVInput extends AbstractLogicalInput {
private static final Log LOG = LogFactory.getLog(ShuffledUnorderedKVInput.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/SortedGroupedMergedInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/SortedGroupedMergedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/SortedGroupedMergedInput.java
index 40db2cb..1accf2a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/SortedGroupedMergedInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/SortedGroupedMergedInput.java
@@ -53,6 +53,9 @@ public class SortedGroupedMergedInput extends MergedLogicalInput {
super(context, inputs);
}
+ /**
+ * Provides an ordered {@link KeyValuesReader}
+ */
@Override
public KeyValuesReader getReader() throws Exception {
return new SortedGroupedMergedKeyValuesReader(getInputs());
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
index 6f41931..2fbd230 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
@@ -48,6 +48,7 @@ import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.api.events.CompositeDataMovementEvent;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
import org.apache.tez.runtime.library.api.KeyValuesWriter;
+import org.apache.tez.runtime.library.api.Partitioner;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
import org.apache.tez.runtime.library.common.sort.impl.ExternalSorter;
@@ -60,8 +61,9 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovem
import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto;
/**
- * <code>OnFileSortedOutput</code> is an {@link AbstractLogicalOutput} which sorts key/value pairs
- * written to it and persists it to a file.
+ * {@link OnFileSortedOutput} is an {@link AbstractLogicalOutput} which sorts
+ * key/value pairs written to it. It also partitions the output based on a
+ * {@link Partitioner}
*/
public class OnFileSortedOutput extends AbstractLogicalOutput {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
index 9db40ed..ae54403 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
@@ -39,6 +39,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.runtime.api.AbstractLogicalOutput;
import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.LogicalOutput;
import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.library.api.KeyValuesWriter;
@@ -52,6 +53,11 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
+/**
+ * {@link OnFileUnorderedKVOutput} is a {@link LogicalOutput} that writes key
+ * value data without applying any ordering or grouping constraints. This can be
+ * used to write raw key value data as is.
+ */
public class OnFileUnorderedKVOutput extends AbstractLogicalOutput {
private static final Log LOG = LogFactory.getLog(OnFileUnorderedKVOutput.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java
index 1570c06..7a26d38 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java
@@ -22,6 +22,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.tez.runtime.library.api.Partitioner;
+/**
+ * Implements a {@link Partitioner} that does hash based partitioning
+ */
@Public
@Evolving
public class HashPartitioner implements Partitioner {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java
index f354ba7..4103078 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java
@@ -18,15 +18,20 @@
package org.apache.tez.runtime.library.processor;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.Processor;
import org.apache.tez.runtime.api.ProcessorContext;
/**
- * Built-in convenience {@link Processor} to be used for pre-warming
+ * Built-in convenience {@link Processor} to be used for pre-warming.
+ * If this is customized by the user then they need to make sure that
+ * the custom class jar is localized for the prewarm vertex and other
+ * vertices that need to take advantage of prewarming
*
*/
@Unstable
+@Public
public class PreWarmProcessor extends SimpleProcessor {
public PreWarmProcessor(ProcessorContext context) {
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java
index d9459d7..725f785 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java
@@ -26,8 +26,15 @@ import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.Processor;
import org.apache.tez.runtime.api.ProcessorContext;
+/**
+ * Implements an {@link AbstractLogicalIOProcessor} and provides empty
+ * implementations of most methods and handles input/output initialization.
+ * This can be used to implement simple {@link Processor}s that dont need to
+ * do event handling etc.
+ */
@Public
@Evolving
public abstract class SimpleProcessor extends AbstractLogicalIOProcessor {
@@ -47,8 +54,19 @@ public abstract class SimpleProcessor extends AbstractLogicalIOProcessor {
postOp();
}
+ /**
+ * Users must implement this method to provide the main
+ * application logic code
+ * @throws Exception
+ */
public abstract void run() throws Exception;
+ /**
+ * Implements input/output initialization. Can be overriden
+ * to implement custom behavior. Called before {@link #run()}
+ * is called.
+ * @throws Exception
+ */
protected void preOp() throws Exception {
if (getInputs() != null) {
for (LogicalInput input : getInputs().values()) {
@@ -62,6 +80,11 @@ public abstract class SimpleProcessor extends AbstractLogicalIOProcessor {
}
}
+ /**
+ * Called after {@link #run()} is called and can be used to
+ * do post-processing like committing output etc
+ * @throws Exception
+ */
protected void postOp() throws Exception {
//No-op
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4a41ccf0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/package-info.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/package-info.java
new file mode 100644
index 0000000..b69472e
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.runtime.library.shuffle.common.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file