You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/08/15 00:57:37 UTC
[1/2] TEZ-1416. tez-api project javadoc/annotations review and clean
up (bikas)
Repository: tez
Updated Branches:
refs/heads/master 9bba71a10 -> 84374adc7
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
index 7cc2197..bda4396 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
@@ -18,14 +18,18 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
- * Represents an input through which a TezProcessor receives data on an edge.
+ * Represents an input through which a {@link Processor} receives data on an edge.
* </p>
*
* This interface has methods which can be used by a {@link org.apache.tez.runtime.api.Processor}
* to control execution of this Input and read data from it.
- *
+ *
+ * Actual implementations are expected to derive from {@link AbstractLogicalInput}
*/
+@Public
public interface Input {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java
index 1961184..fc02878 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java
@@ -18,10 +18,13 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Context handle for the Input to initialize itself.
* This interface is not supposed to be implemented by users
*/
+@Public
public interface InputContext extends TaskContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java
index 25b37b8..7f0a1d9 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java
@@ -22,6 +22,8 @@ package org.apache.tez.runtime.api;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Represents an input through which a TezProcessor receives data on an edge.
@@ -39,7 +41,10 @@ import java.util.List;
* This typically means that the Processor will not block when reading from the
* corresponding Input. This is done via {@link InputContext}.inputIsReady.
* Inputs choose the policy on when they are ready.
+ *
+ * Input implementations are expected to derive from {@link AbstractLogicalInput}
*/
+@Public
public interface InputFrameworkInterface {
/**
* Initializes the <code>Input</code>.
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
index a86dc22..09268d9 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
@@ -20,7 +20,8 @@ package org.apache.tez.runtime.api;
import java.util.List;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.runtime.api.events.InputInitializerEvent;
/**
@@ -29,7 +30,8 @@ import org.apache.tez.runtime.api.events.InputInitializerEvent;
* distribute data across the tasks for the vertex, determine the number of
* tasks at runtime, update the Input payload etc.
*/
-@InterfaceStability.Unstable
+@Unstable
+@Public
public abstract class InputInitializer {
private final InputInitializerContext initializerContext;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java
index 91c4397..e4e15ef 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java
@@ -18,6 +18,7 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -27,6 +28,7 @@ import org.apache.tez.dag.api.UserPayload;
* A context that provides information to the {@link InputInitializer}
*/
@Unstable
+@Public
public interface InputInitializerContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java
index 0b86ae9..f0d166f 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java
@@ -21,6 +21,7 @@ package org.apache.tez.runtime.api;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
@@ -33,6 +34,7 @@ import com.google.common.collect.Lists;
*
*/
@Unstable
+@Public
public class InputSpecUpdate {
private final boolean forAllWorkUnits;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
index cfee97b..0256d1e 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
@@ -18,9 +18,12 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Represents a processor which consumes {@link LogicalInput}s and produces
* {@link LogicalOutput}s
*/
+@Public
public interface LogicalIOProcessor extends Processor {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java
index ff87cec..edae2f8 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java
@@ -20,10 +20,14 @@ package org.apache.tez.runtime.api;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Represents a processor framework interface which consumes {@link LogicalInput}s and produces
- * {@link LogicalOutput}s
+ * {@link LogicalOutput}s.
+ * Users are expected to implements {@link AbstractLogicalIOProcessor}
*/
+@Public
public interface LogicalIOProcessorFrameworkInterface extends ProcessorFrameworkInterface {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java
index 74fc156..9f10d0b 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java
@@ -18,10 +18,16 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* An {@link Input} which handles all incoming physical connections on an
* edge. A {@link LogicalIOProcessor} sees a single Logical Input per incoming
- * edge.
+ * edge. It provides a unified view that hides the details of the actual physical
+ * inputs
+ *
+ * User implementations are expected to derive from {@link AbstractLogicalInput}
*/
+@Public
public interface LogicalInput extends Input {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java
index 33a0f40..191dcd0 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java
@@ -20,5 +20,13 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
+/**
+ * Defines the framework facing interfact for a {@link LogicalInput}
+ *
+ * User implementations are expected to derive from {@link AbstractLogicalInput}
+ */
+@Public
public interface LogicalInputFrameworkInterface extends InputFrameworkInterface {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java
index 90b64a2..2922189 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java
@@ -18,11 +18,16 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* An @link {@link Output} which handles all outgoing physical connections on an
* edge. A {@link LogicalIOProcessor} sees a single Logical Output per outgoing
- * edge.
+ * edge. It's expected to hide the details of output partitioning and physical
+ * outputs from the {@link Processor}
+ * Users are expected to derive from {@link AbstractLogicalOutput}
*/
+@Public
public interface LogicalOutput extends Output {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutputFrameworkInterface.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutputFrameworkInterface.java
index b5855de..f513017 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutputFrameworkInterface.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutputFrameworkInterface.java
@@ -20,5 +20,12 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
+/**
+ * Framework facing interface for the {@link LogicalOutput}
+ * Users are expected to derive from {@link AbstractLogicalOutput}
+ */
+@Public
public interface LogicalOutputFrameworkInterface extends OutputFrameworkInterface {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java
index 5556071..1f304b3 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java
@@ -18,6 +18,7 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
@@ -28,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
*
*/
@Unstable
+@Public
public abstract class MemoryUpdateCallback {
public abstract void memoryAssigned(long assignedSize);
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java
index 410a801..41c519b 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java
@@ -18,12 +18,14 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.dag.api.UserPayload;
/**
* Context for {@link MergedLogicalInput}
* This interface is not supposed to be implemented by users
*/
+@Public
public interface MergedInputContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java
index fd698bd..dedc902 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java
@@ -22,6 +22,9 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
/**
* A LogicalInput that is used to merge the data from multiple inputs and provide a
* single <code>Reader</code> to read that data.
@@ -34,6 +37,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
* take care of initializing and closing the Input after a {@link Processor} completes. </p>
* <p/>
*/
+@Public
+@Evolving
public abstract class MergedLogicalInput implements LogicalInput {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java
new file mode 100644
index 0000000..a280942
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * A simple shared object registry to cache objects in the memory of the
+ * container running the task. This registry can be used to avoid re-creating
+ * objects like common hash-tables, dictionaries etc. and provide performance
+ * improvements. The cached objects have different life-cycles. If a task adds
+ * an object to the cache with Vertex life-cycle then that object is in the
+ * cache while the Vertex (to which the task belongs) is running. DAG life-cycle
+ * is while the DAG (to which that task belongs) is running. Session life-cycle
+ * is while the session (to which that task belongs) is running. <br>
+ * This interface is not supposed to be implemented by users.
+ */
+@Public
+@Evolving
+public interface ObjectRegistry {
+
+ /**
+ * Insert or update object into the registry with Vertex life-cycle. This will
+ * remove an object associated with the same key with a different life-cycle
+ * as there is only one instance of an Object stored for a given key
+ * irrespective of the life-cycle attached to the Object. The object may stay
+ * in the cache while the Vertex (to which the task belongs) is running.
+ *
+ * @param key
+ * Key to identify the Object
+ * @param value
+ * Object to be inserted
+ * @return Previous Object associated with the key attached if present else
+ * null. Could return the same object if the object was associated
+ * with the same key for a different life-cycle.
+ */
+ public Object cacheForVertex(String key, Object value);
+
+ /**
+ * Insert or update object into the registry with DAG life-cycle. This will
+ * remove an object associated with the same key with a different life-cycle
+ * as there is only one instance of an Object stored for a given key
+ * irrespective of the life-cycle attached to the Object. The object may stay
+ * in the cache while the DAG (to which the task belongs) is running.
+ *
+ * @param key
+ * Key to identify the Object
+ * @param value
+ * Object to be inserted
+ * @return Previous Object associated with the key attached if present else
+ * null. Could return the same object if the object was associated
+ * with the same key for a different life-cycle.
+ */
+ public Object cacheForDAG(String key, Object value);
+
+ /**
+ * Insert or update object into the registry with Session life-cycle. This
+ * will remove an object associated with the same key with a different
+ * life-cycle as there is only one instance of an Object stored for a given
+ * key irrespective of the life-cycle attached to the Object. The object may stay
+ * in the cache while the Session (to which the task belongs) is running.
+ *
+ * @param key
+ * Key to identify the Object
+ * @param value
+ * Object to be inserted
+ * @return Previous Object associated with the key attached if present else
+ * null. Could return the same object if the object was associated
+ * with the same key for a different life-cycle.
+ */
+ public Object cacheForSession(String key, Object value);
+
+ /**
+ * Return the object associated with the provided key
+ * @param key Key to find object
+ * @return Object if found else null
+ */
+ public Object get(String key);
+
+ /**
+ * Delete the object associated with the provided key
+ * @param key Key to find object
+ * @return True if an object was found and removed
+ */
+ public boolean delete(String key);
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java
index 77b1609..9125c0d 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java
@@ -18,14 +18,19 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Represents an Output through which a TezProcessor writes information to an edge.
* </p>
*
* This interface has methods which can be used by a {@link org.apache.tez.runtime.api.Processor}
* to control execution of this Output and write data to it.
+ *
+ * Users are expected to derive from {@link AbstractLogicalOutput}
*
*/
+@Public
public interface Output {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
index 0b40695..684772a 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
@@ -19,13 +19,15 @@
package org.apache.tez.runtime.api;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.dag.api.client.VertexStatus;
/**
* OutputCommitter to "finalize" the output and make it user-visible if needed.
* The OutputCommitter is allowed only on a terminal Output.
*/
-@InterfaceStability.Unstable
+@InterfaceStability.Evolving
+@Public
public abstract class OutputCommitter {
private final OutputCommitterContext committerContext;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
index 91b4b63..d254a6a 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
@@ -18,7 +18,7 @@
package org.apache.tez.runtime.api;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.dag.api.UserPayload;
@@ -27,7 +27,7 @@ import org.apache.tez.dag.api.UserPayload;
* information that it needs. This interface is not supposed to be implemented
* by users
*/
-@Unstable
+@Public
public interface OutputCommitterContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java
index d5796e2..a0c7194 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java
@@ -18,10 +18,13 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Context handle for the Output to initialize itself.
* This interface is not supposed to be implemented by users
*/
+@Public
public interface OutputContext extends TaskContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java
index 7f64f3e..617deee 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java
@@ -22,6 +22,8 @@ package org.apache.tez.runtime.api;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Represents the Tez framework part of an {@link org.apache.tez.runtime.api.Output}.
* <p/>
@@ -32,8 +34,11 @@ import java.util.List;
* During initialization, Outputs must specify an initial memory requirement via
* {@link OutputContext}.requestInitialMemory
* <p/>
+ *
+ * Users are expected to derive from {@link AbstractLogicalOutput}
*
*/
+@Public
public interface OutputFrameworkInterface {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java
index ca9100a..32d786e 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java
@@ -18,9 +18,15 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* {@link Processor} represents the <em>Tez</em> entity responsible for
- * consuming {@link Input} and producing {@link Output}.
+ * consuming {@link Input} and producing {@link Output}. It defines the
+ * application logic that processes and transforms the data.
+ *
+ * Users are expected to derive from {@link AbstractLogicalIOProcessor}
*/
+@Public
public interface Processor {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java
index 6a28e1d..2ac6e4c 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java
@@ -21,10 +21,13 @@ package org.apache.tez.runtime.api;
import java.io.IOException;
import java.util.Collection;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Context handle for the Processor to initialize itself.
* This interface is not supposed to be implemented by users
*/
+@Public
public interface ProcessorContext extends TaskContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java
index 7854280..317ba39 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java
@@ -20,13 +20,17 @@ package org.apache.tez.runtime.api;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Represents the Tez framework part of an {@link org.apache.tez.runtime.api.Processor}.
* <p/>
*
* This interface has methods which are used by the Tez framework to control the Processor.
* <p/>
+ * Users are expected to derive from {@link AbstractLogicalIOProcessor}
*/
+@Public
public interface ProcessorFrameworkInterface {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java
index d5602c6..9fc5094 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java
@@ -18,9 +18,16 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
/**
* A <code>Reader</code> represents the data being read in an {@link Input}
+ * It encapsulates the data type etc of the data being provided by the {@link Input}
+ * E.g. There can be Key-Value readers, byte readers etc.
*/
+@Public
+@Evolving
public abstract class Reader {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java
index 8e00a09..38be678 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java
@@ -23,21 +23,18 @@ import java.util.List;
import javax.annotation.Nullable;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
/**
* Base interface for Context classes used to initialize the Input, Output
* and Processor instances.
* This interface is not supposed to be implemented by users
*/
+@Public
public interface TaskContext {
- // Scale the maximum events we fetch per RPC call to mitigate OOM issues
- // on the ApplicationMaster when a thundering herd of reducers fetch events
- // This should not be necessary after HADOOP-8942. TEZ-1398
-
/**
* Get the {@link ApplicationId} for the running app
* @return the {@link ApplicationId}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java
index a9337d5..f418b70 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java
@@ -18,9 +18,15 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
/**
* A <code>Writer</code> represents the data being written by an {@link Output}
- */
+ * It encapsulates the data type etc of the data being provided by the {@link Output}
+ * E.g. There can be Key-Value writers, byte writers etc. */
+@Public
+@Evolving
public abstract class Writer {
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
index bfefd66..e7322c6 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
@@ -20,6 +20,7 @@ package org.apache.tez.runtime.api.events;
import java.util.Iterator;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.runtime.api.Event;
/**
@@ -34,6 +35,7 @@ import org.apache.tez.runtime.api.Event;
* the Physical Outputs that it generates.
*
*/
+@Public
public class CompositeDataMovementEvent extends Event {
protected final int sourceIndexStart;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
index 9865117..b737430 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
@@ -19,6 +19,7 @@
package org.apache.tez.runtime.api.events;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.runtime.api.Event;
/**
@@ -27,6 +28,7 @@ import org.apache.tez.runtime.api.Event;
* ( such as URI for file-based output data, port info in case of
* streaming-based data transfers ) to the Input on the destination vertex.
*/
+@Public
public final class DataMovementEvent extends Event {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
index 5d05e5d..b7a0e01 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
@@ -18,24 +18,30 @@
package org.apache.tez.runtime.api.events;
-import java.util.List;
-
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.InputInitializer;
import org.apache.tez.runtime.api.InputSpecUpdate;
+/**
+ * An event typically sent by the {@link InputInitializer} of a vertex
+ * to configure the tasks of the vertex. It could change the task
+ * placement hints or input specification for the inputs of the tasks
+ */
@Unstable
+@Public
public class InputConfigureVertexTasksEvent extends Event {
private final int numTasks;
- private final List<TaskLocationHint> taskLocationHints;
+ private final VertexLocationHint locationHint;
private final InputSpecUpdate inputSpecUpdate;
- public InputConfigureVertexTasksEvent(int numTasks, List<TaskLocationHint> locationHints,
+ public InputConfigureVertexTasksEvent(int numTasks, VertexLocationHint locationHint,
InputSpecUpdate inputSpecUpdate) {
this.numTasks = numTasks;
- this.taskLocationHints = locationHints;
+ this.locationHint = locationHint;
this.inputSpecUpdate = inputSpecUpdate;
}
@@ -43,8 +49,8 @@ public class InputConfigureVertexTasksEvent extends Event {
return numTasks;
}
- public List<TaskLocationHint> getTaskLocationHints() {
- return taskLocationHints;
+ public VertexLocationHint getLocationHint() {
+ return locationHint;
}
public InputSpecUpdate getInputSpecUpdate() {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
index d7e46a3..8e080be 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.runtime.api.events;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.runtime.api.Event;
@@ -39,6 +40,7 @@ import org.apache.tez.runtime.api.InputInitializer;
* contain the payload in a serialized form.
*/
@Unstable
+@Public
public final class InputDataInformationEvent extends Event {
private final int sourceIndex;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
index 2dc1adf..9607eda 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
@@ -27,6 +27,7 @@ import org.apache.tez.runtime.api.Event;
* source.
* Users are not expected to send this event.
*/
+@Private
public class InputFailedEvent extends Event{
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
index 3d1ed11..fbe79d8 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
@@ -21,14 +21,18 @@
package org.apache.tez.runtime.api.events;
import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.InputInitializer;
import org.apache.tez.runtime.api.Event;
/**
* An event that is routed to the specified {@link InputInitializer}.
+ * This can be used to send information/metadata to the {@link InputInitializer}
*/
@Unstable
+@Public
public class InputInitializerEvent extends Event {
private String targetVertexName;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
index 612a1e3..e838173 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.runtime.api.events;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.runtime.api.Event;
/**
@@ -25,6 +26,7 @@ import org.apache.tez.runtime.api.Event;
* This is not necessarily a fatal event - it's an indication to the AM to retry
* source data generation.
*/
+@Public
public final class InputReadErrorEvent extends Event {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
index a080659..0232607 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.runtime.api.events;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.InputInitializer;
@@ -32,6 +33,7 @@ import com.google.common.base.Preconditions;
* before {@link InputDataInformationEvent}s
*/
@Unstable
+@Public
public class InputUpdatePayloadEvent extends Event {
private final byte[] userPayload;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
index e95db46..c2c6129 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.runtime.api.events;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.Event;
@@ -29,6 +30,7 @@ import com.google.common.base.Preconditions;
* automatic plan recofigurations based on observed statistics
*/
@Unstable
+@Public
public class VertexManagerEvent extends Event {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistry.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistry.java b/tez-api/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistry.java
deleted file mode 100644
index 084c83c..0000000
--- a/tez-api/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistry.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime.common.objectregistry;
-
-/**
- * A simple shared object registry to cache objects in the memory of the
- * container running the task. This registry can be used to avoid re-creating
- * objects like common hash-tables, dictionaries etc. and provide performance
- * improvements. The cached objects have different life-cycles. If a task adds
- * an object to the cache with Vertex life-cycle then that object is in the
- * cache while the Vertex (to which the task belongs) is running. DAG life-cycle
- * is while the DAG (to which that task belongs) is running. Session life-cycle
- * is while the session (to which that task belongs) is running. <br>
- * This interface is not supposed to be implemented by users.
- */
-public interface ObjectRegistry {
-
- /**
- * Insert or update object into the registry with Vertex life-cycle. This will
- * remove an object associated with the same key with a different life-cycle
- * as there is only one instance of an Object stored for a given key
- * irrespective of the life-cycle attached to the Object. The object may stay
- * in the cache while the Vertex (to which the task belongs) is running.
- *
- * @param key
- * Key to identify the Object
- * @param value
- * Object to be inserted
- * @return Previous Object associated with the key attached if present else
- * null. Could return the same object if the object was associated
- * with the same key for a different life-cycle.
- */
- public Object cacheForVertex(String key, Object value);
-
- /**
- * Insert or update object into the registry with DAG life-cycle. This will
- * remove an object associated with the same key with a different life-cycle
- * as there is only one instance of an Object stored for a given key
- * irrespective of the life-cycle attached to the Object. The object may stay
- * in the cache while the DAG (to which the task belongs) is running.
- *
- * @param key
- * Key to identify the Object
- * @param value
- * Object to be inserted
- * @return Previous Object associated with the key attached if present else
- * null. Could return the same object if the object was associated
- * with the same key for a different life-cycle.
- */
- public Object cacheForDAG(String key, Object value);
-
- /**
- * Insert or update object into the registry with Session life-cycle. This
- * will remove an object associated with the same key with a different
- * life-cycle as there is only one instance of an Object stored for a given
- * key irrespective of the life-cycle attached to the Object. The object may stay
- * in the cache while the Session (to which the task belongs) is running.
- *
- * @param key
- * Key to identify the Object
- * @param value
- * Object to be inserted
- * @return Previous Object associated with the key attached if present else
- * null. Could return the same object if the object was associated
- * with the same key for a different life-cycle.
- */
- public Object cacheForSession(String key, Object value);
-
- /**
- * Return the object associated with the provided key
- * @param key Key to find object
- * @return Object if found else null
- */
- public Object get(String key);
-
- /**
- * Delete the object associated with the provided key
- * @param key Key to find object
- * @return True if an object was found and removed
- */
- public boolean delete(String key);
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index a75c107..50365ad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -88,15 +88,14 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.common.LogUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezConverterUtils;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.common.counters.Limits;
-import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.TokenCache;
import org.apache.tez.dag.api.DagTypeConverters;
-import org.apache.tez.dag.api.DuplicateDAGName;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
@@ -687,8 +686,8 @@ public class DAGAppMaster extends AbstractService {
LOG.warn("Failed to generate json for DAG", e);
}
- if (dagConf.getBoolean(TezConfiguration.TEZ_GENERATE_DAG_VIZ,
- TezConfiguration.TEZ_GENERATE_DAG_VIZ_DEFAULT)) {
+ if (dagConf.getBoolean(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS,
+ TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT)) {
generateDAGVizFile(dagId, dagPB);
}
@@ -1802,7 +1801,7 @@ public class DAGAppMaster extends AbstractService {
this.state = DAGAppMasterState.RUNNING;
this.appName = dagPlan.getName();
if (dagNames.contains(dagPlan.getName())) {
- throw new DuplicateDAGName("Duplicate dag name '" + dagPlan.getName() + "'");
+ throw new TezException("Duplicate dag name '" + dagPlan.getName() + "'");
}
dagNames.add(dagPlan.getName());
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
index 73f136d..38f846d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
@@ -24,7 +24,6 @@ import java.util.Map;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
@@ -92,7 +91,7 @@ public class RootInputVertexManager extends VertexManagerPlugin {
cEvent.getInputSpecUpdate() == null ? InputSpecUpdate
.getDefaultSinglePhysicalInputSpecUpdate() : cEvent.getInputSpecUpdate());
getContext().setVertexParallelism(cEvent.getNumTasks(),
- new VertexLocationHint(cEvent.getTaskLocationHints()), null, rootInputSpecUpdate);
+ cEvent.getLocationHint(), null, rootInputSpecUpdate);
}
if (event instanceof InputUpdatePayloadEvent) {
// No tasks should have been started yet. Checked by initial state check.
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-dag/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-dag/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
index f1ddc0b..7fa3798 100644
--- a/tez-dag/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ b/tez-dag/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
@@ -38,11 +38,11 @@ import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.impl.EventMetaData;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.TezUmbilical;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import com.google.common.collect.Multimap;
import com.google.common.util.concurrent.ListenableFuture;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index c99458d..ebefe91 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -3254,7 +3254,7 @@ public class TestVertexImpl {
List<Event> events = Lists.newArrayListWithCapacity(targetTasks + 1);
InputConfigureVertexTasksEvent configEvent = new InputConfigureVertexTasksEvent(
- targetTasks, locationHints, null);
+ targetTasks, new VertexLocationHint(locationHints), null);
events.add(configEvent);
for (int i = 0; i < targetTasks; i++) {
InputDataInformationEvent diEvent = new InputDataInformationEvent(i, null);
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 40319de..35a7c76 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -45,8 +45,8 @@ import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.library.api.KeyValueReader;
import org.apache.tez.runtime.library.api.KeyValueWriter;
import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
index e418306..8491d66 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.mapreduce.hadoop.InputSplitInfoMem;
import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
@@ -120,7 +121,7 @@ public class MRInputAMSplitGenerator extends InputInitializer {
.getNumTasks() + 1);
InputConfigureVertexTasksEvent configureVertexEvent = new InputConfigureVertexTasksEvent(
- inputSplitInfo.getNumTasks(), inputSplitInfo.getTaskLocationHints(),
+ inputSplitInfo.getNumTasks(), new VertexLocationHint(inputSplitInfo.getTaskLocationHints()),
InputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate());
events.add(configureVertexEvent);
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
index a11a7e1..64018fc 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
@@ -57,6 +57,7 @@ import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.api.LogicalOutput;
import org.apache.tez.runtime.api.LogicalOutputFrameworkInterface;
import org.apache.tez.runtime.api.MergedLogicalInput;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.Output;
import org.apache.tez.runtime.api.OutputFrameworkInterface;
import org.apache.tez.runtime.api.Processor;
@@ -76,7 +77,6 @@ import org.apache.tez.runtime.api.impl.TezMergedInputContextImpl;
import org.apache.tez.runtime.api.impl.TezOutputContextImpl;
import org.apache.tez.runtime.api.impl.TezProcessorContextImpl;
import org.apache.tez.runtime.api.impl.TezUmbilical;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.common.resources.MemoryDistributor;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
index 4a0b646..3945d4b 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
@@ -41,8 +41,8 @@ import org.apache.tez.runtime.RuntimeTask;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.api.InputContext;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.common.resources.MemoryDistributor;
public class TezInputContextImpl extends TezTaskContextImpl
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
index 8f1fe85..3e88080 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
@@ -38,9 +38,9 @@ import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.RuntimeTask;
import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.common.resources.MemoryDistributor;
public class TezOutputContextImpl extends TezTaskContextImpl
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
index eefa337..2e8e622 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
@@ -40,9 +40,9 @@ import org.apache.tez.runtime.InputReadyTracker;
import org.apache.tez.runtime.RuntimeTask;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.Input;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.common.resources.MemoryDistributor;
public class TezProcessorContextImpl extends TezTaskContextImpl implements ProcessorContext {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java
index 59db901..8ab4f20 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java
@@ -36,8 +36,8 @@ import org.apache.tez.dag.api.EntityDescriptor;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.RuntimeTask;
import org.apache.tez.runtime.api.MemoryUpdateCallback;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.TaskContext;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.common.resources.MemoryDistributor;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java
index 2adb97a..eccb71c 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java
@@ -24,6 +24,8 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.tez.runtime.api.ObjectRegistry;
+
public class ObjectRegistryImpl implements ObjectRegistry {
public enum ObjectLifeCycle {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java
index 7cfcd10..ad229ce 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java
@@ -18,7 +18,7 @@
package org.apache.tez.runtime.common.objectregistry;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
+import org.apache.tez.runtime.api.ObjectRegistry;
import org.junit.Assert;
import org.junit.Test;
[2/2] git commit: TEZ-1416. tez-api project javadoc/annotations
review and clean up (bikas)
Posted by bi...@apache.org.
TEZ-1416. tez-api project javadoc/annotations review and clean up (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/84374adc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/84374adc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/84374adc
Branch: refs/heads/master
Commit: 84374adc7e17c018bf8309b527e3f2f572901ebf
Parents: 9bba71a
Author: Bikas Saha <bi...@apache.org>
Authored: Thu Aug 14 15:57:38 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu Aug 14 15:57:38 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/client/FrameworkClient.java | 2 +
.../org/apache/tez/client/PreWarmVertex.java | 40 +++++++
.../apache/tez/client/TezAppMasterStatus.java | 6 ++
.../java/org/apache/tez/client/TezClient.java | 5 +
.../org/apache/tez/client/TezClientUtils.java | 8 +-
.../org/apache/tez/client/TezYarnClient.java | 2 +
.../java/org/apache/tez/common/LogUtils.java | 42 ++++++++
.../tez/common/counters/AbstractCounter.java | 2 +-
.../common/counters/AbstractCounterGroup.java | 2 +-
.../tez/common/counters/AbstractCounters.java | 7 +-
.../tez/common/counters/CounterGroup.java | 8 +-
.../tez/common/counters/CounterGroupBase.java | 8 +-
.../tez/common/counters/FileSystemCounter.java | 4 +-
.../apache/tez/common/counters/JobCounter.java | 6 +-
.../tez/common/counters/ResourceBundles.java | 3 +
.../apache/tez/common/counters/TaskCounter.java | 5 +-
.../apache/tez/common/counters/TezCounter.java | 12 +--
.../apache/tez/common/counters/TezCounters.java | 10 +-
.../org/apache/tez/common/impl/LogUtils.java | 42 --------
.../common/security/DAGClientSecurityInfo.java | 2 +
.../main/java/org/apache/tez/dag/api/DAG.java | 4 +-
.../tez/dag/api/DAGSubmissionTimedOut.java | 3 +
.../apache/tez/dag/api/DagTypeConverters.java | 2 +
.../apache/tez/dag/api/DataSinkDescriptor.java | 2 +
.../tez/dag/api/DataSourceDescriptor.java | 2 +
.../apache/tez/dag/api/DuplicateDAGName.java | 41 --------
.../main/java/org/apache/tez/dag/api/Edge.java | 22 +++-
.../apache/tez/dag/api/EdgeManagerPlugin.java | 6 +-
.../tez/dag/api/EdgeManagerPluginContext.java | 5 +
.../dag/api/EdgeManagerPluginDescriptor.java | 6 ++
.../org/apache/tez/dag/api/EdgeProperty.java | 33 +++++-
.../apache/tez/dag/api/EntityDescriptor.java | 9 ++
.../org/apache/tez/dag/api/GroupInputEdge.java | 36 +++++--
.../org/apache/tez/dag/api/InputDescriptor.java | 6 ++
.../tez/dag/api/InputInitializerDescriptor.java | 6 ++
.../tez/dag/api/OutputCommitterDescriptor.java | 6 ++
.../apache/tez/dag/api/OutputDescriptor.java | 6 ++
.../apache/tez/dag/api/ProcessorDescriptor.java | 6 ++
.../apache/tez/dag/api/SessionNotRunning.java | 3 +
.../apache/tez/dag/api/TezConfiguration.java | 32 +++---
.../org/apache/tez/dag/api/TezConstants.java | 4 +
.../org/apache/tez/dag/api/TezException.java | 5 +-
.../tez/dag/api/TezUncheckedException.java | 5 +-
.../org/apache/tez/dag/api/UserPayload.java | 3 +
.../java/org/apache/tez/dag/api/Vertex.java | 8 ++
.../org/apache/tez/dag/api/VertexGroup.java | 3 +
.../apache/tez/dag/api/VertexLocationHint.java | 7 ++
.../apache/tez/dag/api/VertexManagerPlugin.java | 6 +-
.../tez/dag/api/VertexManagerPluginContext.java | 4 +-
.../dag/api/VertexManagerPluginDescriptor.java | 5 +
.../apache/tez/dag/api/client/DAGClient.java | 6 +-
.../apache/tez/dag/api/client/DAGStatus.java | 6 ++
.../org/apache/tez/dag/api/client/Progress.java | 7 ++
.../tez/dag/api/client/StatusGetOpts.java | 5 +
.../apache/tez/dag/api/client/VertexStatus.java | 7 ++
.../rpc/DAGClientAMProtocolBlockingPB.java | 2 +
.../dag/api/client/rpc/DAGClientRPCImpl.java | 2 +
.../runtime/api/AbstractLogicalIOProcessor.java | 9 +-
.../tez/runtime/api/AbstractLogicalInput.java | 3 +
.../tez/runtime/api/AbstractLogicalOutput.java | 5 +-
.../java/org/apache/tez/runtime/api/Event.java | 4 +
.../java/org/apache/tez/runtime/api/Input.java | 8 +-
.../apache/tez/runtime/api/InputContext.java | 3 +
.../runtime/api/InputFrameworkInterface.java | 5 +
.../tez/runtime/api/InputInitializer.java | 6 +-
.../runtime/api/InputInitializerContext.java | 2 +
.../apache/tez/runtime/api/InputSpecUpdate.java | 2 +
.../tez/runtime/api/LogicalIOProcessor.java | 3 +
.../LogicalIOProcessorFrameworkInterface.java | 6 +-
.../apache/tez/runtime/api/LogicalInput.java | 8 +-
.../api/LogicalInputFrameworkInterface.java | 8 ++
.../apache/tez/runtime/api/LogicalOutput.java | 7 +-
.../api/LogicalOutputFrameworkInterface.java | 7 ++
.../tez/runtime/api/MemoryUpdateCallback.java | 2 +
.../tez/runtime/api/MergedInputContext.java | 2 +
.../tez/runtime/api/MergedLogicalInput.java | 5 +
.../apache/tez/runtime/api/ObjectRegistry.java | 104 +++++++++++++++++++
.../java/org/apache/tez/runtime/api/Output.java | 5 +
.../apache/tez/runtime/api/OutputCommitter.java | 4 +-
.../tez/runtime/api/OutputCommitterContext.java | 4 +-
.../apache/tez/runtime/api/OutputContext.java | 3 +
.../runtime/api/OutputFrameworkInterface.java | 5 +
.../org/apache/tez/runtime/api/Processor.java | 8 +-
.../tez/runtime/api/ProcessorContext.java | 3 +
.../api/ProcessorFrameworkInterface.java | 4 +
.../java/org/apache/tez/runtime/api/Reader.java | 7 ++
.../org/apache/tez/runtime/api/TaskContext.java | 7 +-
.../java/org/apache/tez/runtime/api/Writer.java | 8 +-
.../api/events/CompositeDataMovementEvent.java | 2 +
.../runtime/api/events/DataMovementEvent.java | 2 +
.../events/InputConfigureVertexTasksEvent.java | 22 ++--
.../api/events/InputDataInformationEvent.java | 2 +
.../runtime/api/events/InputFailedEvent.java | 1 +
.../api/events/InputInitializerEvent.java | 4 +
.../runtime/api/events/InputReadErrorEvent.java | 2 +
.../api/events/InputUpdatePayloadEvent.java | 2 +
.../runtime/api/events/VertexManagerEvent.java | 2 +
.../common/objectregistry/ObjectRegistry.java | 99 ------------------
.../org/apache/tez/dag/app/DAGAppMaster.java | 9 +-
.../app/dag/impl/RootInputVertexManager.java | 3 +-
.../apache/tez/runtime/task/TezTaskRunner.java | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 2 +-
.../examples/BroadcastAndOneToOneExample.java | 2 +-
.../common/MRInputAMSplitGenerator.java | 3 +-
.../runtime/LogicalIOProcessorRuntimeTask.java | 2 +-
.../runtime/api/impl/TezInputContextImpl.java | 2 +-
.../runtime/api/impl/TezOutputContextImpl.java | 2 +-
.../api/impl/TezProcessorContextImpl.java | 2 +-
.../runtime/api/impl/TezTaskContextImpl.java | 2 +-
.../objectregistry/ObjectRegistryImpl.java | 2 +
.../objectregistry/TestObjectRegistry.java | 2 +-
112 files changed, 664 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d81a68d..19e4097 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -57,6 +57,7 @@ INCOMPATIBLE CHANGES
TEZ-1347. Consolidate MRHelpers.
TEZ-1072. Consolidate monitoring APIs in DAGClient
TEZ-1410. DAGClient#waitForCompletion() methods should not swallow interrupts
+ TEZ-1416. tez-api project javadoc/annotations review and clean up
Release 0.4.0-incubating: 2014-04-05
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
index 1504922..63617b3 100644
--- a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
@@ -20,6 +20,7 @@ package org.apache.tez.client;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezConfiguration;
+@Private
public abstract class FrameworkClient {
private static FrameworkClient localClient = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/client/PreWarmVertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/PreWarmVertex.java b/tez-api/src/main/java/org/apache/tez/client/PreWarmVertex.java
index 9c4e3fc..f3db312 100644
--- a/tez-api/src/main/java/org/apache/tez/client/PreWarmVertex.java
+++ b/tez-api/src/main/java/org/apache/tez/client/PreWarmVertex.java
@@ -18,6 +18,7 @@
package org.apache.tez.client;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -52,18 +53,57 @@ import org.apache.tez.runtime.api.Processor;
* for the actual processing to reduce latency.
*/
@Unstable
+@Public
public class PreWarmVertex extends Vertex {
+ /**
+ * Create a {@link PreWarmVertex} to be used in
+ * {@link TezClient#preWarm(PreWarmVertex)} It may be necessary to call more
+ * methods to add local files etc on the pre-warm vertex post creation so that
+ * it matches the real DAG vertices.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param processorDescriptor
+ * Descriptor of the processor to be run
+ * @param parallelism
+ * Number of containers to be pre-warmed
+ * @param taskResource
+ * Execution cpu/memory resources etc needed
+ */
public PreWarmVertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism,
Resource taskResource) {
super(vertexName, processorDescriptor, parallelism, taskResource);
}
+ /**
+ * Create a {@link PreWarmVertex} to be used in @link
+ * {@link TezClient#preWarm(PreWarmVertex)} This uses a built in pre-warm
+ * processor that implements common functionality. Users may derive from this
+ * processor to add custom functionality but then they must add the jar for
+ * that class to the prewarm vertex and other vertices in their DAG for which
+ * they want the containers to be reused. It may be necessary to call more
+ * methods to add local files etc on the pre-warm vertex post creation so that
+ * it matches the real DAG vertices.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param parallelism
+ * Number of containers to be pre-warmed
+ * @param taskResource
+ * Execution cpu/memory resources etc needed
+ */
public PreWarmVertex(String vertexName, int parallelism, Resource taskResource) {
this(vertexName, new ProcessorDescriptor(
"org.apache.tez.runtime.library.processor.PreWarmProcessor"), parallelism, taskResource);
}
+ /**
+ * Create a configurer for the @link {@link PreWarmVertex}. This may be used to construct the
+ * pre-warm vertex more flexibly.
+ * @param conf
+ * @return
+ */
public static PreWarmVertexConfigurer createConfigurer(Configuration conf) {
return new PreWarmVertexConfigurer(conf);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java b/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java
index f8cd310..d24dbf0 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java
@@ -18,6 +18,12 @@
package org.apache.tez.client;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
+/**
+ * Defines status for the App Master executing the DAG
+ */
+@Public
public enum TezAppMasterStatus {
/** App Master initializing itself */
INITIALIZING,
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 8f0e66f..24744e3 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -27,6 +27,7 @@ import javax.annotation.Nullable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -77,6 +78,7 @@ import com.google.protobuf.ServiceException;
* session mode configuration, the same application can be running in session or
* non-session mode.
*/
+@Public
public class TezClient {
private static final Log LOG = LogFactory.getLog(TezClient.class);
@@ -584,11 +586,14 @@ public class TezClient {
}
}
+ @VisibleForTesting
// for testing
+ @Private
protected FrameworkClient createFrameworkClient() {
return FrameworkClient.createFrameworkClient(amConfig.getTezConfiguration());
}
+ @VisibleForTesting
// for testing
protected DAGClientAMProtocolBlockingPB getSessionAMProxy(ApplicationId appId)
throws TezException, IOException {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 6b39f83..efe4237 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -75,9 +75,9 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
+import org.apache.tez.common.LogUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezYARNUtils;
-import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.ACLManager;
import org.apache.tez.common.security.Groups;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -462,7 +462,7 @@ public class TezClientUtils {
// FIX sun bug mentioned in TEZ-327
vargs.add("-Dsun.nio.ch.bugLevel=''");
- vargs.add(TezConfiguration.TEZ_APPLICATION_MASTER_CLASS);
+ vargs.add(TezConstants.TEZ_APPLICATION_MASTER_CLASS);
if (dag == null) {
vargs.add("--" + TezConstants.TEZ_SESSION_MODE_CLI_OPTION);
}
@@ -870,8 +870,7 @@ public class TezClientUtils {
return proxy;
}
- @Private
- public static void createSessionToken(String tokenIdentifier,
+ static void createSessionToken(String tokenIdentifier,
JobTokenSecretManager jobTokenSecretManager,
Credentials credentials) {
JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
@@ -882,6 +881,7 @@ public class TezClientUtils {
TokenCache.setSessionToken(sessionToken, credentials);
}
+ @Private
/**
* Add computed Xmx value to java opts if both -Xms and -Xmx are not specified
* @param javaOpts Current java opts
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
index 649e0e7..d58a0bb 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
@@ -20,6 +20,7 @@ package org.apache.tez.client;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -29,6 +30,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.dag.api.TezConfiguration;
+@Private
public class TezYarnClient extends FrameworkClient {
private final YarnClient yarnClient;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/LogUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/LogUtils.java b/tez-api/src/main/java/org/apache/tez/common/LogUtils.java
new file mode 100644
index 0000000..4074160
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/common/LogUtils.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+
+@Private
+public class LogUtils {
+
+ public static void logCredentials(Log log, Credentials credentials, String identifier) {
+ if (log.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("#" + identifier + "Tokens=").append(credentials.numberOfTokens());
+ if (credentials.numberOfTokens() > 0) {
+ sb.append(", Services: ");
+ for (Token<?> t : credentials.getAllTokens()) {
+ sb.append(t.getService()).append(",");
+ }
+ }
+ log.debug(sb.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java
index e64a26c..7bc1109 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
/**
* An abstract counter class to provide common implementation of
- * the counter interface in both mapred and mapreduce packages.
+ * the counter interface
*/
@InterfaceAudience.Private
public abstract class AbstractCounter implements TezCounter {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
index d8896ed..e9a291b 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
@@ -33,7 +33,7 @@ import com.google.common.collect.Iterators;
/**
* An abstract class to provide common implementation of the
- * generic counter group in both mapred and mapreduce package.
+ * generic counter group
*
* @param <T> type of the counter for the group
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java
index 4244dc2..6b9399d 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java
@@ -32,7 +32,8 @@ import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -48,8 +49,8 @@ import com.google.common.collect.Maps;
* @param <C> type of counter inside the counters
* @param <G> type of group inside the counters
*/
-@InterfaceAudience.Public
-@InterfaceStability.Stable
+@Public
+@Unstable
public abstract class AbstractCounters<C extends TezCounter,
G extends CounterGroupBase<C>>
implements Writable, Iterable<G> {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java
index bc7986d..c13e084 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java
@@ -18,15 +18,15 @@
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* A group of {@link TezCounter}s that logically belong together. Typically,
* it is an {@link Enum} subclass and the counters are the values.
*/
-@InterfaceAudience.Public
-@InterfaceStability.Stable
+@Public
+@Unstable
public interface CounterGroup extends CounterGroupBase<TezCounter> {
// essentially a typedef so user doesn't have to use generic syntax
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java
index 3b702ba..be4bf77 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java
@@ -18,9 +18,9 @@
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.io.Writable;
/**
@@ -28,8 +28,8 @@ import org.apache.hadoop.io.Writable;
*
* @param <T> type of the counter for the group
*/
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@Public
+@Unstable
public interface CounterGroupBase<T extends TezCounter>
extends Writable, Iterable<T> {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounter.java
index 08f4c5d..57d1053 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounter.java
@@ -18,9 +18,9 @@
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
-@InterfaceAudience.Private
+@Private
public enum FileSystemCounter {
BYTES_READ,
BYTES_WRITTEN,
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java
index 1eb2be8..14e76bc 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java
@@ -18,12 +18,10 @@
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
// Per-job counters
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@Private
public enum JobCounter {
NUM_FAILED_MAPS,
NUM_FAILED_REDUCES,
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/ResourceBundles.java b/tez-api/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
index 6cfe138..1742645 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
@@ -21,9 +21,12 @@ package org.apache.tez.common.counters;
import java.util.Locale;
import java.util.ResourceBundle;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
/**
* Helper class to handle resource bundles in a saner way
*/
+@Private
public class ResourceBundles {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java
index 8ebc6c8..e89ed44 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java
@@ -18,11 +18,10 @@
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
// Counters used by Task classes
-@InterfaceAudience.Private
-
+@Private
public enum TaskCounter {
// TODO Eventually, rename counters to be non-MR specific and map them to MR equivalent.
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java
index 394c820..2b40ed2 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java
@@ -18,23 +18,23 @@
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.io.Writable;
/**
- * A named counter that tracks the progress of a map/reduce job.
+ * A named counter that tracks the progress of a job.
*
* <p><code>Counters</code> represent global counters, defined either by the
- * Map-Reduce framework or applications. Each <code>Counter</code> is named by
+ * framework or applications. Each <code>Counter</code> is named by
* an {@link Enum} and has a long for the value.</p>
*
* <p><code>Counters</code> are bunched into Groups, each comprising of
* counters from a particular <code>Enum</code> class.
*/
-@InterfaceAudience.Public
-@InterfaceStability.Stable
+@Public
+@Unstable
public interface TezCounter extends Writable {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java
index 1c9521a..ca03f41 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java
@@ -17,19 +17,19 @@
*/
package org.apache.tez.common.counters;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* <p><code>Counters</code> holds per job/task counters, defined either by the
- * Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ * framework or applications. Each <code>Counter</code> can be of
* any {@link Enum} type.</p>
*
* <p><code>Counters</code> are bunched into {@link CounterGroup}s, each
* comprising of counters from a particular <code>Enum</code> class.
*/
-@InterfaceAudience.Public
-@InterfaceStability.Stable
+@Public
+@Unstable
public class TezCounters extends AbstractCounters<TezCounter, CounterGroup> {
// Mix framework group implementation into CounterGroup interface
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/impl/LogUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/impl/LogUtils.java b/tez-api/src/main/java/org/apache/tez/common/impl/LogUtils.java
deleted file mode 100644
index d5e324e..0000000
--- a/tez-api/src/main/java/org/apache/tez/common/impl/LogUtils.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.impl;
-
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-
-@Private
-public class LogUtils {
-
- public static void logCredentials(Log log, Credentials credentials, String identifier) {
- if (log.isDebugEnabled()) {
- StringBuilder sb = new StringBuilder();
- sb.append("#" + identifier + "Tokens=").append(credentials.numberOfTokens());
- if (credentials.numberOfTokens() > 0) {
- sb.append(", Services: ");
- for (Token<?> t : credentials.getAllTokens()) {
- sb.append(t.getService()).append(",");
- }
- }
- log.debug(sb.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java b/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java
index 7320dea..a522c6a 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java
@@ -20,6 +20,7 @@ package org.apache.tez.common.security;
import java.lang.annotation.Annotation;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.SecurityInfo;
@@ -29,6 +30,7 @@ import org.apache.hadoop.security.token.TokenSelector;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
+@Private
public class DAGClientSecurityInfo extends SecurityInfo {
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 4779259..6f61271 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -35,13 +35,14 @@ import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.DAGAccessControls;
+import org.apache.tez.common.LogUtils;
import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
@@ -64,6 +65,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+@Public
public class DAG {
private static final Log LOG = LogFactory.getLog(DAG.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java
index 0622ed5..347ed3d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java
@@ -18,9 +18,12 @@
package org.apache.tez.dag.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Exception thrown when DAG submission to a Tez Session times out.
*/
+@Public
public class DAGSubmissionTimedOut extends TezException {
private static final long serialVersionUID = 8521202283261990622L;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index 62e4a9c..cd12f16 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -30,6 +30,7 @@ import java.util.TreeSet;
import javax.annotation.Nullable;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.security.Credentials;
@@ -74,6 +75,7 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexLocationHintProto;
import com.google.protobuf.ByteString;
import com.google.protobuf.ByteString.Output;
+@Private
public class DagTypeConverters {
public static PlanLocalResourceVisibility convertToDAGPlan(LocalResourceVisibility visibility){
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
index b5e4e8f..849e701 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
@@ -20,12 +20,14 @@ package org.apache.tez.dag.api;
import javax.annotation.Nullable;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.security.Credentials;
/**
* Defines the output and output committer for a data sink
*
*/
+@Public
public class DataSinkDescriptor {
private final OutputDescriptor outputDescriptor;
private final OutputCommitterDescriptor committerDescriptor;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
index cae8ab3..ecd36b9 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
@@ -23,6 +23,7 @@ import javax.annotation.Nullable;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
@@ -33,6 +34,7 @@ import org.apache.tez.runtime.api.events.InputDataInformationEvent;
* Defines the input and input initializer for a data source
*
*/
+@Public
public class DataSourceDescriptor {
private final InputDescriptor inputDescriptor;
private final InputInitializerDescriptor initializerDescriptor;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/DuplicateDAGName.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DuplicateDAGName.java b/tez-api/src/main/java/org/apache/tez/dag/api/DuplicateDAGName.java
deleted file mode 100644
index 52a835d..0000000
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DuplicateDAGName.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-/**
- * This exception is thrown when there are duplicate DAG names in a session.
- *
- */
-public class DuplicateDAGName extends TezException {
-
- private static final long serialVersionUID = -6405001382308895028L;
-
- public DuplicateDAGName(Throwable cause) {
- super(cause);
- }
-
- public DuplicateDAGName(String message) {
- super(message);
- }
-
- public DuplicateDAGName(String message, Throwable cause) {
- super(message, cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
index 31a45f3..a7f531f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
@@ -18,7 +18,16 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+/**
+ * Edge defines the connection between a producer and consumer vertex in the DAG.
+ * @link {@link EdgeProperty} defines the relationship between them. The producer
+ * vertex provides input to the edge and the consumer vertex reads output from the
+ * edge.
+ *
+ */
+@Public
public class Edge {
private final Vertex inputVertex;
@@ -34,15 +43,26 @@ public class Edge {
this.edgeProperty = edgeProperty;
}
- // RENAME to source and destination
+ /**
+ * The @link {@link Vertex} that provides input to the edge
+ * @return {@link Vertex}
+ */
public Vertex getInputVertex() {
return inputVertex;
}
+ /**
+ * The @link {@link Vertex} that reads output from the edge
+ * @retur {@link Vertex}
+ */
public Vertex getOutputVertex() {
return outputVertex;
}
+ /**
+ * The @link {@link EdgeProperty} for this edge
+ * @return {@link EdgeProperty}
+ */
public EdgeProperty getEdgeProperty() {
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java
index 07bf3fe..7e8d3ae 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java
@@ -21,7 +21,8 @@ package org.apache.tez.dag.api;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.InputReadErrorEvent;
@@ -30,7 +31,8 @@ import org.apache.tez.runtime.api.events.InputReadErrorEvent;
* consumer vertices. The routing is bi-directional. Users can customize the
* routing by providing an implementation of this interface.
*/
-@InterfaceStability.Unstable
+@Public
+@Unstable
public abstract class EdgeManagerPlugin {
private final EdgeManagerPluginContext context;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java
index d55ea6d..fbeebba 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java
@@ -18,11 +18,16 @@
package org.apache.tez.dag.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
/**
* Context information provided to {@link EdgeManagerPlugin}s
* This interface is not supposed to be implemented by users
*
*/
+@Public
+@Unstable
public interface EdgeManagerPluginContext {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
index b3fb811..ff3a5ea 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
@@ -18,6 +18,12 @@
package org.apache.tez.dag.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
+/**
+ * Describes the @link {@link EdgeManagerPlugin}
+ */
+@Public
public class EdgeManagerPluginDescriptor extends EntityDescriptor<EdgeManagerPluginDescriptor> {
public EdgeManagerPluginDescriptor(String edgeManagerPluginClassName) {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
index 6ed3c92..c481708 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
@@ -18,9 +18,20 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import com.google.common.base.Preconditions;
+/**
+ * An @link {@link EdgeProperty} defines the relation between the source and
+ * destination vertices of an edge. The relation consists of defining their
+ * communication pattern and dependencies. It also defines the user code that
+ * actually does the job of writing out data at the source and reading that data
+ * at the destination via the @link {@link InputDescriptor} and @link
+ * {@link OutputDescriptor}
+ */
+@Public
public class EdgeProperty {
/**
@@ -64,14 +75,16 @@ public class EdgeProperty {
*/
PERSISTED,
/**
- * Source data is stored reliably and will always be available
+ * Source data is stored reliably and will always be available. This is not supported yet.
*/
+ @Unstable
PERSISTED_RELIABLE,
/**
* Data produced by the source task is available only while the source task
* is running. This requires the destination task to run concurrently with
- * the source task.
+ * the source task. This is not supported yet.
*/
+ @Unstable
EPHEMERAL
}
@@ -86,8 +99,10 @@ public class EdgeProperty {
*/
SEQUENTIAL,
/**
- * Destination task must run concurrently with the source task
+ * Destination task must run concurrently with the source task.
+ * This is not supported yet.
*/
+ @Unstable
CONCURRENT
}
@@ -148,14 +163,26 @@ public class EdgeProperty {
this.outputDescriptor = edgeSource;
}
+ /**
+ * Get the {@link DataMovementType}
+ * @return {@link DataMovementType}
+ */
public DataMovementType getDataMovementType() {
return dataMovementType;
}
+ /**
+ * Get the {@link DataSourceType}
+ * @return {@link DataSourceType}
+ */
public DataSourceType getDataSourceType() {
return dataSourceType;
}
+ /**
+ * Get the {@link SchedulingType}
+ * @return {@link SchedulingType}
+ */
public SchedulingType getSchedulingType() {
return schedulingType;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
index 468be99..c46be22 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
@@ -23,11 +23,20 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import com.google.common.base.Preconditions;
+/**
+ * Describes a given user code entity. Consists of the name of the class implementing
+ * the user logic and a payload that can be used to configure an object instance of
+ * that class. In addition some history information can be set for logging/debugging.
+ * <br>This is not supposed to be extended by users. Users are expected to use the derived
+ * classes for specific entities
+ */
+@Public
@SuppressWarnings("unchecked")
public abstract class EntityDescriptor<T extends EntityDescriptor<T>> implements Writable {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
index e7f25ea..eb3aa17 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
@@ -18,7 +18,17 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.tez.runtime.api.MergedLogicalInput;
+/**
+ * A composite edge that represents a common connection between a
+ * group of input vertices and a single output vertex. This can be
+ * used to perform e.g. an efficient union of the data produced by
+ * the input vertices. The output vertex tasks see a unified/merged
+ * view of the data from all the input vertices.
+ */
+@Public
public class GroupInputEdge {
private final VertexGroup inputVertexGroup;
@@ -28,17 +38,23 @@ public class GroupInputEdge {
// InputVertex(EdgeInput) ----- Edge ----- OutputVertex(EdgeOutput)]
/**
- * An Edge that connects a VertexGroup to a destination Vertex. The framework
- * takes care of connecting the VertexGroup members with the destination
- * vertex. The tasks of the destination vertex see only 1 input named after
- * the VertexGroup instead of individual inputs from group members. These
- * individual inputs are merged using the mergedInput before presenting them
- * to the destination task.
+ * An Edge that connects a {@link VertexGroup} to a destination Vertex. The
+ * framework takes care of connecting the {@link VertexGroup} members with the
+ * destination vertex. The tasks of the destination vertex see only 1 input
+ * named after the VertexGroup instead of individual inputs from group
+ * members. These individual inputs are merged using the mergedInput before
+ * presenting them to the destination task.
*
- * @param inputVertexGroup source VertexGroup
- * @param outputVertex destination Vertex
- * @param edgeProperty edge properties
- * @param mergedInput MergedLogicalInput
+ * @param inputVertexGroup
+ * source {@link VertexGroup}
+ * @param outputVertex
+ * destination Vertex
+ * @param edgeProperty
+ * the common {@link EdgeProperty} for this {@link GroupInputEdge}
+ * @param mergedInput
+ * {@link MergedLogicalInput} This input is responsible for merging
+ * the data from the input vertex tasks to create a single input for
+ * the output vertex tasks
*/
public GroupInputEdge(VertexGroup inputVertexGroup,
Vertex outputVertex,
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
index b80be02..2a23c08 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
@@ -19,7 +19,13 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.tez.runtime.api.Input;
+/**
+ * Describes the {@link Input}
+ */
+@Public
public class InputDescriptor extends EntityDescriptor<InputDescriptor> {
@Private // for Writable
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
index 4f6e63c..6db478d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
@@ -19,7 +19,13 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.tez.runtime.api.InputInitializer;
+/**
+ * Describes the {@link InputInitializer}
+ */
+@Public
public class InputInitializerDescriptor extends EntityDescriptor<InputInitializerDescriptor> {
@Private // for Writable
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
index 43c3e3b..b2fa2c8 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
@@ -19,7 +19,13 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.tez.runtime.api.OutputCommitter;
+/**
+ * Describes the {@link OutputCommitter}
+ */
+@Public
public class OutputCommitterDescriptor extends EntityDescriptor<OutputCommitterDescriptor> {
@Private // for Writable
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
index 60752e3..95c9326 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
@@ -19,7 +19,13 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.tez.runtime.api.Output;
+/**
+ * Describes the {@link Output}
+ */
+@Public
public class OutputDescriptor extends EntityDescriptor<OutputDescriptor> {
@Private // for Writable
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
index 1640615..a2ec7f3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
@@ -19,7 +19,13 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.tez.runtime.api.Processor;
+/**
+ * Describes the {@link Processor}
+ */
+@Public
public class ProcessorDescriptor extends EntityDescriptor<ProcessorDescriptor> {
@Private // for Writable
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java b/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java
index e8a274c..22544d5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java
@@ -18,10 +18,13 @@
package org.apache.tez.dag.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Exception thrown when the client cannot communicate with the Tez Session
* as the Tez Session is no longer running.
*/
+@Public
public class SessionNotRunning extends TezException {
private static final long serialVersionUID = -287996170505550316L;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index d3511a4..7a46d54 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -19,10 +19,15 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.LocalResource;
+/**
+ * Defines the configurations for Tez
+ */
+@Public
public class TezConfiguration extends Configuration {
public final static String TEZ_SITE_XML = "tez-site.xml";
@@ -39,8 +44,11 @@ public class TezConfiguration extends Configuration {
super(conf);
}
+ @Private
public static final String TEZ_PREFIX = "tez.";
+ @Private
public static final String TEZ_AM_PREFIX = TEZ_PREFIX + "am.";
+ @Private
public static final String TEZ_TASK_PREFIX = TEZ_PREFIX + "task.";
/** Whether to delete all the tez framework scratch data in staging dir on completion */
@@ -58,9 +66,6 @@ public class TezConfiguration extends Configuration {
* name.
*/
public static final String TEZ_CREDENTIALS_PATH = TEZ_PREFIX + "credentials.path";
-
- public static final String TEZ_APPLICATION_MASTER_CLASS =
- "org.apache.tez.dag.app.DAGAppMaster";
/** Execution mode for submitting DAG's to the AM */
public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session";
@@ -119,6 +124,7 @@ public class TezConfiguration extends Configuration {
+ "launch.env";
public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = "";
+ @Private
public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX +
"am.complete.cancel.delegation.tokens";
public static final boolean TEZ_AM_CANCEL_DELEGATION_TOKEN_DEFAULT = true;
@@ -226,18 +232,13 @@ public class TezConfiguration extends Configuration {
+ "resource.cpu.vcores";
public static final int TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT = 1;
- public static final String
- TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION = TEZ_AM_PREFIX
- + "slowstart-dag-scheduler.min-resource-fraction";
- public static final float
- TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION_DEFAULT = 0.5f;
-
/**
* The complete path to the serialized dag plan file
* <code>TEZ_AM_PLAN_PB_BINARY</code>. Used to make the plan available to
* individual tasks if needed. This will typically be a path in the job submit
* directory.
*/
+ @Private
public static final String TEZ_AM_PLAN_REMOTE_PATH = TEZ_AM_PREFIX
+ "dag-am-plan.remote.path";
@@ -377,12 +378,12 @@ public class TezConfiguration extends Configuration {
TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS =
TEZ_AM_PREFIX + "container.reuse.locality.delay-allocation-millis";
public static final long
- TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT = 1000l;
+ TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT = 250l;
/**
* The minimum amount of time to hold on to a container if no task can be
* assigned to it immediately. Only active when reuse is enabled. Set to -1 to
- * never release a container in a session.
+ * never release a container.
*/
public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS =
TEZ_AM_PREFIX + "container.idle.release-timeout-min.millis";
@@ -464,6 +465,7 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_APPLICATION_TYPE = "TEZ";
+ // TODO move this to TEZGrouper which is in runtime-library
public static final String TEZ_AM_GROUPING_SPLIT_COUNT = TEZ_AM_PREFIX +
"grouping.split-count";
public static final String TEZ_AM_GROUPING_SPLIT_BY_LENGTH = TEZ_AM_PREFIX +
@@ -504,6 +506,7 @@ public class TezConfiguration extends Configuration {
public static final float TEZ_AM_GROUPING_RACK_SPLIT_SIZE_REDUCTION_DEFAULT = 0.75f;
+ @Private
/**
* Session-related properties
*/
@@ -538,9 +541,9 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_QUEUE_NAME =
TEZ_PREFIX + "queue.name";
- public static final String TEZ_GENERATE_DAG_VIZ =
- TEZ_PREFIX + "generate.dag.viz";
- public static final boolean TEZ_GENERATE_DAG_VIZ_DEFAULT = true;
+ public static final String TEZ_GENERATE_DEBUG_ARTIFACTS =
+ TEZ_PREFIX + "generate.debug.artifacts";
+ public static final boolean TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT = true;
/**
* Set of tasks for which specific launch command options need to be added.
@@ -630,6 +633,7 @@ public class TezConfiguration extends Configuration {
/**
* Tez AM Inline Mode flag. Not valid till Tez-684 get checked-in
*/
+ @Private
public static final String TEZ_AM_INLINE_TASK_EXECUTION_ENABLED =
TEZ_AM_PREFIX + "inline.task.execution.enabled";
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 7767762..406a09f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -26,6 +26,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
@InterfaceAudience.Private
public class TezConstants {
+
+ public static final String TEZ_APPLICATION_MASTER_CLASS =
+ "org.apache.tez.dag.app.DAGAppMaster";
+
/**
* Command-line argument to be set when running the Tez AM in session mode.
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java
index e3b14e7..29c70d7 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java
@@ -18,9 +18,12 @@
package org.apache.tez.dag.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
- * Base TezException
+ * Checked Exception thrown upon error
*/
+@Public
public class TezException extends Exception {
private static final long serialVersionUID = 6337442733802964447L;
public TezException(Throwable cause) { super(cause); }
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java
index f55f6dd..af798b3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java
@@ -18,9 +18,12 @@
package org.apache.tez.dag.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
- * Base Tez Unchecked Exception
+ * Unchecked Exception thrown upon error
*/
+@Public
public class TezUncheckedException extends RuntimeException {
private static final long serialVersionUID = -4956339297375386184L;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
index e91d7df..1d97e36 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
@@ -20,10 +20,13 @@ package org.apache.tez.dag.api;
import javax.annotation.Nullable;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Wrapper class to hold user payloads
* Provides a version to help in evolving the payloads
*/
+@Public
public final class UserPayload {
private final byte[] payload;
private final int version;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 5624c4b..3a21c96 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.VertexGroup.GroupInfo;
@@ -34,6 +35,13 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+/**
+ * Defines a vertex in the DAG. It represents the application logic that
+ * processes and transforms the input data to create the output data. The
+ * vertex represents the template from which tasks are created to execute
+ * the application in parallel across a distributed execution environment.
+ */
+@Public
public class Vertex {
private final String vertexName;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
index 894d649..0594835 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
@@ -22,6 +22,8 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -36,6 +38,7 @@ import com.google.common.collect.Sets;
* member vertices of the VertexGroup.
* A VertexGroup is not part of the final DAG.
*/
+@Public
public class VertexGroup {
static class GroupInfo {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
index 979d363..898b6be 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
@@ -22,10 +22,17 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.ContainerId;
import com.google.common.base.Preconditions;
+/**
+ * Describes the placements hints for tasks in a vertex.
+ * The system will make a best-effort attempt to run the tasks
+ * close to the specified locations.
+ */
+@Public
public class VertexLocationHint {
private final List<TaskLocationHint> taskLocationHints;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
index 40e2cc0..f07734f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
@@ -21,7 +21,8 @@ package org.apache.tez.dag.api;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
@@ -31,7 +32,8 @@ import org.apache.tez.runtime.api.events.VertexManagerEvent;
* The plugin will be notified of interesting events in the vertex execution life
* cycle and can respond to them by via the context object
*/
-@InterfaceStability.Unstable
+@Unstable
+@Public
public abstract class VertexManagerPlugin {
private final VertexManagerPluginContext context;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
index 8998199..0d0a680 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
@@ -25,6 +25,7 @@ import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -34,10 +35,11 @@ import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import com.google.common.base.Preconditions;
-@Unstable
/**
* Object with API's to interact with the Tez execution engine
*/
+@Unstable
+@Public
public interface VertexManagerPluginContext {
public class TaskWithLocationHint {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
index a8986b2..c2bce76 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
@@ -19,7 +19,12 @@
package org.apache.tez.dag.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+/**
+ * Describes the {@link VertexManagerPlugin}
+ */
+@Public
public class VertexManagerPluginDescriptor extends EntityDescriptor<VertexManagerPluginDescriptor> {
@Private // for Writable
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
index d1e8e27..b745726 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -25,15 +25,17 @@ import java.util.Set;
import javax.annotation.Nullable;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
-/*
- * Interface class for monitoring the <code>DAG</code> running in a Tez DAG
+/**
+ * Class for monitoring the <code>DAG</code> running in a Tez DAG
* Application Master.
*/
+@Public
public abstract class DAGClient implements Closeable {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
index 1252ef8..8fc65e3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
@@ -23,13 +23,19 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.util.StringUtils;
import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProtoOrBuilder;
import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
import org.apache.tez.dag.api.TezUncheckedException;
+/**
+ * Describes the status of the {@link DAG}
+ */
+@Public
public class DAGStatus {
private static final String LINE_SEPARATOR = System
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
index a252683..98a7783 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
@@ -18,8 +18,15 @@
package org.apache.tez.dag.api.client;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.tez.dag.api.records.DAGProtos.ProgressProtoOrBuilder;
+/**
+ * Describes the progress made by DAG execution
+ */
+@Public
+@Evolving
public class Progress {
ProgressProtoOrBuilder proxy = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java
index 922ab24..1a9df7a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java
@@ -18,10 +18,15 @@
package org.apache.tez.dag.api.client;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
/**
* Status Get Options used when making calls like getDAGStatus and
* getVertexStatus in DAGClient
*/
+@Public
+@Evolving
public enum StatusGetOpts {
/** Retrieve Counters with Status */
GET_COUNTERS
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
index 6b5a8e3..3684eef 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
@@ -22,12 +22,19 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.records.DAGProtos;
import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProtoOrBuilder;
import org.apache.tez.dag.api.TezUncheckedException;
+/**
+ * Describes the status of the {@link Vertex}
+ */
+@Public
public class VertexStatus {
public enum State {
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
index 4dd971e..9d7ebee 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
@@ -18,9 +18,11 @@
package org.apache.tez.dag.api.client.rpc;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol;
+@Private
@ProtocolInfo(
protocolName = "org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB",
protocolVersion = 1)
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
index 701ee2a..e01ebd1 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -29,6 +29,7 @@ import javax.annotation.Nullable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -57,6 +58,7 @@ import org.apache.tez.dag.api.records.DAGProtos.DAGStatusStateProto;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ServiceException;
+@Private
public class DAGClientRPCImpl extends DAGClient {
private static final Log LOG = LogFactory.getLog(DAGClientRPCImpl.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java
index 0af498d..7714321 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java
@@ -17,14 +17,15 @@
*/
package org.apache.tez.runtime.api;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Abstract representation of the interface {@link LogicalIOProcessor}.
- * Implements the base logic of some methods into this class.
- * It will reduce the code for any processor implementation.
+ * Implements the base logic of some methods into this class and is expected
+ * to be the base class that is derived to implement the user {@link Processor}
*
*/
-@InterfaceAudience.Public
+@Public
public abstract class AbstractLogicalIOProcessor implements LogicalIOProcessor,
LogicalIOProcessorFrameworkInterface {
private final ProcessorContext context;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java
index d1facc7..dea79b7 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java
@@ -19,6 +19,8 @@ package org.apache.tez.runtime.api;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* An abstract class which should be the base class for all implementations of LogicalInput.
*
@@ -35,6 +37,7 @@ import java.util.List;
* <p/>
*
*/
+@Public
public abstract class AbstractLogicalInput implements LogicalInput, LogicalInputFrameworkInterface {
private final int numPhysicalInputs;
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java
index 317390a..f36b7f2 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java
@@ -19,6 +19,8 @@ package org.apache.tez.runtime.api;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* An abstract class which should be the base class for all implementations of LogicalOutput.
*
@@ -28,6 +30,7 @@ import java.util.List;
* This includes default implementations of a new method for convenience.
*
*/
+@Public
public abstract class AbstractLogicalOutput implements LogicalOutput, LogicalOutputFrameworkInterface {
private final int numPhysicalOutputs;
@@ -62,7 +65,7 @@ public abstract class AbstractLogicalOutput implements LogicalOutput, LogicalOut
}
/**
- * Return ahe {@link org.apache.tez.runtime.api.OutputContext} for this specific instance of
+ * Return the {@link org.apache.tez.runtime.api.OutputContext} for this specific instance of
* the LogicalOutput
*
* @return the {@link org.apache.tez.runtime.api.OutputContext} for the output
http://git-wip-us.apache.org/repos/asf/tez/blob/84374adc/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
index 6f3d667..d9c00d3 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
@@ -18,11 +18,15 @@
package org.apache.tez.runtime.api;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Base class for all events generated within the Tez execution engine.
* Used as the primary mode of communication between the AM, Inputs, Processors
* and Outputs.
+ * Users are not expected to implement or derive from this class
*/
+@Public
public abstract class Event {
}