You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/09/25 09:31:54 UTC
[47/50] [abbrv] git commit: Rename tez-engine-api to tez-runtime-api
and tez-engine is split into 2: - tez-engine-library for user-visible
Input/Output/Processor implementations - tez-engine-internals for framework
internals
Rename tez-engine-api to tez-runtime-api and tez-engine is split into 2:
- tez-engine-library for user-visible Input/Output/Processor implementations
- tez-engine-internals for framework internals
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/b212ca1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/b212ca1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/b212ca1d
Branch: refs/heads/master
Commit: b212ca1d2a087e80329511236022bf297919b960
Parents: 5d86b93
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Sep 24 15:42:53 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Sep 24 15:42:53 2013 -0700
----------------------------------------------------------------------
pom.xml | 10 +-
.../org/apache/tez/common/TezJobConfig.java | 213 ++--
.../java/org/apache/tez/engine/api/Event.java | 28 -
.../java/org/apache/tez/engine/api/Input.java | 71 --
.../tez/engine/api/LogicalIOProcessor.java | 43 -
.../org/apache/tez/engine/api/LogicalInput.java | 37 -
.../apache/tez/engine/api/LogicalOutput.java | 36 -
.../java/org/apache/tez/engine/api/Output.java | 71 --
.../org/apache/tez/engine/api/Processor.java | 55 -
.../java/org/apache/tez/engine/api/Reader.java | 26 -
.../apache/tez/engine/api/TezInputContext.java | 32 -
.../apache/tez/engine/api/TezOutputContext.java | 33 -
.../tez/engine/api/TezProcessorContext.java | 41 -
.../apache/tez/engine/api/TezTaskContext.java | 130 --
.../java/org/apache/tez/engine/api/Writer.java | 26 -
.../engine/api/events/DataMovementEvent.java | 109 --
.../tez/engine/api/events/InputFailedEvent.java | 89 --
.../api/events/InputInformationEvent.java | 41 -
.../engine/api/events/InputReadErrorEvent.java | 65 -
.../common/objectregistry/ObjectLifeCycle.java | 37 -
.../common/objectregistry/ObjectRegistry.java | 56 -
.../objectregistry/ObjectRegistryFactory.java | 32 -
.../java/org/apache/tez/runtime/api/Event.java | 28 +
.../java/org/apache/tez/runtime/api/Input.java | 71 ++
.../tez/runtime/api/LogicalIOProcessor.java | 43 +
.../apache/tez/runtime/api/LogicalInput.java | 37 +
.../apache/tez/runtime/api/LogicalOutput.java | 36 +
.../java/org/apache/tez/runtime/api/Output.java | 71 ++
.../org/apache/tez/runtime/api/Processor.java | 55 +
.../java/org/apache/tez/runtime/api/Reader.java | 26 +
.../apache/tez/runtime/api/TezInputContext.java | 32 +
.../tez/runtime/api/TezOutputContext.java | 33 +
.../tez/runtime/api/TezProcessorContext.java | 41 +
.../apache/tez/runtime/api/TezTaskContext.java | 130 ++
.../java/org/apache/tez/runtime/api/Writer.java | 26 +
.../runtime/api/events/DataMovementEvent.java | 109 ++
.../runtime/api/events/InputFailedEvent.java | 89 ++
.../api/events/InputInformationEvent.java | 41 +
.../runtime/api/events/InputReadErrorEvent.java | 65 +
.../common/objectregistry/ObjectLifeCycle.java | 37 +
.../common/objectregistry/ObjectRegistry.java | 56 +
.../objectregistry/ObjectRegistryFactory.java | 32 +
tez-api/src/main/proto/Events.proto | 2 +-
tez-dag/pom.xml | 2 +-
.../apache/hadoop/mapred/YarnTezDagChild.java | 36 +-
.../org/apache/tez/dag/app/DAGAppMaster.java | 2 +-
.../dag/app/TaskAttemptListenerImpTezDag.java | 8 +-
.../org/apache/tez/dag/app/dag/EdgeManager.java | 6 +-
.../java/org/apache/tez/dag/app/dag/Task.java | 2 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 6 +-
.../apache/tez/dag/app/dag/VertexScheduler.java | 2 +-
.../dag/event/TaskAttemptEventStatusUpdate.java | 2 +-
.../dag/app/dag/event/TaskEventAddTezEvent.java | 2 +-
.../app/dag/event/VertexEventRouteEvent.java | 2 +-
.../VertexEventSourceTaskAttemptCompleted.java | 2 +-
.../event/VertexEventTaskAttemptCompleted.java | 2 +-
.../dag/app/dag/impl/BroadcastEdgeManager.java | 6 +-
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 6 +-
.../org/apache/tez/dag/app/dag/impl/Edge.java | 16 +-
.../dag/impl/ImmediateStartVertexScheduler.java | 2 +-
.../dag/app/dag/impl/OneToOneEdgeManager.java | 6 +-
.../app/dag/impl/ScatterGatherEdgeManager.java | 6 +-
.../dag/app/dag/impl/ShuffleVertexManager.java | 12 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 4 +-
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 4 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 18 +-
.../app/rm/AMSchedulerEventTALaunchRequest.java | 2 +-
.../rm/container/AMContainerEventAssignTA.java | 2 +-
.../app/rm/container/AMContainerHelpers.java | 8 +-
.../dag/app/rm/container/AMContainerImpl.java | 2 +-
.../dag/app/rm/container/AMContainerTask.java | 2 +-
.../apache/tez/dag/utils/TezEngineChildJVM.java | 122 --
.../tez/dag/utils/TezRuntimeChildJVM.java | 122 ++
.../TezDependentTaskCompletionEvent.java | 228 ----
...TezTaskDependencyCompletionEventsUpdate.java | 64 -
.../TezDependentTaskCompletionEvent.java | 228 ++++
...TezTaskDependencyCompletionEventsUpdate.java | 64 +
.../tez/dag/app/dag/impl/TestDAGImpl.java | 2 +-
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 4 +-
.../dag/app/dag/impl/TestVertexScheduler.java | 2 +-
.../tez/dag/app/rm/TestContainerReuse.java | 6 +-
.../dag/app/rm/container/TestAMContainer.java | 6 +-
tez-engine/findbugs-exclude.xml | 16 -
tez-engine/pom.xml | 92 --
.../java/org/apache/hadoop/io/BufferUtils.java | 81 --
.../org/apache/hadoop/io/HashComparator.java | 24 -
.../java/org/apache/tez/common/Constants.java | 57 -
.../org/apache/tez/common/ContainerContext.java | 64 -
.../org/apache/tez/common/ContainerTask.java | 74 --
.../tez/common/TezTaskUmbilicalProtocol.java | 57 -
.../org/apache/tez/engine/api/KVReader.java | 81 --
.../org/apache/tez/engine/api/KVWriter.java | 40 -
.../org/apache/tez/engine/api/Partitioner.java | 52 -
.../api/events/TaskAttemptCompletedEvent.java | 28 -
.../api/events/TaskAttemptFailedEvent.java | 35 -
.../api/events/TaskStatusUpdateEvent.java | 70 --
.../tez/engine/api/impl/EventMetaData.java | 152 ---
.../apache/tez/engine/api/impl/EventType.java | 29 -
.../apache/tez/engine/api/impl/InputSpec.java | 88 --
.../apache/tez/engine/api/impl/OutputSpec.java | 87 --
.../apache/tez/engine/api/impl/TaskSpec.java | 146 ---
.../apache/tez/engine/api/impl/TezEvent.java | 248 ----
.../engine/api/impl/TezHeartbeatRequest.java | 137 ---
.../engine/api/impl/TezHeartbeatResponse.java | 105 --
.../engine/api/impl/TezInputContextImpl.java | 84 --
.../engine/api/impl/TezOutputContextImpl.java | 85 --
.../api/impl/TezProcessorContextImpl.java | 86 --
.../tez/engine/api/impl/TezTaskContextImpl.java | 145 ---
.../tez/engine/api/impl/TezUmbilical.java | 36 -
.../broadcast/input/BroadcastInputManager.java | 138 ---
.../broadcast/input/BroadcastKVReader.java | 225 ----
.../BroadcastShuffleInputEventHandler.java | 88 --
.../input/BroadcastShuffleManager.java | 489 --------
.../broadcast/output/FileBasedKVWriter.java | 125 --
.../apache/tez/engine/common/ConfigUtils.java | 148 ---
.../engine/common/InputAttemptIdentifier.java | 95 --
.../tez/engine/common/InputIdentifier.java | 56 -
.../tez/engine/common/TezEngineUtils.java | 153 ---
.../tez/engine/common/ValuesIterator.java | 194 ---
.../apache/tez/engine/common/YARNMaster.java | 57 -
.../tez/engine/common/combine/Combiner.java | 42 -
.../common/localshuffle/LocalShuffle.java | 120 --
.../objectregistry/ObjectRegistryImpl.java | 65 -
.../objectregistry/ObjectRegistryModule.java | 43 -
.../common/security/JobTokenIdentifier.java | 98 --
.../common/security/JobTokenSecretManager.java | 137 ---
.../common/security/JobTokenSelector.java | 53 -
.../tez/engine/common/security/Master.java | 57 -
.../common/security/SecureShuffleUtils.java | 142 ---
.../tez/engine/common/security/TokenCache.java | 205 ----
.../common/shuffle/impl/ExceptionReporter.java | 25 -
.../tez/engine/common/shuffle/impl/Fetcher.java | 624 ----------
.../common/shuffle/impl/InMemoryReader.java | 156 ---
.../common/shuffle/impl/InMemoryWriter.java | 100 --
.../tez/engine/common/shuffle/impl/MapHost.java | 124 --
.../engine/common/shuffle/impl/MapOutput.java | 227 ----
.../common/shuffle/impl/MergeManager.java | 782 ------------
.../engine/common/shuffle/impl/MergeThread.java | 108 --
.../tez/engine/common/shuffle/impl/Shuffle.java | 278 -----
.../shuffle/impl/ShuffleClientMetrics.java | 91 --
.../common/shuffle/impl/ShuffleHeader.java | 94 --
.../shuffle/impl/ShuffleInputEventHandler.java | 134 ---
.../common/shuffle/impl/ShuffleScheduler.java | 521 --------
.../common/shuffle/server/ShuffleHandler.java | 572 ---------
.../engine/common/sort/impl/ExternalSorter.java | 194 ---
.../tez/engine/common/sort/impl/IFile.java | 559 ---------
.../common/sort/impl/IFileInputStream.java | 276 -----
.../common/sort/impl/IFileOutputStream.java | 129 --
.../common/sort/impl/PipelinedSorter.java | 932 ---------------
.../engine/common/sort/impl/TezIndexRecord.java | 45 -
.../tez/engine/common/sort/impl/TezMerger.java | 798 -------------
.../sort/impl/TezRawKeyValueIterator.java | 70 --
.../engine/common/sort/impl/TezSpillRecord.java | 146 ---
.../common/sort/impl/dflt/DefaultSorter.java | 1108 ------------------
.../sort/impl/dflt/InMemoryShuffleSorter.java | 126 --
.../sort/impl/dflt/SortBufferInputStream.java | 271 -----
.../engine/common/task/impl/ValuesIterator.java | 149 ---
.../local/output/TezLocalTaskOutputFiles.java | 249 ----
.../common/task/local/output/TezTaskOutput.java | 165 ---
.../task/local/output/TezTaskOutputFiles.java | 246 ----
.../engine/hadoop/compat/NullProgressable.java | 33 -
.../tez/engine/lib/input/LocalMergedInput.java | 52 -
.../engine/lib/input/ShuffledMergedInput.java | 179 ---
.../lib/input/ShuffledMergedInputLegacy.java | 30 -
.../lib/input/ShuffledUnorderedKVInput.java | 76 --
.../engine/lib/output/InMemorySortedOutput.java | 81 --
.../lib/output/LocalOnFileSorterOutput.java | 63 -
.../engine/lib/output/OnFileSortedOutput.java | 123 --
.../lib/output/OnFileUnorderedKVOutput.java | 98 --
.../LogicalIOProcessorRuntimeTask.java | 475 --------
.../tez/engine/newruntime/RuntimeTask.java | 106 --
.../tez/engine/newruntime/RuntimeUtils.java | 62 -
.../engine/shuffle/common/DiskFetchedInput.java | 111 --
.../tez/engine/shuffle/common/FetchResult.java | 70 --
.../tez/engine/shuffle/common/FetchedInput.java | 144 ---
.../shuffle/common/FetchedInputAllocator.java | 31 -
.../shuffle/common/FetchedInputCallback.java | 29 -
.../tez/engine/shuffle/common/Fetcher.java | 608 ----------
.../engine/shuffle/common/FetcherCallback.java | 31 -
.../tez/engine/shuffle/common/InputHost.java | 90 --
.../shuffle/common/MemoryFetchedInput.java | 89 --
.../tez/engine/shuffle/common/ShuffleUtils.java | 81 --
tez-engine/src/main/proto/Events.proto | 28 -
tez-engine/src/main/proto/ShufflePayloads.proto | 33 -
.../objectregistry/TestObjectRegistry.java | 56 -
.../tez/mapreduce/examples/MRRSleepJob.java | 10 +-
.../mapreduce/examples/OrderedWordCount.java | 4 +-
.../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 4 +-
tez-mapreduce/pom.xml | 7 +-
.../apache/hadoop/mapred/LocalJobRunnerTez.java | 2 +-
.../tez/mapreduce/combine/MRCombiner.java | 16 +-
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 97 +-
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 6 +-
.../hadoop/MultiStageMRConfToTezTranslator.java | 12 +-
.../hadoop/MultiStageMRConfigUtil.java | 2 +-
.../tez/mapreduce/hadoop/mapred/MRReporter.java | 4 +-
.../hadoop/mapreduce/MapContextImpl.java | 2 +-
.../mapreduce/TaskAttemptContextImpl.java | 2 +-
.../mapreduce/TaskInputOutputContextImpl.java | 2 +-
.../org/apache/tez/mapreduce/input/MRInput.java | 9 +-
.../apache/tez/mapreduce/output/MROutput.java | 8 +-
.../tez/mapreduce/partition/MRPartitioner.java | 8 +-
.../apache/tez/mapreduce/processor/MRTask.java | 17 +-
.../tez/mapreduce/processor/MRTaskReporter.java | 8 +-
.../mapreduce/processor/map/MapProcessor.java | 16 +-
.../processor/reduce/ReduceProcessor.java | 22 +-
.../org/apache/tez/mapreduce/TestUmbilical.java | 6 +-
.../hadoop/TestConfigTranslationMRToTez.java | 2 +-
.../mapreduce/hadoop/TestDeprecatedKeys.java | 18 +-
.../tez/mapreduce/processor/MapUtils.java | 12 +-
.../processor/map/TestMapProcessor.java | 24 +-
.../processor/reduce/TestReduceProcessor.java | 24 +-
tez-runtime-internals/findbugs-exclude.xml | 16 +
tez-runtime-internals/pom.xml | 95 ++
.../org/apache/tez/common/ContainerContext.java | 64 +
.../org/apache/tez/common/ContainerTask.java | 74 ++
.../tez/common/TezTaskUmbilicalProtocol.java | 57 +
.../runtime/LogicalIOProcessorRuntimeTask.java | 475 ++++++++
.../org/apache/tez/runtime/RuntimeTask.java | 106 ++
.../org/apache/tez/runtime/RuntimeUtils.java | 62 +
.../api/events/TaskAttemptCompletedEvent.java | 28 +
.../api/events/TaskAttemptFailedEvent.java | 35 +
.../api/events/TaskStatusUpdateEvent.java | 70 ++
.../tez/runtime/api/impl/EventMetaData.java | 152 +++
.../apache/tez/runtime/api/impl/EventType.java | 29 +
.../apache/tez/runtime/api/impl/InputSpec.java | 88 ++
.../apache/tez/runtime/api/impl/OutputSpec.java | 87 ++
.../apache/tez/runtime/api/impl/TaskSpec.java | 146 +++
.../apache/tez/runtime/api/impl/TezEvent.java | 248 ++++
.../runtime/api/impl/TezHeartbeatRequest.java | 137 +++
.../runtime/api/impl/TezHeartbeatResponse.java | 105 ++
.../runtime/api/impl/TezInputContextImpl.java | 84 ++
.../runtime/api/impl/TezOutputContextImpl.java | 85 ++
.../api/impl/TezProcessorContextImpl.java | 86 ++
.../runtime/api/impl/TezTaskContextImpl.java | 145 +++
.../tez/runtime/api/impl/TezUmbilical.java | 36 +
.../objectregistry/ObjectRegistryImpl.java | 68 ++
.../objectregistry/ObjectRegistryModule.java | 46 +
.../src/main/proto/Events.proto | 28 +
.../objectregistry/TestObjectRegistry.java | 60 +
tez-runtime-library/pom.xml | 79 ++
.../java/org/apache/hadoop/io/BufferUtils.java | 81 ++
.../org/apache/hadoop/io/HashComparator.java | 24 +
.../tez/runtime/library/api/KVReader.java | 81 ++
.../tez/runtime/library/api/KVWriter.java | 40 +
.../tez/runtime/library/api/Partitioner.java | 52 +
.../broadcast/input/BroadcastInputManager.java | 138 +++
.../broadcast/input/BroadcastKVReader.java | 225 ++++
.../BroadcastShuffleInputEventHandler.java | 88 ++
.../input/BroadcastShuffleManager.java | 489 ++++++++
.../broadcast/output/FileBasedKVWriter.java | 125 ++
.../tez/runtime/library/common/ConfigUtils.java | 148 +++
.../tez/runtime/library/common/Constants.java | 61 +
.../library/common/InputAttemptIdentifier.java | 95 ++
.../runtime/library/common/InputIdentifier.java | 56 +
.../runtime/library/common/TezRuntimeUtils.java | 152 +++
.../runtime/library/common/ValuesIterator.java | 194 +++
.../tez/runtime/library/common/YARNMaster.java | 57 +
.../library/common/combine/Combiner.java | 42 +
.../common/localshuffle/LocalShuffle.java | 120 ++
.../common/security/JobTokenIdentifier.java | 98 ++
.../common/security/JobTokenSecretManager.java | 137 +++
.../common/security/JobTokenSelector.java | 53 +
.../runtime/library/common/security/Master.java | 57 +
.../common/security/SecureShuffleUtils.java | 142 +++
.../library/common/security/TokenCache.java | 205 ++++
.../common/shuffle/impl/ExceptionReporter.java | 25 +
.../library/common/shuffle/impl/Fetcher.java | 624 ++++++++++
.../common/shuffle/impl/InMemoryReader.java | 156 +++
.../common/shuffle/impl/InMemoryWriter.java | 100 ++
.../library/common/shuffle/impl/MapHost.java | 124 ++
.../library/common/shuffle/impl/MapOutput.java | 227 ++++
.../common/shuffle/impl/MergeManager.java | 782 ++++++++++++
.../common/shuffle/impl/MergeThread.java | 108 ++
.../library/common/shuffle/impl/Shuffle.java | 278 +++++
.../shuffle/impl/ShuffleClientMetrics.java | 91 ++
.../common/shuffle/impl/ShuffleHeader.java | 94 ++
.../shuffle/impl/ShuffleInputEventHandler.java | 134 +++
.../common/shuffle/impl/ShuffleScheduler.java | 521 ++++++++
.../common/shuffle/server/ShuffleHandler.java | 572 +++++++++
.../common/sort/impl/ExternalSorter.java | 194 +++
.../runtime/library/common/sort/impl/IFile.java | 559 +++++++++
.../common/sort/impl/IFileInputStream.java | 276 +++++
.../common/sort/impl/IFileOutputStream.java | 129 ++
.../common/sort/impl/PipelinedSorter.java | 932 +++++++++++++++
.../common/sort/impl/TezIndexRecord.java | 45 +
.../library/common/sort/impl/TezMerger.java | 798 +++++++++++++
.../sort/impl/TezRawKeyValueIterator.java | 70 ++
.../common/sort/impl/TezSpillRecord.java | 146 +++
.../common/sort/impl/dflt/DefaultSorter.java | 1108 ++++++++++++++++++
.../sort/impl/dflt/InMemoryShuffleSorter.java | 126 ++
.../sort/impl/dflt/SortBufferInputStream.java | 271 +++++
.../common/task/impl/ValuesIterator.java | 149 +++
.../local/output/TezLocalTaskOutputFiles.java | 249 ++++
.../common/task/local/output/TezTaskOutput.java | 165 +++
.../task/local/output/TezTaskOutputFiles.java | 246 ++++
.../library/hadoop/compat/NullProgressable.java | 33 +
.../runtime/library/input/LocalMergedInput.java | 52 +
.../library/input/ShuffledMergedInput.java | 179 +++
.../input/ShuffledMergedInputLegacy.java | 30 +
.../library/input/ShuffledUnorderedKVInput.java | 76 ++
.../library/output/InMemorySortedOutput.java | 81 ++
.../library/output/LocalOnFileSorterOutput.java | 63 +
.../library/output/OnFileSortedOutput.java | 123 ++
.../library/output/OnFileUnorderedKVOutput.java | 98 ++
.../shuffle/common/DiskFetchedInput.java | 111 ++
.../library/shuffle/common/FetchResult.java | 70 ++
.../library/shuffle/common/FetchedInput.java | 144 +++
.../shuffle/common/FetchedInputAllocator.java | 31 +
.../shuffle/common/FetchedInputCallback.java | 29 +
.../runtime/library/shuffle/common/Fetcher.java | 608 ++++++++++
.../library/shuffle/common/FetcherCallback.java | 31 +
.../library/shuffle/common/InputHost.java | 90 ++
.../shuffle/common/MemoryFetchedInput.java | 89 ++
.../library/shuffle/common/ShuffleUtils.java | 81 ++
.../src/main/proto/ShufflePayloads.proto | 33 +
.../org/apache/tez/mapreduce/YARNRunner.java | 4 +-
318 files changed, 18962 insertions(+), 18856 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 63f17eb..ce10f6d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,12 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-engine</artifactId>
+ <artifactId>tez-runtime-internals</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -247,7 +252,8 @@
<modules>
<module>tez-api</module>
<module>tez-common</module>
- <module>tez-engine</module>
+ <module>tez-runtime-library</module>
+ <module>tez-runtime-internals</module>
<module>tez-yarn-client</module>
<module>tez-mapreduce</module>
<module>tez-mapreduce-examples</module>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
index 2c4b911..ace87ca 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -42,16 +42,16 @@ public class TezJobConfig {
/**
* Configuration key to enable/disable IFile readahead.
*/
- public static final String TEZ_ENGINE_IFILE_READAHEAD =
- "tez.engine.ifile.readahead";
- public static final boolean DEFAULT_TEZ_ENGINE_IFILE_READAHEAD = true;
+ public static final String TEZ_RUNTIME_IFILE_READAHEAD =
+ "tez.runtime.ifile.readahead";
+ public static final boolean DEFAULT_TEZ_RUNTIME_IFILE_READAHEAD = true;
/**
* Configuration key to set the IFile readahead length in bytes.
*/
- public static final String TEZ_ENGINE_IFILE_READAHEAD_BYTES =
- "tez.engine.ifile.readahead.bytes";
- public static final int DEFAULT_TEZ_ENGINE_IFILE_READAHEAD_BYTES =
+ public static final String TEZ_RUNTIME_IFILE_READAHEAD_BYTES =
+ "tez.runtime.ifile.readahead.bytes";
+ public static final int DEFAULT_TEZ_RUNTIME_IFILE_READAHEAD_BYTES =
4 * 1024 * 1024;
/**
@@ -62,102 +62,103 @@ public class TezJobConfig {
public static final long DEFAULT_RECORDS_BEFORE_PROGRESS = 10000;
/**
- * List of directories avialble to the engine.
+ * List of directories avialble to the Runtime.
*/
@Private
- public static final String LOCAL_DIRS = "tez.engine.local.dirs";
+ public static final String LOCAL_DIRS = "tez.runtime.local.dirs";
public static final String DEFAULT_LOCAL_DIRS = "/tmp";
/**
* One local dir for the speicfic job.
*/
- public static final String JOB_LOCAL_DIR = "tez.engine.job.local.dir";
+ public static final String JOB_LOCAL_DIR = "tez.runtime.job.local.dir";
/**
* The directory which contains the localized files for this task.
*/
@Private
- public static final String TASK_LOCAL_RESOURCE_DIR = "tez.engine.task-local-resource.dir";
+ public static final String TASK_LOCAL_RESOURCE_DIR = "tez.runtime.task-local-resource.dir";
public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
- public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
+ public static final String TEZ_TASK_WORKING_DIR = "tez.runtime.task.working.dir";
/**
*
*/
- public static final String TEZ_ENGINE_IO_SORT_FACTOR =
- "tez.engine.io.sort.factor";
- public static final int DEFAULT_TEZ_ENGINE_IO_SORT_FACTOR = 100;
+ public static final String TEZ_RUNTIME_IO_SORT_FACTOR =
+ "tez.runtime.io.sort.factor";
+ public static final int DEFAULT_TEZ_RUNTIME_IO_SORT_FACTOR = 100;
/**
*
*/
- public static final String TEZ_ENGINE_SORT_SPILL_PERCENT =
- "tez.engine.sort.spill.percent";
- public static float DEFAULT_TEZ_ENGINE_SORT_SPILL_PERCENT = 0.8f;
+ public static final String TEZ_RUNTIME_SORT_SPILL_PERCENT =
+ "tez.runtime.sort.spill.percent";
+ public static float DEFAULT_TEZ_RUNTIME_SORT_SPILL_PERCENT = 0.8f;
/**
*
*/
- public static final String TEZ_ENGINE_IO_SORT_MB = "tez.engine.io.sort.mb";
- public static final int DEFAULT_TEZ_ENGINE_IO_SORT_MB = 100;
+ public static final String TEZ_RUNTIME_IO_SORT_MB = "tez.runtime.io.sort.mb";
+ public static final int DEFAULT_TEZ_RUNTIME_IO_SORT_MB = 100;
/**
*
*/
- public static final String TEZ_ENGINE_INDEX_CACHE_MEMORY_LIMIT_BYTES =
- "tez.engine.index.cache.memory.limit.bytes";
- public static final int DEFAULT_TEZ_ENGINE_INDEX_CACHE_MEMORY_LIMIT_BYTES =
+ public static final String TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES =
+ "tez.runtime.index.cache.memory.limit.bytes";
+ public static final int DEFAULT_TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES =
1024 * 1024;
/**
*
*/
- public static final String TEZ_ENGINE_COMBINE_MIN_SPILLS =
- "tez.engine.combine.min.spills";
- public static final int DEFAULT_TEZ_ENGINE_COMBINE_MIN_SPILLS = 3;
+ public static final String TEZ_RUNTIME_COMBINE_MIN_SPILLS =
+ "tez.runtime.combine.min.spills";
+ public static final int DEFAULT_TEZ_RUNTIME_COMBINE_MIN_SPILLS = 3;
/**
*
*/
- public static final String TEZ_ENGINE_SORT_THREADS =
- "tez.engine.sort.threads";
- public static final int DEFAULT_TEZ_ENGINE_SORT_THREADS = 1;
+ public static final String TEZ_RUNTIME_SORT_THREADS =
+ "tez.runtime.sort.threads";
+ public static final int DEFAULT_TEZ_RUNTIME_SORT_THREADS = 1;
/**
- * Specifies a partitioner class, which is used in Tez engine components like OnFileSortedOutput
+ * Specifies a partitioner class, which is used in Tez Runtime components
+ * like OnFileSortedOutput
*/
- public static final String TEZ_ENGINE_PARTITIONER_CLASS = "tez.engine.partitioner.class";
+ public static final String TEZ_RUNTIME_PARTITIONER_CLASS = "tez.runtime.partitioner.class";
/**
* Specifies a combiner class (primarily for Shuffle)
*/
- public static final String TEZ_ENGINE_COMBINER_CLASS = "tez.engine.combiner.class";
+ public static final String TEZ_RUNTIME_COMBINER_CLASS = "tez.runtime.combiner.class";
- public static final String TEZ_ENGINE_NUM_EXPECTED_PARTITIONS = "tez.engine.num.expected.partitions";
+ public static final String TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS = "tez.runtime.num.expected.partitions";
/**
*
*/
- public static final String COUNTERS_MAX_KEY = "tez.engine.job.counters.max";
+ public static final String COUNTERS_MAX_KEY = "tez.runtime.job.counters.max";
public static final int COUNTERS_MAX_DEFAULT = 120;
/**
*
*/
- public static final String COUNTER_GROUP_NAME_MAX_KEY = "tez.engine.job.counters.group.name.max";
+ public static final String COUNTER_GROUP_NAME_MAX_KEY = "tez.runtime.job.counters.group.name.max";
public static final int COUNTER_GROUP_NAME_MAX_DEFAULT = 128;
/**
*
*/
- public static final String COUNTER_NAME_MAX_KEY = "tez.engine.job.counters.counter.name.max";
+ public static final String COUNTER_NAME_MAX_KEY = "tez.runtime.job.counters.counter.name.max";
public static final int COUNTER_NAME_MAX_DEFAULT = 64;
/**
*
*/
- public static final String COUNTER_GROUPS_MAX_KEY = "tez.engine.job.counters.groups.max";
+ public static final String COUNTER_GROUPS_MAX_KEY = "tez.runtime.job.counters.groups.max";
public static final int COUNTER_GROUPS_MAX_DEFAULT = 50;
@@ -166,145 +167,145 @@ public class TezJobConfig {
* in-memory shuffle should be used.
*/
@Private
- public static final String TEZ_ENGINE_SHUFFLE_USE_IN_MEMORY =
- "tez.engine.shuffle.use.in-memory";
- public static final boolean DEFAULT_TEZ_ENGINE_SHUFFLE_USE_IN_MEMORY = false;
+ public static final String TEZ_RUNTIME_SHUFFLE_USE_IN_MEMORY =
+ "tez.runtime.shuffle.use.in-memory";
+ public static final boolean DEFAULT_TEZ_RUNTIME_SHUFFLE_USE_IN_MEMORY = false;
// TODO NEWTEZ Remove these config parameters. Will be part of an event.
@Private
- public static final String TEZ_ENGINE_SHUFFLE_PARTITION_RANGE =
- "tez.engine.shuffle.partition-range";
- public static int TEZ_ENGINE_SHUFFLE_PARTITION_RANGE_DEFAULT = 1;
+ public static final String TEZ_RUNTIME_SHUFFLE_PARTITION_RANGE =
+ "tez.runtime.shuffle.partition-range";
+ public static int TEZ_RUNTIME_SHUFFLE_PARTITION_RANGE_DEFAULT = 1;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_PARALLEL_COPIES =
- "tez.engine.shuffle.parallel.copies";
- public static final int DEFAULT_TEZ_ENGINE_SHUFFLE_PARALLEL_COPIES = 20;
+ public static final String TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES =
+ "tez.runtime.shuffle.parallel.copies";
+ public static final int DEFAULT_TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES = 20;
/**
* TODO Is this user configurable.
*/
- public static final String TEZ_ENGINE_METRICS_SESSION_ID =
- "tez.engine.metrics.session.id";
- public static final String DEFAULT_TEZ_ENGINE_METRICS_SESSION_ID = "";
+ public static final String TEZ_RUNTIME_METRICS_SESSION_ID =
+ "tez.runtime.metrics.session.id";
+ public static final String DEFAULT_TEZ_RUNTIME_METRICS_SESSION_ID = "";
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_FETCH_FAILURES =
- "tez.engine.shuffle.fetch.failures.limit";
- public final static int DEFAULT_TEZ_ENGINE_SHUFFLE_FETCH_FAILURES_LIMIT = 10;
+ public static final String TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES =
+ "tez.runtime.shuffle.fetch.failures.limit";
+ public final static int DEFAULT_TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT = 10;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_NOTIFY_READERROR =
- "tez.engine.shuffle.notify.readerror";
- public static final boolean DEFAULT_TEZ_ENGINE_SHUFFLE_NOTIFY_READERROR = true;
+ public static final String TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR =
+ "tez.runtime.shuffle.notify.readerror";
+ public static final boolean DEFAULT_TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR = true;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_CONNECT_TIMEOUT =
- "tez.engine.shuffle.connect.timeout";
- public static final int DEFAULT_TEZ_ENGINE_SHUFFLE_STALLED_COPY_TIMEOUT =
+ public static final String TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT =
+ "tez.runtime.shuffle.connect.timeout";
+ public static final int DEFAULT_TEZ_RUNTIME_SHUFFLE_STALLED_COPY_TIMEOUT =
3 * 60 * 1000;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_READ_TIMEOUT = "tez.engine.shuffle.read.timeout";
- public final static int DEFAULT_TEZ_ENGINE_SHUFFLE_READ_TIMEOUT =
+ public static final String TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT = "tez.runtime.shuffle.read.timeout";
+ public final static int DEFAULT_TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT =
3 * 60 * 1000;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_ENABLE_SSL =
- "tez.engine.shuffle.ssl.enable";
- public static final boolean DEFAULT_TEZ_ENGINE_SHUFFLE_ENABLE_SSL = false;
+ public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_SSL =
+ "tez.runtime.shuffle.ssl.enable";
+ public static final boolean DEFAULT_TEZ_RUNTIME_SHUFFLE_ENABLE_SSL = false;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_INPUT_BUFFER_PERCENT =
- "tez.engine.shuffle.input.buffer.percent";
- public static final float DEFAULT_TEZ_ENGINE_SHUFFLE_INPUT_BUFFER_PERCENT =
+ public static final String TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT =
+ "tez.runtime.shuffle.input.buffer.percent";
+ public static final float DEFAULT_TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT =
0.90f;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_MEMORY_LIMIT_PERCENT =
- "tez.engine.shuffle.memory.limit.percent";
- public static final float DEFAULT_TEZ_ENGINE_SHUFFLE_MEMORY_LIMIT_PERCENT =
+ public static final String TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT =
+ "tez.runtime.shuffle.memory.limit.percent";
+ public static final float DEFAULT_TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT =
0.25f;
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_MERGE_PERCENT =
- "tez.engine.shuffle.merge.percent";
- public static final float DEFAULT_TEZ_ENGINE_SHUFFLE_MERGE_PERCENT = 0.90f;
+ public static final String TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT =
+ "tez.runtime.shuffle.merge.percent";
+ public static final float DEFAULT_TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT = 0.90f;
/**
* TODO TEZAM3 default value ?
*/
- public static final String TEZ_ENGINE_SHUFFLE_MEMTOMEM_SEGMENTS =
- "tez.engine.shuffle.memory-to-memory.segments";
+ public static final String TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS =
+ "tez.runtime.shuffle.memory-to-memory.segments";
/**
*
*/
- public static final String TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM =
- "tez.engine.shuffle.memory-to-memory.enable";
- public static final boolean DEFAULT_TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM =
+ public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM =
+ "tez.runtime.shuffle.memory-to-memory.enable";
+ public static final boolean DEFAULT_TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM =
false;
/**
*
*/
- public static final String TEZ_ENGINE_INPUT_BUFFER_PERCENT =
- "tez.engine.task.input.buffer.percent";
- public static final float DEFAULT_TEZ_ENGINE_INPUT_BUFFER_PERCENT = 0.0f;
+ public static final String TEZ_RUNTIME_INPUT_BUFFER_PERCENT =
+ "tez.runtime.task.input.buffer.percent";
+ public static final float DEFAULT_TEZ_RUNTIME_INPUT_BUFFER_PERCENT = 0.0f;
// TODO Rename.
- public static final String TEZ_ENGINE_GROUP_COMPARATOR_CLASS =
- "tez.engine.group.comparator.class";
+ public static final String TEZ_RUNTIME_GROUP_COMPARATOR_CLASS =
+ "tez.runtime.group.comparator.class";
// TODO Better name.
- public static final String TEZ_ENGINE_INTERNAL_SORTER_CLASS =
- "tez.engine.internal.sorter.class";
+ public static final String TEZ_RUNTIME_INTERNAL_SORTER_CLASS =
+ "tez.runtime.internal.sorter.class";
- public static final String TEZ_ENGINE_INTERMEDIATE_OUTPUT_KEY_COMPARATOR_CLASS =
- "tez.engine.intermediate-output.key.comparator.class";
- public static final String TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_COMPARATOR_CLASS =
- "tez.engine.intermediate-input.key.comparator.class";
-
- public static final String TEZ_ENGINE_INTERMEDIATE_OUTPUT_KEY_CLASS =
- "tez.engine.intermediate-output.key.class";
- public static final String TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_CLASS =
- "tez.engine.intermediate-input.key.class";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_COMPARATOR_CLASS =
+ "tez.runtime.intermediate-output.key.comparator.class";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_INPUT_KEY_COMPARATOR_CLASS =
+ "tez.runtime.intermediate-input.key.comparator.class";
+
+ public static final String TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS =
+ "tez.runtime.intermediate-output.key.class";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_INPUT_KEY_CLASS =
+ "tez.runtime.intermediate-input.key.class";
- public static final String TEZ_ENGINE_INTERMEDIATE_OUTPUT_VALUE_CLASS =
- "tez.engine.intermediate-output.value.class";
- public static final String TEZ_ENGINE_INTERMEDIATE_INPUT_VALUE_CLASS =
- "tez.engine.intermediate-input.value.class";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS =
+ "tez.runtime.intermediate-output.value.class";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_INPUT_VALUE_CLASS =
+ "tez.runtime.intermediate-input.value.class";
- public static final String TEZ_ENGINE_INTERMEDIATE_OUTPUT_SHOULD_COMPRESS =
- "tez.engine.intermediate-output.should-compress";
- public static final String TEZ_ENGINE_INTERMEDIATE_INPUT_IS_COMPRESSED =
- "tez.engine.intermdiate-input.is-compressed";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_OUTPUT_SHOULD_COMPRESS =
+ "tez.runtime.intermediate-output.should-compress";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_INPUT_IS_COMPRESSED =
+ "tez.runtime.intermdiate-input.is-compressed";
- public static final String TEZ_ENGINE_INTERMEDIATE_OUTPUT_COMPRESS_CODEC =
- "tez.engine.intermediate-output.compress.codec";
- public static final String TEZ_ENGINE_INTERMEDIATE_INPUT_COMPRESS_CODEC =
- "tez.engine.intermediate-input.compress.codec";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_OUTPUT_COMPRESS_CODEC =
+ "tez.runtime.intermediate-output.compress.codec";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_INPUT_COMPRESS_CODEC =
+ "tez.runtime.intermediate-input.compress.codec";
- public static final String TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS =
- "tez.engine.intermediate-input.key.secondary.comparator.class";
+ public static final String TEZ_RUNTIME_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS =
+ "tez.runtime.intermediate-input.key.secondary.comparator.class";
// TODO This should be in DAGConfiguration
/* config for tracking the local file where all the credentials for the job
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/Event.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/Event.java b/tez-api/src/main/java/org/apache/tez/engine/api/Event.java
deleted file mode 100644
index 80da655..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/Event.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * Base class for all events generated within the Tez execution engine.
- * Used as the primary mode of communication between the AM, Inputs, Processors
- * and Outputs.
- */
-public abstract class Event {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/Input.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/Input.java b/tez-api/src/main/java/org/apache/tez/engine/api/Input.java
deleted file mode 100644
index e333075..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/Input.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-import java.util.List;
-
-/**
- * Represents an input through which a TezProcessor receives data on an edge.
- * </p>
- *
- * <code>Input</code> classes must have a 0 argument public constructor for Tez
- * to construct the <code>Input</code>. Tez will take care of initializing and
- * closing the Input after a {@link Processor} completes. </p>
- */
-public interface Input {
-
- /**
- * Initializes the <code>Input</code>.
- *
- * @param inputContext
- * the {@link TezInputContext}
- * @return
- * @throws Exception
- * if an error occurs
- */
- public List<Event> initialize(TezInputContext inputContext)
- throws Exception;
-
- /**
- * Gets an instance of the {@link Reader} for this <code>Output</code>
- *
- * @return
- * @throws Exception
- * if an error occurs
- */
- public Reader getReader() throws Exception;
-
- /**
- * Handles user and system generated {@link Events}s, which typically carry
- * information such as an output being available on the previous vertex.
- *
- * @param inputEvents
- * the list of {@link Event}s
- */
- public void handleEvents(List<Event> inputEvents);
-
- /**
- * Closes the <code>Input</code>
- *
- * @return
- * @throws Exception
- * if an error occurs
- */
- public List<Event> close() throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/LogicalIOProcessor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/LogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/engine/api/LogicalIOProcessor.java
deleted file mode 100644
index 90be09e..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/LogicalIOProcessor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-import java.util.Map;
-
-/**
- * Represents a processor which consumes {@link LogicalInput}s and produces
- * {@link LogicalOutput}s
- */
-public interface LogicalIOProcessor extends Processor {
-
- /**
- * Runs the {@link LogicalProcessor}
- *
- * @param inputs
- * a map of the source vertex name to {@link LogicalInput} - one per
- * incoming edge.
- * @param outputs
- * a map of the destination vertex name to {@link LogicalOutput} -
- * one per outgoing edge
- * @throws Exception TODO
- */
- public void run(Map<String, LogicalInput> inputs,
- Map<String, LogicalOutput> outputs) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/LogicalInput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/LogicalInput.java b/tez-api/src/main/java/org/apache/tez/engine/api/LogicalInput.java
deleted file mode 100644
index 4a47ccf..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/LogicalInput.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * An @link {@link Input} which handles all incoming physical connections on an
- * edge. A {@link LogicalIOProcessor} sees a single Logical Input per incoming
- * edge.
- */
-public interface LogicalInput extends Input {
-
- /**
- * Sets the number of physical inputs that this <code>LogicalInput</code> will
- * receive. This will be called by the Tez framework before initializing the
- * <code>LogicalInput</code>
- *
- * @param numInputs
- * the number of physical inputs.
- */
- public void setNumPhysicalInputs(int numInputs);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/LogicalOutput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/LogicalOutput.java b/tez-api/src/main/java/org/apache/tez/engine/api/LogicalOutput.java
deleted file mode 100644
index 4626fbd..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/LogicalOutput.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * An @link {@link Output} which handles all outgoing physical connections on an
- * edge. A {@link LogicalIOProcessor} sees a single Logical Output per outgoing
- * edge.
- */
-public interface LogicalOutput extends Output {
- /**
- * Sets the number of physical ouputs that this <code>LogicalOutput</code>
- * will receive. This will be called by the Tez framework before initializing
- * the <code>LogicalOutput</code>
- *
- * @param numOutputs
- * the number of physical outputs
- */
- public void setNumPhysicalOutputs(int numOutputs);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/Output.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/Output.java b/tez-api/src/main/java/org/apache/tez/engine/api/Output.java
deleted file mode 100644
index ec679ed..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/Output.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-import java.util.List;
-
-/**
- * Represents an Output through which a TezProcessor writes information on an
- * edge. </p>
- *
- * <code>Output</code> implementations must have a 0 argument public constructor
- * for Tez to construct the <code>Output</code>. Tez will take care of
- * initializing and closing the Input after a {@link Processor} completes. </p>
- */
-public interface Output {
-
- /**
- * Initializes the <code>Output</code>
- *
- * @param outputContext
- * the {@link TezOutputContext}
- * @return
- * @throws Exception
- * if an error occurs
- */
- public List<Event> initialize(TezOutputContext outputContext)
- throws Exception;
-
- /**
- * Gets an instance of the {@link Writer} in an <code>Output</code>
- *
- * @return
- * @throws Exception
- * if an error occurs
- */
- public Writer getWriter() throws Exception;
-
- /**
- * Handles user and system generated {@link Events}s, which typically carry
- * information such as a downstream vertex being ready to consume input.
- *
- * @param outputEvents
- * the list of {@link Event}s
- */
- public void handleEvents(List<Event> outputEvents);
-
- /**
- * Closes the <code>Output</code>
- *
- * @return
- * @throws Exception
- * if an error occurs
- */
- public List<Event> close() throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/Processor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/Processor.java b/tez-api/src/main/java/org/apache/tez/engine/api/Processor.java
deleted file mode 100644
index 05e6b84..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/Processor.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * {@link Processor} represents the <em>Tez</em> entity responsible for
- * consuming {@link Input} and producing {@link Output}.
- */
-public interface Processor {
-
- /**
- * Initializes the <code>Processor</code>
- *
- * @param processorContext
- * @throws IOException
- * if an error occurs
- */
- public void initialize(TezProcessorContext processorContext)
- throws Exception;
-
- /**
- * Handles user and system generated {@link Events}s.
- *
- * @param processorEvents
- * the list of {@link Event}s
- */
- public void handleEvents(List<Event> processorEvents);
-
- /**
- * Closes the <code>Processor</code>
- *
- * @throws IOException
- * if an error occurs
- */
- public void close() throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/Reader.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/Reader.java b/tez-api/src/main/java/org/apache/tez/engine/api/Reader.java
deleted file mode 100644
index 502c5f2..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/Reader.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * A <code>Reader</code> represents the data being read in an {@link Input}
- */
-public interface Reader {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/TezInputContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/TezInputContext.java b/tez-api/src/main/java/org/apache/tez/engine/api/TezInputContext.java
deleted file mode 100644
index ddf1ff8..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/TezInputContext.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * Context handle for the Input to initialize itself.
- */
-public interface TezInputContext extends TezTaskContext {
-
- /**
- * Get the Vertex Name of the Source that generated data for this Input
- * @return Name of the Source Vertex
- */
- public String getSourceVertexName();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/TezOutputContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/TezOutputContext.java b/tez-api/src/main/java/org/apache/tez/engine/api/TezOutputContext.java
deleted file mode 100644
index 791a0f0..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/TezOutputContext.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * Context handle for the Output to initialize itself.
- */
-public interface TezOutputContext extends TezTaskContext {
-
- /**
- * Get the Vertex Name of the Destination that is the recipient of this
- * Output's data
- * @return Name of the Destination Vertex
- */
- public String getDestinationVertexName();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/TezProcessorContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/TezProcessorContext.java b/tez-api/src/main/java/org/apache/tez/engine/api/TezProcessorContext.java
deleted file mode 100644
index 2bbbe81..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/TezProcessorContext.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-import java.io.IOException;
-
-/**
- * Context handle for the Processor to initialize itself.
- */
-public interface TezProcessorContext extends TezTaskContext {
-
- /**
- * Set the overall progress of this Task Attempt
- * @param progress Progress in the range from [0.0 - 1.0f]
- */
- public void setProgress(float progress);
-
- /**
- * Check whether this attempt can commit its output
- * @return true if commit allowed
- * @throws IOException
- */
- public boolean canCommit() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/TezTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/TezTaskContext.java b/tez-api/src/main/java/org/apache/tez/engine/api/TezTaskContext.java
deleted file mode 100644
index 706e646..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/TezTaskContext.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.common.counters.TezCounters;
-
-/**
- * Base interface for Context classes used to initialize the Input, Output
- * and Processor instances.
- */
-public interface TezTaskContext {
-
- // TODO NEWTEZ
- // Scale the maximum events we fetch per RPC call to mitigate OOM issues
- // on the ApplicationMaster when a thundering herd of reducers fetch events
- // This should not be necessary after HADOOP-8942
-
- /**
- * Get the {@link ApplicationId} for the running app
- * @return the {@link ApplicationId}
- */
- public ApplicationId getApplicationId();
-
- /**
- * Get the current DAG Attempt Number
- * @return DAG Attempt Number
- */
- public int getDAGAttemptNumber();
-
- /**
- * Get the index of this Task
- * @return Task Index
- */
- public int getTaskIndex();
-
- /**
- * Get the current Task Attempt Number
- * @return Task Attempt Number
- */
- public int getTaskAttemptNumber();
-
- /**
- * Get the name of the DAG
- * @return the DAG name
- */
- public String getDAGName();
-
- /**
- * Get the name of the Vertex in which the task is running
- * @return Vertex Name
- */
- public String getTaskVertexName();
-
- public TezCounters getCounters();
-
- /**
- * Send Events to the AM and/or dependent Vertices
- * @param events Events to be sent
- */
- public void sendEvents(List<Event> events);
-
- /**
- * Get the User Payload for the Input/Output/Processor
- * @return User Payload
- */
- public byte[] getUserPayload();
-
- /**
- * Get the work diectories for the Input/Output/Processor
- * @return an array of work dirs
- */
- public String[] getWorkDirs();
-
- /**
- * Returns an identifier which is unique to the specific Input, Processor or
- * Output
- *
- * @return
- */
- public String getUniqueIdentifier();
-
- /**
- * Report a fatal error to the framework. This will cause the entire task to
- * fail and should not be used for reporting temporary or recoverable errors
- *
- * @param exception an exception representing the error
- */
- public void fatalError(Throwable exception, String message);
-
- /**
- * Returns meta-data for the specified service. As an example, when the MR
- * ShuffleHandler is used - this would return the jobToken serialized as bytes
- *
- * @param serviceName
- * the name of the service for which meta-data is required
- * @return a ByteBuffer representing the meta-data
- */
- public ByteBuffer getServiceConsumerMetaData(String serviceName);
-
- /**
- * Return Provider meta-data for the specified service As an example, when the
- * MR ShuffleHandler is used - this would return the shuffle port serialized
- * as bytes
- *
- * @param serviceName
- * the name of the service for which provider meta-data is required
- * @return a ByteBuffer representing the meta-data
- */
- public ByteBuffer getServiceProviderMetaData(String serviceName);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/Writer.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/Writer.java b/tez-api/src/main/java/org/apache/tez/engine/api/Writer.java
deleted file mode 100644
index c9503a3..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/Writer.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api;
-
-/**
- * A <code>Writer</code> represents the data being written by an {@link Output}
- */
-public interface Writer {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/events/DataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/events/DataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/engine/api/events/DataMovementEvent.java
deleted file mode 100644
index 3f35555..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/events/DataMovementEvent.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api.events;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.tez.engine.api.Event;
-
-/**
- * Event used by user code to send information between tasks. An output can
- * generate an Event of this type to sending information regarding output data
- * ( such as URI for file-based output data, port info in case of
- * streaming-based data transfers ) to the Input on the destination vertex.
- */
-public final class DataMovementEvent extends Event {
-
- /**
- * Index(i) of the i-th (physical) Input or Output that generated an Event.
- * For a Processor-generated event, this is ignored.
- */
- private final int sourceIndex;
-
- /**
- * Index(i) of the i-th (physical) Input or Output that is meant to receive
- * this Event. For a Processor event, this is ignored.
- */
- private int targetIndex;
-
- /**
- * User Payload for this Event
- */
- private final byte[] userPayload;
-
- /**
- * Version number to indicate what attempt generated this Event
- */
- private int version;
-
- /**
- * User Event constructor
- * @param sourceIndex Index to identify the physical edge of the input/output
- * that generated the event
- * @param userPayload User Payload of the User Event
- */
- public DataMovementEvent(int sourceIndex,
- byte[] userPayload) {
- this.userPayload = userPayload;
- this.sourceIndex = sourceIndex;
- }
-
- @Private
- public DataMovementEvent(int sourceIndex,
- int targetIndex,
- byte[] userPayload) {
- this.userPayload = userPayload;
- this.sourceIndex = sourceIndex;
- this.targetIndex = targetIndex;
- }
-
- /**
- * Constructor for Processor-generated User Events
- * @param userPayload
- */
- public DataMovementEvent(byte[] userPayload) {
- this(-1, userPayload);
- }
-
- public byte[] getUserPayload() {
- return userPayload;
- }
-
- public int getSourceIndex() {
- return sourceIndex;
- }
-
- public int getTargetIndex() {
- return targetIndex;
- }
-
- @Private
- public void setTargetIndex(int targetIndex) {
- this.targetIndex = targetIndex;
- }
-
- public int getVersion() {
- return version;
- }
-
- @Private
- public void setVersion(int version) {
- this.version = version;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/events/InputFailedEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/events/InputFailedEvent.java b/tez-api/src/main/java/org/apache/tez/engine/api/events/InputFailedEvent.java
deleted file mode 100644
index 57de09b..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/events/InputFailedEvent.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api.events;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.tez.engine.api.Event;
-
-/**
- * Event sent from the AM to an Input to indicate that one of it's sources has
- * failed - effectively the input is no longer available from the particular
- * source.
- * Users are not expected to send this event.
- */
-public class InputFailedEvent extends Event{
-
- /**
- * Index(i) of the i-th (physical) Input or Output that generated the data.
- * For a Processor-generated event, this is ignored.
- */
- private final int sourceIndex;
-
- /**
- * Index(i) of the i-th (physical) Input or Output that is meant to receive
- * this Event. For a Processor event, this is ignored.
- */
- private int targetIndex;
-
- /**
- * Version number to indicate what attempt generated this Event
- */
- private int version;
-
- /**
- * User Event constructor
- * @param sourceIndex Index to identify the physical edge of the input/output
- * that generated the event
- * @param userPayload User Payload of the User Event
- */
- public InputFailedEvent(int sourceIndex) {
- this.sourceIndex = sourceIndex;
- }
-
- @Private
- public InputFailedEvent(int sourceIndex,
- int targetIndex,
- int version) {
- this.sourceIndex = sourceIndex;
- this.targetIndex = targetIndex;
- this.version = version;
- }
-
- public int getSourceIndex() {
- return sourceIndex;
- }
-
- public int getTargetIndex() {
- return targetIndex;
- }
-
- @Private
- public void setTargetIndex(int targetIndex) {
- this.targetIndex = targetIndex;
- }
-
- public int getVersion() {
- return version;
- }
-
- @Private
- public void setVersion(int version) {
- this.version = version;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/events/InputInformationEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/events/InputInformationEvent.java b/tez-api/src/main/java/org/apache/tez/engine/api/events/InputInformationEvent.java
deleted file mode 100644
index 3656d45..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/events/InputInformationEvent.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api.events;
-
-import org.apache.tez.engine.api.Event;
-
-/**
- * Event used to send user specific data from the user
- * code in the AM to the task input
- */
-public class InputInformationEvent extends Event {
-
- /**
- * User Payload for this Event
- */
- private final byte[] userPayload;
- public InputInformationEvent(byte[] userPayload) {
- this.userPayload = userPayload;
- }
-
- public byte[] getUserPayload() {
- return userPayload;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/api/events/InputReadErrorEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/api/events/InputReadErrorEvent.java b/tez-api/src/main/java/org/apache/tez/engine/api/events/InputReadErrorEvent.java
deleted file mode 100644
index fa49b79..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/api/events/InputReadErrorEvent.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.api.events;
-
-import org.apache.tez.engine.api.Event;
-
-/**
- * Event generated by an Input to indicate error when trying to retrieve data.
- * This is not necessarily a fatal event - it's an indication to the AM to retry
- * source data generation.
- */
-public final class InputReadErrorEvent extends Event {
-
- /**
- * Diagnostics/trace of the error that occurred on the Input's edge.
- */
- private final String diagnostics;
-
- /**
- * Index of the physical edge on which the error occurred.
- */
- private final int index;
-
- /**
- * Version of the data on which the error occurred.
- */
- private final int version;
-
- public InputReadErrorEvent(String diagnostics, int index,
- int version) {
- super();
- this.diagnostics = diagnostics;
- this.index = index;
- this.version = version;
- }
-
- public String getDiagnostics() {
- return diagnostics;
- }
-
- public int getIndex() {
- return index;
- }
-
- public int getVersion() {
- return version;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectLifeCycle.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectLifeCycle.java b/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectLifeCycle.java
deleted file mode 100644
index 7099299..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectLifeCycle.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.common.objectregistry;
-
-/**
- * Defines the valid lifecycle and scope of Objects stored in ObjectRegistry.
- * Objects are guaranteed to not be valid outside of their defined life-cycle
- * period. Objects are not guaranteed to be retained through the defined period
- * as they may be evicted for various reasons.
- */
-public enum ObjectLifeCycle {
- /** Objects are valid for the lifetime of the Tez JVM/Session
- */
- SESSION,
- /** Objects are valid for the lifetime of the DAG.
- */
- DAG,
- /** Objects are valid for the lifetime of the Vertex.
- */
- VERTEX,
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistry.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistry.java b/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistry.java
deleted file mode 100644
index a27903d..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistry.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.common.objectregistry;
-
-/**
- * Preliminary version of a simple shared object cache to re-use
- * objects across multiple tasks within the same container/JVM.
- */
-public interface ObjectRegistry {
-
- /**
- * Insert or update object into the registry. This will remove an object
- * associated with the same key with a different life-cycle as there is only
- * one instance of an Object stored for a given key irrespective of the
- * life-cycle attached to the Object.
- * @param lifeCycle What life-cycle is the Object valid for
- * @param key Key to identify the Object
- * @param value Object to be inserted
- * @return Previous Object associated with the key attached if present
- * else null. Could return the same object if the object was associated with
- * the same key for a different life-cycle.
- */
- public Object add(ObjectLifeCycle lifeCycle, String key, Object value);
-
- /**
- * Return the object associated with the provided key
- * @param key Key to find object
- * @return Object if found else null
- */
- public Object get(String key);
-
- /**
- * Delete the object associated with the provided key
- * @param lifeCycle What life-cycle is the Object valid for
- * @param key Key to find object
- * @return True if an object was found and removed
- */
- public boolean delete(String key);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistryFactory.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistryFactory.java b/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistryFactory.java
deleted file mode 100644
index 94352b3..0000000
--- a/tez-api/src/main/java/org/apache/tez/engine/common/objectregistry/ObjectRegistryFactory.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.common.objectregistry;
-
-import com.google.inject.Inject;
-
-public class ObjectRegistryFactory {
-
- @Inject
- private static ObjectRegistry objectRegistry;
-
- public static ObjectRegistry getObjectRegistry() {
- return objectRegistry;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
new file mode 100644
index 0000000..6f3d667
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+/**
+ * Base class for all events generated within the Tez execution engine.
+ * Used as the primary mode of communication between the AM, Inputs, Processors
+ * and Outputs.
+ */
+public abstract class Event {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
new file mode 100644
index 0000000..a15c106
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import java.util.List;
+
+/**
+ * Represents an input through which a TezProcessor receives data on an edge.
+ * </p>
+ *
+ * <code>Input</code> classes must have a 0 argument public constructor for Tez
+ * to construct the <code>Input</code>. Tez will take care of initializing and
+ * closing the Input after a {@link Processor} completes. </p>
+ */
+public interface Input {
+
+ /**
+ * Initializes the <code>Input</code>.
+ *
+ * @param inputContext
+ * the {@link TezInputContext}
+ * @return
+ * @throws Exception
+ * if an error occurs
+ */
+ public List<Event> initialize(TezInputContext inputContext)
+ throws Exception;
+
+ /**
+ * Gets an instance of the {@link Reader} for this <code>Output</code>
+ *
+ * @return
+ * @throws Exception
+ * if an error occurs
+ */
+ public Reader getReader() throws Exception;
+
+ /**
+ * Handles user and system generated {@link Events}s, which typically carry
+ * information such as an output being available on the previous vertex.
+ *
+ * @param inputEvents
+ * the list of {@link Event}s
+ */
+ public void handleEvents(List<Event> inputEvents);
+
+ /**
+ * Closes the <code>Input</code>
+ *
+ * @return
+ * @throws Exception
+ * if an error occurs
+ */
+ public List<Event> close() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b212ca1d/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
new file mode 100644
index 0000000..9372c70
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import java.util.Map;
+
+/**
+ * Represents a processor which consumes {@link LogicalInput}s and produces
+ * {@link LogicalOutput}s
+ */
+public interface LogicalIOProcessor extends Processor {
+
+ /**
+ * Runs the {@link LogicalProcessor}
+ *
+ * @param inputs
+ * a map of the source vertex name to {@link LogicalInput} - one per
+ * incoming edge.
+ * @param outputs
+ * a map of the destination vertex name to {@link LogicalOutput} -
+ * one per outgoing edge
+ * @throws Exception TODO
+ */
+ public void run(Map<String, LogicalInput> inputs,
+ Map<String, LogicalOutput> outputs) throws Exception;
+
+}