You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2014/12/10 04:33:30 UTC
[07/53] tez git commit: Upmerging to master
Upmerging to master
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fb365f41
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fb365f41
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fb365f41
Branch: refs/heads/master
Commit: fb365f41e3ac2f8dad5065d93644a326206bcda2
Parents: ca3015d 2103129
Author: Jonathan Eagles <je...@gmail.com>
Authored: Thu Oct 2 15:45:29 2014 -0500
Committer: Jonathan Eagles <je...@gmail.com>
Committed: Thu Oct 2 15:45:29 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 55 +-
.../java/org/apache/tez/client/TezClient.java | 6 +-
.../org/apache/tez/client/TezClientUtils.java | 59 +--
.../org/apache/tez/common/ATSConstants.java | 94 ++++
.../main/java/org/apache/tez/dag/api/DAG.java | 136 ++---
.../tez/dag/api/DAGNotRunningException.java | 34 ++
.../apache/tez/dag/api/DagTypeConverters.java | 29 ++
.../apache/tez/dag/api/DataSinkDescriptor.java | 48 ++
.../tez/dag/api/DataSourceDescriptor.java | 44 ++
.../java/org/apache/tez/dag/api/Vertex.java | 7 +-
.../tez/dag/api/client/DAGClientImpl.java | 477 +++++++++++++++++
.../dag/api/client/DAGClientTimelineImpl.java | 508 ++++++++++++++++++
.../apache/tez/dag/api/client/VertexStatus.java | 2 +-
.../dag/api/client/rpc/DAGClientRPCImpl.java | 228 +-------
tez-api/src/main/proto/DAGApiRecords.proto | 1 +
.../org/apache/tez/client/TestTezClient.java | 37 +-
.../apache/tez/client/TestTezClientUtils.java | 4 +-
.../apache/tez/common/TestTezCommonUtils.java | 54 +-
.../org/apache/tez/dag/api/TestDAGPlan.java | 8 +-
.../org/apache/tez/dag/api/TestDAGVerify.java | 81 +--
.../tez/dag/api/client/TestATSHttpClient.java | 167 ++++++
.../tez/dag/api/client/rpc/TestDAGClient.java | 24 +-
.../tez/common/EnvironmentUpdateUtils.java | 127 +++++
.../tez/dag/utils/EnvironmentUpdateUtils.java | 92 ----
.../tez/common/TestEnvironmentUpdateUtils.java | 34 ++
.../dag/utils/TestEnvironmentUpdateUtils.java | 34 --
.../java/org/apache/tez/client/LocalClient.java | 2 +-
.../tez/dag/api/client/DAGClientHandler.java | 25 +-
.../java/org/apache/tez/dag/app/AppContext.java | 3 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 7 +
.../org/apache/tez/dag/app/RecoveryParser.java | 1 +
.../java/org/apache/tez/dag/app/dag/DAG.java | 2 +
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 11 +
.../dag/impl/ImmediateStartVertexManager.java | 9 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 97 ++--
.../app/launcher/LocalContainerLauncher.java | 6 +-
.../dag/app/rm/LocalTaskSchedulerService.java | 23 +-
.../dag/app/rm/YarnTaskSchedulerService.java | 122 ++---
.../app/rm/container/AMContainerHelpers.java | 9 +-
.../dag/app/rm/container/AMContainerImpl.java | 8 +-
.../tez/dag/history/events/DAGStartedEvent.java | 4 +
.../dag/history/events/VertexFinishedEvent.java | 24 +-
.../dag/history/events/VertexStartedEvent.java | 4 +
.../impl/HistoryEventJsonConversion.java | 2 +-
.../tez/dag/history/utils/ATSConstants.java | 76 ---
.../apache/tez/dag/history/utils/DAGUtils.java | 1 +
.../apache/tez/runtime/task/TezTaskRunner.java | 29 +-
.../apache/tez/dag/app/MockDAGAppMaster.java | 18 +-
.../tez/dag/app/TestMockDAGAppMaster.java | 138 +++++
.../tez/dag/app/dag/impl/TestDAGImpl.java | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 153 +++++-
.../dag/app/dag/impl/TestVertexRecovery.java | 4 +-
.../app/rm/TestLocalTaskSchedulerService.java | 68 +++
.../TestHistoryEventsProtoConversion.java | 4 +-
.../impl/TestHistoryEventJsonConversion.java | 2 +-
.../tez/dag/history/utils/TestDAGUtils.java | 2 +-
.../org/apache/tez/examples/ExampleDriver.java | 6 +-
.../apache/tez/examples/HashJoinExample.java | 403 +++++++++++++++
.../org/apache/tez/examples/JoinDataGen.java | 1 -
.../org/apache/tez/examples/JoinExample.java | 364 -------------
.../org/apache/tez/examples/JoinValidate.java | 4 +-
.../tez/examples/SortMergeJoinExample.java | 374 ++++++++++++++
.../org/apache/tez/mapreduce/input/MRInput.java | 30 +-
.../apache/tez/mapreduce/output/MROutput.java | 16 +-
.../tez/mapreduce/hadoop/TestMRHelpers.java | 20 +-
.../ats/HistoryEventTimelineConversion.java | 12 +-
.../ats/TestHistoryEventTimelineConversion.java | 2 +-
.../vertexmanager/ShuffleVertexManager.java | 73 ++-
.../library/common/shuffle/impl/Fetcher.java | 5 +-
.../runtime/library/common/sort/impl/IFile.java | 14 +-
.../library/shuffle/common/HttpConnection.java | 8 +-
.../shuffle/common/impl/ShuffleManager.java | 4 +-
.../src/main/proto/ShufflePayloads.proto | 5 +-
.../vertexmanager/TestShuffleVertexManager.java | 109 +++-
.../library/output/TestOnFileSortedOutput.java | 2 +-
.../library/shuffle/common/TestFetcher.java | 2 +-
.../tez/mapreduce/examples/MRRSleepJob.java | 14 +-
.../examples/TestOrderedWordCount.java | 9 +
.../org/apache/tez/test/MiniTezCluster.java | 43 ++
.../org/apache/tez/test/TestAMRecovery.java | 4 +-
.../org/apache/tez/test/TestDAGRecovery.java | 2 -
.../org/apache/tez/test/TestDAGRecovery2.java | 2 -
.../tez/test/TestExceptionPropagation.java | 516 +++++++++++++++++++
.../java/org/apache/tez/test/TestTezJobs.java | 149 +++++-
84 files changed, 4181 insertions(+), 1258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/fb365f41/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 8dabca0,0be2a65..ced524a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -19,17 -38,19 +38,21 @@@ ALL CHANGE
TEZ-1580. Change TestOrderedWordCount to optionally use MR configs.
TEZ-1524. Resolve user group information only if ACLs are enabled.
TEZ-1581. GroupByOrderByMRRTest no longer functional.
- TEZ-1563. TezClient.submitDAGSession alters DAG local resources regardless of DAG submission.
TEZ-1157. Optimize broadcast shuffle to download data only once per host.
+ TEZ-1594. Initial TezUI into TEZ-8 branch
+ TEZ-1595. Document timeline server setup for use with the Tez UI
-
- Release 0.5.1: Unreleased
-
- INCOMPATIBLE CHANGES
- TEZ-1539. Change InputInitializerEvent semantics to SEND_ONCE_ON_TASK_SUCCESS
-
- ALL CHANGES
+ TEZ-1607. support mr envs in mrrsleep and testorderedwordcount
+ TEZ-1499. Add SortMergeJoinExample to tez-examples
+ TEZ-1613. Decrease running time for TestAMRecovery
+ TEZ-1240. Add system test for propagation of diagnostics for errors
+ TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can get
+ negative if JVM memory is larger than 2GB
+ TEZ-1611. Change DataSource/Sink to be able to supply URIs for credentials
+ TEZ-1592. Vertex should wait for all initializers to finish before moving to INITED state
+ TEZ-1612. ShuffleVertexManager's EdgeManager should not hard code source num tasks
+ TEZ-1555. TestTezClientUtils.validateSetTezJarLocalResourcesDefinedButEmpty
+ failing on Windows
+ TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging
TEZ-1494. DAG hangs waiting for ShuffleManager.getNextInput()
TEZ-1515. Remove usage of ResourceBundles in Counters.
TEZ-1527. Fix indentation of Vertex status in DAGClient output.