You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [2/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt Tue Aug 19 23:49:39 2014
@@ -15,6 +15,208 @@ Trunk - Unreleased
YARN-524 TestYarnVersionInfo failing if generated properties doesn't
include an SVN URL. (stevel)
+ YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk.
+ (Zhijie Shen via xgong)
+
+Release 2.6.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ YARN-2131. Add a way to format the RMStateStore. (Robert Kanter via kasha)
+
+ YARN-1367. Changed NM to not kill containers on NM resync if RM work-preserving
+ restart is enabled. (Anubhav Dhoot via jianhe)
+
+ YARN-1366. Changed AMRMClient to re-register with RM and send outstanding requests
+ back to RM on work-preserving RM restart. (Rohith via jianhe)
+
+ YARN-2181. Added preemption info to logs and RM web UI. (Wangda Tan via
+ jianhe)
+
+ YARN-1354. Recover applications upon nodemanager restart. (Jason Lowe via
+ junping_du)
+
+ YARN-1337. Recover containers upon nodemanager restart. (Jason Lowe via
+ junping_du)
+
+ YARN-2277. Added cross-origin support for the timeline server web services.
+ (Jonathan Eagles via zjshen)
+
+ YARN-2378. Added support for moving applications across queues in
+ CapacityScheduler. (Subramaniam Venkatraman Krishnan via jianhe)
+
+ YARN-2411. Support simple user and group mappings to queues. (Ram Venkatesh
+ via jianhe)
+
+ IMPROVEMENTS
+
+ YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
+ (Akira AJISAKA via aw)
+
+ YARN-1918. Typo in description and error message for
+ 'yarn.resourcemanager.cluster-id' (Anandha L Ranganathan via aw)
+
+ YARN-2242. Improve exception information on AM launch crashes. (Li Lu
+ via junping_du)
+
+ YARN-2274. FairScheduler: Add debug information about cluster capacity,
+ availability and reservations. (kasha)
+
+ YARN-2228. Augmented TimelineServer to load pseudo authentication filter when
+ authentication = simple. (Zhijie Shen via vinodkv)
+
+ YARN-1341. Recover NMTokens upon nodemanager restart. (Jason Lowe via
+ junping_du)
+
+ YARN-2208. AMRMTokenManager need to have a way to roll over AMRMToken. (xgong)
+
+ YARN-2323. FairShareComparator creates too many Resource objects (Hong Zhiguo
+ via Sandy Ryza)
+
+ YARN-2045. Data persisted in NM should be versioned (Junping Du via jlowe)
+
+ YARN-2013. The diagnostics is always the ExitCodeException stack when the container
+ crashes. (Tsuyoshi OZAWA via junping_du)
+
+ YARN-2295. Refactored DistributedShell to use public APIs of protocol records.
+ (Li Lu via jianhe)
+
+ YARN-1342. Recover container tokens upon nodemanager restart. (Jason Lowe via
+ devaraj)
+
+ YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue delays
+ convergence towards fairness. (Ashwin Shankar via kasha)
+
+ YARN-2211. Persist AMRMToken master key in RMStateStore for RM recovery.
+ (Xuan Gong via jianhe)
+
+ YARN-2328. FairScheduler: Verify update and continuous scheduling threads are
+ stopped when the scheduler is stopped. (kasha)
+
+ YARN-2347. Consolidated RMStateVersion and NMDBSchemaVersion into Version in
+ yarn-server-common. (Junping Du via zjshen)
+
+ YARN-1994. Expose YARN/MR endpoints on multiple interfaces. (Craig Welch,
+ Milan Potocnik, Arpit Agarwal via xgong)
+
+ YARN-2343. Improve NMToken expire exception message. (Li Lu via jianhe)
+
+ YARN-2370. Fix comment in o.a.h.y.server.resourcemanager.schedulerAppSchedulingInfo
+ (Wenwu Peng via junping_du)
+
+ YARN-2298. Move TimelineClient to yarn-common project (Zhijie Shen via
+ junping_du)
+
+ YARN-2288. Made persisted data in LevelDB timeline store be versioned. (Junping Du
+ via zjshen)
+
+ YARN-2352. FairScheduler: Collect metrics on duration of critical methods that
+ affect performance. (kasha)
+
+ YARN-2212. ApplicationMaster needs to find a way to update the AMRMToken
+ periodically. (xgong)
+
+ YARN-2026. Fair scheduler: Consider only active queues for computing fairshare.
+ (Ashwin Shankar via kasha)
+
+ YARN-1954. Added waitFor to AMRMClient(Async). (Tsuyoshi Ozawa via zjshen)
+
+ YARN-2302. Refactor TimelineWebServices. (Zhijie Shen via junping_du)
+
+ YARN-2337. ResourceManager sets ClientRMService in RMContext multiple times.
+ (Zhihai Xu via kasha)
+
+ YARN-2138. Cleaned up notifyDone* APIs in RMStateStore. (Varun Saxena via
+ jianhe)
+
+ YARN-2373. Changed WebAppUtils to use Configuration#getPassword for
+ accessing SSL passwords. (Larry McCay via jianhe)
+
+ YARN-2317. Updated the document about how to write YARN applications. (Li Lu via
+ zjshen)
+
+ YARN-2399. FairScheduler: Merge AppSchedulable and FSSchedulerApp into
+ FSAppAttempt. (kasha)
+
+ YARN-1370. Fair scheduler to re-populate container allocation state.
+ (Anubhav Dhoot via kasha)
+
+ YARN-2389. Added functionality for schedulers to kill all applications in a
+ queue. (Subramaniam Venkatraman Krishnan via jianhe)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ YARN-2251. Avoid negative elapsed time in JHS/MRAM web UI and services.
+ (Zhijie Shen via junping_du)
+
+ YARN-2088. Fixed a bug in GetApplicationsRequestPBImpl#mergeLocalToBuilder.
+ (Binglin Chang via jianhe)
+
+ YARN-2260. Fixed ResourceManager's RMNode to correctly remember containers
+ when nodes resync during work-preserving RM restart. (Jian He via vinodkv)
+
+ YARN-2264. Fixed a race condition in DrainDispatcher which may cause random
+ test failures. (Li Lu via jianhe)
+
+ YARN-2219. Changed ResourceManager to avoid AMs and NMs getting exceptions
+ after RM recovery but before scheduler learns about apps and app-attempts.
+ (Jian He via vinodkv)
+
+ YARN-2244. FairScheduler missing handling of containers for unknown
+ application attempts. (Anubhav Dhoot via kasha)
+
+ YARN-2321. NodeManager web UI can incorrectly report Pmem enforcement
+ (Leitao Guo via jlowe)
+
+ YARN-2273. NPE in ContinuousScheduling thread when we lose a node.
+ (Wei Yan via kasha)
+
+ YARN-2313. Livelock can occur in FairScheduler when there are lots of
+ running apps (Tsuyoshi Ozawa via Sandy Ryza)
+
+ YARN-2147. client lacks delegation token exception details when
+ application submit fails (Chen He via jlowe)
+
+ YARN-1796. container-executor shouldn't require o-r permissions (atm)
+
+ YARN-2354. DistributedShell may allocate more containers than client
+ specified after AM restarts. (Li Lu via jianhe)
+
+ YARN-2051. Fix bug in PBimpls and add more unit tests with reflection.
+ (Binglin Chang via junping_du)
+
+ YARN-2374. Fixed TestDistributedShell#testDSShell failure due to hostname
+ dismatch. (Varun Vasudev via jianhe)
+
+ YARN-2359. Application hangs when it fails to launch AM container.
+ (Zhihai Xu via kasha)
+
+ YARN-2388. Fixed TestTimelineWebServices failure due to HADOOP-10791. (zjshen)
+
+ YARN-2008. Fixed CapacityScheduler to calculate headroom based on max available
+ capacity instead of configured max capacity. (Craig Welch via jianhe)
+
+ YARN-2400. Fixed TestAMRestart fails intermittently. (Jian He via xgong)
+
+ YARN-2361. RMAppAttempt state machine entries for KILLED state has duplicate
+ event entries. (Zhihai Xu via kasha)
+
+ YARN-2070. Made DistributedShell publish the short user name to the timeline
+ server. (Robert Kanter via zjshen)
+
+ YARN-2397. Avoided loading two authentication filters for RM and TS web
+ interfaces. (Varun Vasudev via zjshen)
+
+ YARN-2409. RM ActiveToStandBy transition missing stoping previous rmDispatcher.
+ (Rohith via jianhe)
+
+ YARN-2249. Avoided AM release requests being lost on work preserving RM
+ restart. (Jian He via zjshen)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -23,6 +225,40 @@ Release 2.5.0 - UNRELEASED
YARN-1757. NM Recovery. Auxiliary service support. (Jason Lowe via kasha)
+ YARN-1864. Fair Scheduler Dynamic Hierarchical User Queues (Ashwin Shankar
+ via Sandy Ryza)
+
+ YARN-1362. Distinguish between nodemanager shutdown for decommission vs shutdown
+ for restart. (Jason Lowe via junping_du)
+
+ YARN-1338. Recover localized resource cache state upon nodemanager restart
+ (Jason Lowe via junping_du)
+
+ YARN-1368. Added core functionality of recovering container state into
+ schedulers after ResourceManager Restart so as to preserve running work in
+ the cluster. (Jian He via vinodkv)
+
+ YARN-1702. Added kill app functionality to RM web services. (Varun Vasudev
+ via vinodkv)
+
+ YARN-1339. Recover DeletionService state upon nodemanager restart. (Jason Lowe
+ via junping_du)
+
+ YARN-1365. Changed ApplicationMasterService to allow an app to re-register
+ after RM restart. (Anubhav Dhoot via jianhe)
+
+ YARN-2052. Embedded an epoch number in container id to ensure the uniqueness
+ of container id after RM restarts. (Tsuyoshi OZAWA via jianhe)
+
+ YARN-1713. Added get-new-app and submit-app functionality to RM web services.
+ (Varun Vasudev via vinodkv)
+
+ YARN-2233. Implemented ResourceManager web-services to create, renew and
+ cancel delegation tokens. (Varun Vasudev via vinodkv)
+
+ YARN-2247. Made RM web services authenticate users via kerberos and delegation
+ token. (Varun Vasudev via zjshen)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -56,6 +292,143 @@ Release 2.5.0 - UNRELEASED
YARN-1870. FileInputStream is not closed in ProcfsBasedProcessTree#constructProcessSMAPInfo.
(Fengdong Yu via junping_du)
+ YARN-1970. Prepare YARN codebase for JUnit 4.11. (cnauroth)
+
+ YARN-483. Improve documentation on log aggregation in yarn-default.xml
+ (Akira Ajisaka via Sandy Ryza)
+
+ YARN-2036. Document yarn.resourcemanager.hostname in ClusterSetup (Ray
+ Chiang via Sandy Ryza)
+
+ YARN-766. TestNodeManagerShutdown in branch-2 should use Shell to form the output path and a format
+ issue in trunk. (Contributed by Siddharth Seth)
+
+ YARN-1982. Renamed the daemon name to be TimelineServer instead of History
+ Server and deprecated the old usage. (Zhijie Shen via vinodkv)
+
+ YARN-1987. Wrapper for leveldb DBIterator to aid in handling database exceptions.
+ (Jason Lowe via kasha)
+
+ YARN-1751. Improve MiniYarnCluster for log aggregation testing (Ming Ma
+ via jlowe)
+
+ YARN-1981. Nodemanager version is not updated when a node reconnects (Jason
+ Lowe via jeagles)
+
+ YARN-1938. Added kerberos login for the Timeline Server. (Zhijie Shen via
+ vinodkv)
+
+ YARN-2017. Merged some of the common scheduler code. (Jian He via vinodkv)
+
+ YARN-2049. Added delegation-token support for the Timeline Server. (Zhijie
+ Shen via vinodkv)
+
+ YARN-1936. Added security support for the Timeline Client. (Zhijie Shen via
+ vinodkv)
+
+ YARN-1937. Added owner-only ACLs support for Timeline Client and server.
+ (Zhijie Shen via vinodkv)
+
+ YARN-2012. Fair Scheduler: allow default queue placement rule to take an
+ arbitrary queue (Ashwin Shankar via Sandy Ryza)
+
+ YARN-2059. Added admin ACLs support to Timeline Server. (Zhijie Shen via
+ vinodkv)
+
+ YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting
+ resources when cluster is free (Karthik Kambatla via Sandy Ryza)
+
+ YARN-2071. Modified levelDB store permissions to be readable only by the
+ server user. (Zhijie Shen via vinodkv)
+
+ YARN-2107. Refactored timeline classes into o.a.h.y.s.timeline package. (Vinod
+ Kumar Vavilapalli via zjshen)
+
+ YARN-596. Use scheduling policies throughout the queue hierarchy to decide
+ which containers to preempt (Wei Yan via Sandy Ryza)
+
+ YARN-2054. Better defaults for YARN ZK configs for retries and retry-inteval
+ when HA is enabled. (kasha)
+
+ YARN-1877. Document yarn.resourcemanager.zk-auth and its scope.
+ (Robert Kanter via kasha)
+
+ YARN-2115. Replaced RegisterNodeManagerRequest's ContainerStatus with a new
+ NMContainerStatus which has more information that is needed for
+ work-preserving RM-restart. (Jian He via vinodkv)
+
+ YARN-1474. Make schedulers services. (Tsuyoshi Ozawa via kasha)
+
+ YARN-1913. With Fair Scheduler, cluster can logjam when all resources are
+ consumed by AMs (Wei Yan via Sandy Ryza)
+
+ YARN-2061. Revisit logging levels in ZKRMStateStore. (Ray Chiang via kasha)
+
+ YARN-1977. Add tests on getApplicationRequest with filtering start time range. (junping_du)
+
+ YARN-2122. In AllocationFileLoaderService, the reloadThread should be created
+ in init() and started in start(). (Robert Kanter via kasha)
+
+ YARN-2132. ZKRMStateStore.ZKAction#runWithRetries doesn't log the exception
+ it encounters. (Vamsee Yarlagadda via kasha)
+
+ YARN-2030. Augmented RMStateStore with state machine.(Binglin Chang via jianhe)
+
+ YARN-1424. RMAppAttemptImpl should return the
+ DummyApplicationResourceUsageReport for all invalid accesses.
+ (Ray Chiang via kasha)
+
+ YARN-2091. Add more values to ContainerExitStatus and pass it from NM to
+ RM and then to app masters (Tsuyoshi OZAWA via bikas)
+
+ YARN-2125. Changed ProportionalCapacityPreemptionPolicy to log CSV in debug
+ level. (Wangda Tan via jianhe)
+
+ YARN-2159. Better logging in SchedulerNode#allocateContainer.
+ (Ray Chiang via kasha)
+
+ YARN-2191. Added a new test to ensure NM will clean up completed applications
+ in the case of RM restart. (Wangda Tan via jianhe)
+
+ YARN-2195. Clean a piece of code in ResourceRequest. (Wei Yan via devaraj)
+
+ YARN-2074. Changed ResourceManager to not count AM preemptions towards app
+ failures. (Jian He via vinodkv)
+
+ YARN-2192. TestRMHA fails when run with a mix of Schedulers.
+ (Anubhav Dhoot via kasha)
+
+ YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha)
+
+ YARN-2072. RM/NM UIs and webservices are missing vcore information.
+ (Nathan Roberts via tgraves)
+
+ YARN-2152. Added missing information into ContainerTokenIdentifier so that
+ NodeManagers can report the same to RM when RM restarts. (Jian He via vinodkv)
+
+ YARN-2171. Improved CapacityScheduling to not lock on nodemanager-count when
+ AMs heartbeat in. (Jason Lowe via vinodkv)
+
+ YARN-614. Changed ResourceManager to not count disk failure, node loss and
+ RM restart towards app failures. (Xuan Gong via jianhe)
+
+ YARN-2224. Explicitly enable vmem check in
+ TestContainersMonitor#testContainerKillOnMemoryOverflow.
+ (Anubhav Dhoot via kasha)
+
+ YARN-2022. Preempting an Application Master container can be kept as least priority
+ when multiple applications are marked for preemption by
+ ProportionalCapacityPreemptionPolicy (Sunil G via mayank)
+
+ YARN-2241. ZKRMStateStore: On startup, show nicer messages if znodes already
+ exist. (Robert Kanter via kasha)
+
+ YARN-1408 Preemption caused Invalid State Event: ACQUIRED at KILLED and
+ caused a task timeout for 30mins. (Sunil G via mayank)
+
+ YARN-2300. Improved the documentation of the sample requests for RM REST API -
+ submitting an app. (Varun Vasudev via zjshen)
+
OPTIMIZATIONS
BUG FIXES
@@ -68,7 +441,127 @@ Release 2.5.0 - UNRELEASED
YARN-1784. TestContainerAllocation assumes CapacityScheduler.
(Robert Kanter via kasha)
-Release 2.4.1 - UNRELEASED
+ YARN-1940. deleteAsUser() terminates early without deleting more files on
+ error (Rushabh S Shah via jlowe)
+
+ YARN-1865. ShellScriptBuilder does not check for some error conditions.
+ (Remus Rusanu via ivanmi)
+
+ YARN-738. TestClientRMTokens is failing irregularly while running all yarn
+ tests (Ming Ma via jlowe)
+
+ YARN-2018. TestClientRMService.testTokenRenewalWrongUser fails after
+ HADOOP-10562 (Ming Ma via Arpit Agarwal)
+
+ YARN-2011. Fix typo and warning in TestLeafQueue (Chen He via junping_du)
+
+
+ YARN-2042. String shouldn't be compared using == in
+ QueuePlacementRule#NestedUserQueue#getQueueForApp (Chen He via Sandy Ryza)
+
+ YARN-2050. Fix LogCLIHelpers to create the correct FileContext (Ming Ma
+ via jlowe)
+
+ YARN-2089. FairScheduler: QueuePlacementPolicy and QueuePlacementRule
+ are missing audience annotations. (Zhihai Xu via kasha)
+
+ YARN-2096. Race in TestRMRestart#testQueueMetricsOnRMRestart.
+ (Anubhav Dhoot via kasha)
+
+ YARN-2105. Fix TestFairScheduler after YARN-2012. (Ashwin Shankar via
+ Sandy Ryza)
+
+ YARN-2112. Fixed yarn-common's pom.xml to include jackson dependencies so
+ that both Timeline Server and client can access them. (Zhijie Shen via
+ vinodkv)
+
+ YARN-1868. YARN status web ui does not show correctly in IE 11.
+ (Chuan Liu via cnauroth)
+
+ YARN-2103. Inconsistency between viaProto flag and initial value of
+ SerializedExceptionProto.Builder (Binglin Chang via junping_du)
+
+ YARN-1550. NPE in FairSchedulerAppsBlock#render. (Anubhav Dhoot via kasha)
+
+ YARN-2119. DEFAULT_PROXY_ADDRESS should use DEFAULT_PROXY_PORT.
+ (Anubhav Dhoot via kasha)
+
+ YARN-2118. Fixed the type mismatch in Map#containsKey check of
+ TimelineWebServices#injectOwnerInfo. (Ted Yu via zjshen)
+
+ YARN-2117. Fixed the issue that secret file reader is potentially not
+ closed in TimelineAuthenticationFilterInitializer. (Chen He via zjshen)
+
+ YARN-2121. Fixed NPE handling in Timeline Server's TimelineAuthenticator.
+ (Zhijie Shen via vinodkv)
+
+ YARN-2128. FairScheduler: Incorrect calculation of amResource usage.
+ (Wei Yan via kasha)
+
+ YARN-2124. Fixed NPE in ProportionalCapacityPreemptionPolicy. (Wangda Tan
+ via jianhe)
+
+ YARN-2148. TestNMClient failed due more exit code values added and passed
+ to AM (Wangda Tan via bikas)
+
+ YARN-2075. Fixed the test failure of TestRMAdminCLI. (Kenji Kikushima via
+ zjshen)
+
+ YARN-2155. FairScheduler: Incorrect threshold check for preemption.
+ (Wei Yan via kasha)
+
+ YARN-1885. Fixed a bug that RM may not send application-clean-up signal
+ to NMs where the completed applications previously ran in case of RM restart.
+ (Wangda Tan via jianhe)
+
+ YARN-2167. LeveldbIterator should get closed in
+ NMLeveldbStateStoreService#loadLocalizationState() within finally block
+ (Junping Du via jlowe)
+
+ YARN-2187. FairScheduler: Disable max-AM-share check by default.
+ (Robert Kanter via kasha)
+
+ YARN-2111. In FairScheduler.attemptScheduling, we don't count containers
+ as assigned if they have 0 memory but non-zero cores (Sandy Ryza)
+
+ YARN-2204. TestAMRestart#testAMRestartWithExistingContainers assumes
+ CapacityScheduler. (Robert Kanter via kasha)
+
+ YARN-2163. WebUI: Order of AppId in apps table should be consistent with
+ ApplicationId.compareTo(). (Wangda Tan via raviprak)
+
+ YARN-2104. Scheduler queue filter failed to work because index of queue
+ column changed. (Wangda Tan via jlowe)
+
+ YARN-2201. Made TestRMWebServicesAppsModification be independent of the
+ changes on yarn-default.xml. (Varun Vasudev via zjshen)
+
+ YARN-2216 YARN-2065 AM cannot create new containers after restart
+ (Jian He via stevel)
+
+ YARN-2232. Fixed ResourceManager to allow DelegationToken owners to be able
+ to cancel their own tokens in secure mode. (Varun Vasudev via vinodkv)
+
+ YARN-2250. FairScheduler.findLowestCommonAncestorQueue returns null when
+ queues not identical (Krisztian Horvath via Sandy Ryza)
+
+ YARN-2158. Improved assertion messages of TestRMWebServicesAppsModification.
+ (Varun Vasudev via zjshen)
+
+ YARN-2269. Remove external links from YARN UI. (Craig Welch via xgong)
+
+ YARN-2270. Made TestFSDownload#testDownloadPublicWithStatCache be skipped
+ when thereâs no ancestor permissions. (Akira Ajisaka via zjshen)
+
+ YARN-2319. Made the MiniKdc instance start/close before/after the class of
+ TestRMWebServicesDelegationTokens. (Wenwu Peng via zjshen)
+
+ YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha)
+
+ YARN-1726. ResourceSchedulerWrapper broken due to AbstractYarnScheduler.
+ (Wei Yan via kasha)
+
+Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
@@ -78,6 +571,16 @@ Release 2.4.1 - UNRELEASED
YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen)
+ YARN-1696. Added documentation for ResourceManager fail-over. (Karthik
+ Kambatla, Masatake Iwasaki, Tsuyoshi OZAWA via vinodkv)
+
+ YARN-1701. Improved default paths of the timeline store and the generic
+ history store. (Tsuyoshi Ozawa via zjshen)
+
+ YARN-1962. Changed Timeline Service client configuration to be off by default
+ given the non-readiness of the feature yet. (Mohammad Kamrul Islam via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -144,6 +647,43 @@ Release 2.4.1 - UNRELEASED
YARN-1281. Fixed TestZKRMStateStoreZKClientConnections to not fail
intermittently due to ZK-client timeouts. (Tsuyoshi Ozawa via vinodkv)
+ YARN-1932. Javascript injection on the job status page (Mit Desai via
+ jlowe)
+
+ YARN-1975. Used resources shows escaped html in CapacityScheduler and
+ FairScheduler page (Mit Desai via jlowe)
+
+ YARN-1929. Fixed a deadlock in ResourceManager that occurs when failover
+ happens right at the time of shutdown. (Karthik Kambatla via vinodkv)
+
+ YARN-1201. TestAMAuthorization fails with local hostname cannot be resolved.
+ (Wangda Tan via junping_du)
+
+ YARN-1861. Fixed a bug in RM to reset leader-election on fencing that was
+ causing both RMs to be stuck in standby mode when automatic failover is
+ enabled. (Karthik Kambatla and Xuan Gong via vinodkv)
+
+ YARN-1957. Consider the max capacity of the queue when computing the ideal
+ capacity for preemption. (Carlo Curino via cdouglas)
+
+ YARN-1986. In Fifo Scheduler, node heartbeat in between creating app and
+ attempt causes NPE (Hong Zhiguo via Sandy Ryza)
+
+ YARN-1976. Fix yarn application CLI to print the scheme of the tracking url
+ of failed/killed applications. (Junping Du via jianhe)
+
+ YARN-2016. Fix a bug in GetApplicationsRequestPBImpl to add the missed fields
+ to proto. (Junping Du via jianhe)
+
+ YARN-2053. Fixed a bug in AMS to not add null NMToken into NMTokens list from
+ previous attempts for work-preserving AM restart. (Wangda Tan via jianhe)
+
+ YARN-2066. Wrong field is referenced in GetApplicationsRequestPBImpl#mergeLocalToBuilder()
+ (Hong Zhiguo via junping_du)
+
+ YARN-2081. Fixed TestDistributedShell failure after YARN-1962. (Zhiguo Hong
+ via zjshen)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh Tue Aug 19 23:49:39 2014
@@ -16,20 +16,34 @@
# limitations under the License.
-# Start all yarn daemons. Run this on master node.
-
-echo "starting yarn daemons"
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+function hadoop_usage
+{
+ echo "Usage: start-yarn.sh [--config confdir]"
+}
+
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
+ exit 1
+fi
# start resourceManager
-"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager
+"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" start resourcemanager
# start nodeManager
-"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR start nodemanager
+"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}" start nodemanager
# start proxyserver
-#"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start proxyserver
+#"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" start proxyserver
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh Tue Aug 19 23:49:39 2014
@@ -18,18 +18,34 @@
# Stop all yarn daemons. Run this on master node.
-echo "stopping yarn daemons"
+function hadoop_usage
+{
+ echo "Usage: stop-yarn.sh [--config confdir]"
+}
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-# stop resourceManager
-"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager
-# stop nodeManager
-"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR stop nodemanager
-# stop proxy server
-"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop proxyserver
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
+ exit 1
+fi
+
+# start resourceManager
+"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" stop resourcemanager
+# start nodeManager
+"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}" stop nodemanager
+# start proxyserver
+#"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" stop proxyserver
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn Tue Aug 19 23:49:39 2014
@@ -15,253 +15,182 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-# The Hadoop command script
-#
-# Environment Variables
-#
-# JAVA_HOME The java implementation to use. Overrides JAVA_HOME.
-#
-# YARN_USER_CLASSPATH Additional user CLASSPATH entries.
-#
-# YARN_USER_CLASSPATH_FIRST If set to non empty value then the user classpath
-# specified in YARN_USER_CLASSPATH will be
-# appended at the beginning of YARN's final
-# classpath instead of at the end.
-#
-# YARN_HEAPSIZE The maximum amount of heap to use, in MB.
-# Default is 1000.
-#
-# YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command
-# eg YARN_NODEMANAGER_HEAPSIZE sets the heap
-# size for the NodeManager. If you set the
-# heap size in YARN_{COMMAND}_OPTS or YARN_OPTS
-# they take precedence.
-#
-# YARN_OPTS Extra Java runtime options.
-#
-# YARN_CLIENT_OPTS when the respective command is run.
-# YARN_{COMMAND}_OPTS etc YARN_NODEMANAGER_OPTS applies to NodeManager
-# for e.g. YARN_CLIENT_OPTS applies to
-# more than one command (fs, dfs, fsck,
-# dfsadmin etc)
-#
-# YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf.
-#
-# YARN_ROOT_LOGGER The root appender. Default is INFO,console
-#
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin" > /dev/null; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-function print_usage(){
- echo "Usage: yarn [--config confdir] COMMAND"
+function hadoop_usage
+{
+ echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)] COMMAND"
echo "where COMMAND is one of:"
- echo " resourcemanager run the ResourceManager"
- echo " nodemanager run a nodemanager on each slave"
- echo " historyserver run the application history server"
- echo " rmadmin admin tools"
- echo " version print the version"
- echo " jar <jar> run a jar file"
- echo " application prints application(s) report/kill application"
- echo " applicationattempt prints applicationattempt(s) report"
- echo " container prints container(s) report"
- echo " node prints node report(s)"
- echo " logs dump container logs"
- echo " classpath prints the class path needed to get the"
- echo " Hadoop jar and the required libraries"
- echo " daemonlog get/set the log level for each daemon"
+ echo " application prints application(s) report/kill application"
+ echo " applicationattempt prints applicationattempt(s) report"
+ echo " classpath prints the class path needed to get the"
+ echo " Hadoop jar and the required libraries"
+ echo " container prints container(s) report"
+ echo " daemonlog get/set the log level for each daemon"
+ echo " jar <jar> run a jar file"
+ echo " logs dump container logs"
+ echo " node prints node report(s)"
+ echo " nodemanager run a nodemanager on each slave"
+ echo " proxyserver run the web app proxy server"
+ echo " resourcemanager run the ResourceManager"
+ echo " resourcemanager -format-state-store deletes the RMStateStore"
+ echo " rmadmin admin tools"
+ echo " timelineserver run the timeline server"
+ echo " version print the version"
echo " or"
- echo " CLASSNAME run the class named CLASSNAME"
+ echo " CLASSNAME run the class named CLASSNAME"
echo "Most commands print help when invoked w/o parameters."
}
-# if no args specified, show usage
-if [ $# = 0 ]; then
- print_usage
+
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ this="${BASH_SOURCE-$0}"
+ bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
exit 1
fi
+# if no args specified, show usage
+if [[ $# = 0 ]]; then
+ hadoop_exit_with_usage 1
+fi
+
# get arguments
COMMAND=$1
shift
-case $COMMAND in
- # usage flags
- --help|-help|-h)
- print_usage
+case "${COMMAND}" in
+ application|applicationattempt|container)
+ CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ classpath)
+ hadoop_finalize
+ echo "${CLASSPATH}"
exit
- ;;
+ ;;
+ daemonlog)
+ CLASS=org.apache.hadoop.log.LogLevel
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ jar)
+ CLASS=org.apache.hadoop.util.RunJar
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ historyserver)
+ daemon="true"
+ echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
+ echo "Instead use the timelineserver command for it." 1>&2
+ echo "Starting the History Server anyway..." 1>&2
+ CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+ ;;
+ logs)
+ CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ node)
+ CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ nodemanager)
+ daemon="true"
+ CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
+ YARN_OPTS="${YARN_OPTS} ${YARN_NODEMANAGER_OPTS}"
+ if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
+ JAVA_HEAP_MAX="-Xmx${YARN_NODEMANAGER_HEAPSIZE}m"
+ fi
+ ;;
+ proxyserver)
+ daemon="true"
+ CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
+ YARN_OPTS="${YARN_OPTS} ${YARN_PROXYSERVER_OPTS}"
+ if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
+ JAVA_HEAP_MAX="-Xmx${YARN_PROXYSERVER_HEAPSIZE}m"
+ fi
+ ;;
+ resourcemanager)
+ daemon="true"
+ CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
+ YARN_OPTS="${YARN_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
+ if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
+ JAVA_HEAP_MAX="-Xmx${YARN_RESOURCEMANAGER_HEAPSIZE}m"
+ fi
+ ;;
+ rmadmin)
+ CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ timelineserver)
+ daemon="true"
+ CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+ YARN_OPTS="${YARN_OPTS} ${YARN_TIMELINESERVER_OPTS}"
+ if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
+ JAVA_HEAP_MAX="-Xmx${YARN_TIMELINESERVER_HEAPSIZE}m"
+ fi
+ ;;
+ version)
+ CLASS=org.apache.hadoop.util.VersionInfo
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ -*)
+ hadoop_exit_with_usage 1
+ ;;
+ *)
+ CLASS="${COMMAND}"
+ ;;
esac
-if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
- . "${YARN_CONF_DIR}/yarn-env.sh"
-fi
-
-# some Java parameters
-if [ "$JAVA_HOME" != "" ]; then
- #echo "run java in $JAVA_HOME"
- JAVA_HOME=$JAVA_HOME
-fi
-
-if [ "$JAVA_HOME" = "" ]; then
- echo "Error: JAVA_HOME is not set."
- exit 1
-fi
+# set HADOOP_OPTS to YARN_OPTS so that we can use
+# finalize, etc, without doing anything funky
+HADOOP_OPTS="${YARN_OPTS}"
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
+daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
+daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
- #echo "run with heapsize $YARN_HEAPSIZE"
- JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
- #echo $JAVA_HEAP_MAX
+if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
+ YARN_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
+ HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
fi
-# CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR
-if [ ! -d "$HADOOP_CONF_DIR" ]; then
- echo No HADOOP_CONF_DIR set.
- echo Please specify it either in yarn-env.sh or in the environment.
- exit 1
-fi
+hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
-CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}"
+# Add YARN custom options to comamnd line in case someone actaully
+# used these.
+#
+# Note that we are replacing ' ' with '\ ' so that when we exec
+# stuff it works
+#
+hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${HADOOP_LOG_DIR/ /\ }"
+hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE/ /\ }"
+hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${HADOOP_YARN_HOME/ /\ }"
+hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-# for developers, add Hadoop classes to CLASSPATH
-if [ -d "$HADOOP_YARN_HOME/yarn-api/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-api/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-common/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-common/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-mapreduce/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-mapreduce/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-master-worker/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-master-worker/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/build/test/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/target/test/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/build/tools" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/build/tools
-fi
+hadoop_finalize
-CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_DIR}/*
-CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_LIB_JARS_DIR}/*
+export CLASSPATH
-# Add user defined YARN_USER_CLASSPATH to the class path (if defined)
-if [ -n "$YARN_USER_CLASSPATH" ]; then
- if [ -n "$YARN_USER_CLASSPATH_FIRST" ]; then
- # User requested to add the custom entries at the beginning
- CLASSPATH=${YARN_USER_CLASSPATH}:${CLASSPATH}
+if [[ -n "${daemon}" ]]; then
+ if [[ -n "${secure_service}" ]]; then
+ hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" \
+ "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \
+ "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
else
- # By default we will just append the extra entries at the end
- CLASSPATH=${CLASSPATH}:${YARN_USER_CLASSPATH}
+ hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \
+ "${daemon_pidfile}" "${daemon_outfile}" "$@"
fi
-fi
-
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
-
-# default log directory & file
-if [ "$YARN_LOG_DIR" = "" ]; then
- YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$YARN_LOGFILE" = "" ]; then
- YARN_LOGFILE='yarn.log'
-fi
-
-# restore ordinary behaviour
-unset IFS
-
-# figure out which class to run
-if [ "$COMMAND" = "classpath" ] ; then
- echo $CLASSPATH
- exit
-elif [ "$COMMAND" = "rmadmin" ] ; then
- CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "application" ] ||
- [ "$COMMAND" = "applicationattempt" ] ||
- [ "$COMMAND" = "container" ]; then
- CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
- set -- $COMMAND $@
-elif [ "$COMMAND" = "node" ] ; then
- CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "resourcemanager" ] ; then
- CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
- CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
- YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS"
- if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
- fi
-elif [ "$COMMAND" = "historyserver" ] ; then
- CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties
- CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
- YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS"
- if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m"
- fi
-elif [ "$COMMAND" = "nodemanager" ] ; then
- CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
- CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
- YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
- if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m"
- fi
-elif [ "$COMMAND" = "proxyserver" ] ; then
- CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
- YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS"
- if [ "$YARN_PROXYSERVER_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_PROXYSERVER_HEAPSIZE""m"
- fi
-elif [ "$COMMAND" = "version" ] ; then
- CLASS=org.apache.hadoop.util.VersionInfo
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "jar" ] ; then
- CLASS=org.apache.hadoop.util.RunJar
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "logs" ] ; then
- CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "daemonlog" ] ; then
- CLASS=org.apache.hadoop.log.LogLevel
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
+ exit $?
else
- CLASS=$COMMAND
+ hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
fi
-
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
-YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_YARN_HOME"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
- YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi
-
-exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@"
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh Tue Aug 19 23:49:39 2014
@@ -13,53 +13,81 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# included in all the hadoop scripts with source command
-# should not be executable directly
-bin=`which "$0"`
-bin=`dirname "${bin}"`
-bin=`cd "$bin"; pwd`
+function hadoop_subproject_init
+{
+
+ # at some point in time, someone thought it would be a good idea to
+ # create separate vars for every subproject. *sigh*
+ # let's perform some overrides and setup some defaults for bw compat
+ # this way the common hadoop var's == subproject vars and can be
+ # used interchangeable from here on out
+ # ...
+ # this should get deprecated at some point.
+
+ if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then
+ . "${YARN_CONF_DIR}/yarn-env.sh"
+ elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then
+ . "${HADOOP_CONF_DIR}/yarn-env.sh"
+ fi
+
+ if [[ -n "${YARN_CONF_DIR}" ]]; then
+ HADOOP_CONF_DIR="${YARN_CONF_DIR}"
+ fi
+
+ YARN_CONF_DIR="${HADOOP_CONF_DIR}"
+
+ # YARN_CONF_DIR needs precedence over HADOOP_CONF_DIR
+ # and the various jar dirs
+ hadoop_add_classpath "${YARN_CONF_DIR}" before
+
+ HADOOP_LOG_DIR="${YARN_LOG_DIR:-$HADOOP_LOG_DIR}"
+ YARN_LOG_DIR="${HADOOP_LOG_DIR}"
+
+ HADOOP_LOGFILE="${YARN_LOGFILE:-$HADOOP_LOGFILE}"
+ YARN_LOGFILE="${HADOOP_LOGFILE}"
+
+ HADOOP_NICENESS="${YARN_NICENESS:-$HADOOP_NICENESS}"
+ YARN_NICENESS="${HADOOP_NICENESS}"
+
+ HADOOP_STOP_TIMEOUT="${YARN_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}"
+ YARN_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}"
+
+ HADOOP_PID_DIR="${YARN_PID_DIR:-$HADOOP_PID_DIR}"
+ YARN_PID_DIR="${HADOOP_PID_DIR}"
+
+ HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-INFO,console}"
+ YARN_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
+
+ HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}"
+
+ HADOOP_IDENT_STRING="${YARN_IDENT_STRING:-$HADOOP_IDENT_STRING}"
+ YARN_IDENT_STRING="${HADOOP_IDENT_STRING}"
+
+ YARN_OPTS="${YARN_OPTS:-$HADOOP_OPTS}"
+
+ # YARN-1429 added the completely superfluous YARN_USER_CLASSPATH
+ # env var. We're going to override HADOOP_USER_CLASSPATH to keep
+ # consistency with the rest of the duplicate/useless env vars
+ HADOOP_USER_CLASSPATH="${YARN_USER_CLASSPATH:-$HADOOP_USER_CLASSPATH}"
+ YARN_USER_CLASSPATH="${HADOOP_USER_CLASSPATH}"
+
+ HADOOP_USER_CLASSPATH_FIRST="${YARN_USER_CLASSPATH_FIRST:-$HADOOP_USER_CLASSPATH_FIRST}"
+ YARN_USER_CLASSPATH_FIRST="${HADOOP_USER_CLASSPATH_FIRST}"
+}
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
- . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
-elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
- . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
-elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
- . "$HADOOP_HOME"/libexec/hadoop-config.sh
+if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
+ _yc_this="${BASH_SOURCE-$0}"
+ HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_yc_this}")" >/dev/null && pwd -P)
+fi
+
+if [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
else
echo "Hadoop common not found."
exit
fi
-# Same glibc bug that discovered in Hadoop.
-# Without this you can see very large vmem settings on containers.
-export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
-
-#check to see if the conf dir is given as an optional argument
-if [ $# -gt 1 ]
-then
- if [ "--config" = "$1" ]
- then
- shift
- confdir=$1
- shift
- YARN_CONF_DIR=$confdir
- fi
-fi
-
-# Allow alternate conf dir location.
-export YARN_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-
-#check to see it is specified whether to use the slaves or the
-# masters file
-if [ $# -gt 1 ]
-then
- if [ "--hosts" = "$1" ]
- then
- shift
- slavesfile=$1
- shift
- export YARN_SLAVES="${YARN_CONF_DIR}/$slavesfile"
- fi
-fi
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh Tue Aug 19 23:49:39 2014
@@ -15,146 +15,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-# Runs a yarn command as a daemon.
-#
-# Environment Variables
-#
-# YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf.
-# YARN_LOG_DIR Where log files are stored. PWD by default.
-# YARN_MASTER host:path where hadoop code should be rsync'd from
-# YARN_PID_DIR The pid files are stored. /tmp by default.
-# YARN_IDENT_STRING A string representing this instance of hadoop. $USER by default
-# YARN_NICENESS The scheduling priority for daemons. Defaults to 0.
-##
-
-usage="Usage: yarn-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <yarn-command> "
-
-# if no args specified, show usage
-if [ $# -le 1 ]; then
- echo $usage
- exit 1
-fi
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-# get arguments
-startStop=$1
-shift
-command=$1
-shift
-
-hadoop_rotate_log ()
+function hadoop_usage
{
- log=$1;
- num=5;
- if [ -n "$2" ]; then
- num=$2
- fi
- if [ -f "$log" ]; then # rotate logs
- while [ $num -gt 1 ]; do
- prev=`expr $num - 1`
- [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
- num=$prev
- done
- mv "$log" "$log.$num";
- fi
+ echo "Usage: yarn-daemon.sh [--config confdir] (start|stop|status) <hadoop-command> <args...>"
}
-if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
- . "${YARN_CONF_DIR}/yarn-env.sh"
-fi
-
-if [ "$YARN_IDENT_STRING" = "" ]; then
- export YARN_IDENT_STRING="$USER"
-fi
-
-# get log directory
-if [ "$YARN_LOG_DIR" = "" ]; then
- export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-
-if [ ! -w "$YARN_LOG_DIR" ] ; then
- mkdir -p "$YARN_LOG_DIR"
- chown $YARN_IDENT_STRING $YARN_LOG_DIR
-fi
-
-if [ "$YARN_PID_DIR" = "" ]; then
- YARN_PID_DIR=/tmp
-fi
-
-# some variables
-export YARN_LOGFILE=yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.log
-export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,RFA}
-log=$YARN_LOG_DIR/yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.out
-pid=$YARN_PID_DIR/yarn-$YARN_IDENT_STRING-$command.pid
-YARN_STOP_TIMEOUT=${YARN_STOP_TIMEOUT:-5}
-
-# Set default scheduling priority
-if [ "$YARN_NICENESS" = "" ]; then
- export YARN_NICENESS=0
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ this="${BASH_SOURCE-$0}"
+ bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
+ exit 1
fi
-case $startStop in
-
- (start)
-
- [ -w "$YARN_PID_DIR" ] || mkdir -p "$YARN_PID_DIR"
-
- if [ -f $pid ]; then
- if kill -0 `cat $pid` > /dev/null 2>&1; then
- echo $command running as process `cat $pid`. Stop it first.
- exit 1
- fi
- fi
-
- if [ "$YARN_MASTER" != "" ]; then
- echo rsync from $YARN_MASTER
- rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $YARN_MASTER/ "$HADOOP_YARN_HOME"
- fi
-
- hadoop_rotate_log $log
- echo starting $command, logging to $log
- cd "$HADOOP_YARN_HOME"
- nohup nice -n $YARN_NICENESS "$HADOOP_YARN_HOME"/bin/yarn --config $YARN_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
- echo $! > $pid
- sleep 1
- head "$log"
- # capture the ulimit output
- echo "ulimit -a" >> $log
- ulimit -a >> $log 2>&1
- ;;
-
- (stop)
-
- if [ -f $pid ]; then
- TARGET_PID=`cat $pid`
- if kill -0 $TARGET_PID > /dev/null 2>&1; then
- echo stopping $command
- kill $TARGET_PID
- sleep $YARN_STOP_TIMEOUT
- if kill -0 $TARGET_PID > /dev/null 2>&1; then
- echo "$command did not stop gracefully after $YARN_STOP_TIMEOUT seconds: killing with kill -9"
- kill -9 $TARGET_PID
- fi
- else
- echo no $command to stop
- fi
- else
- echo no $command to stop
- fi
- ;;
-
- (*)
- echo $usage
- exit 1
- ;;
-
-esac
-
+daemonmode=$1
+shift
+exec "${HADOOP_YARN_HOME}/bin/yarn" \
+--config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh Tue Aug 19 23:49:39 2014
@@ -16,23 +16,31 @@
# limitations under the License.
-# Run a Yarn command on all slave hosts.
-
-usage="Usage: yarn-daemons.sh [--config confdir] [--hosts hostlistfile] [start
-|stop] command args..."
+function hadoop_usage
+{
+ echo "Usage: yarn-daemons.sh [--config confdir] [--hosts hostlistfile] (start|stop|status) <yarn-command> <args...>"
+}
+
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
-# if no args specified, show usage
-if [ $# -le 1 ]; then
- echo $usage
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
exit 1
fi
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-exec "$bin/slaves.sh" --config $YARN_CONF_DIR cd "$HADOOP_YARN_HOME" \; "$bin/yarn-daemon.sh" --config $YARN_CONF_DIR "$@"
+hadoop_connect_to_hosts "${bin}/yarn-daemon.sh" \
+--config "${HADOOP_CONF_DIR}" "$@"
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd Tue Aug 19 23:49:39 2014
@@ -120,11 +120,12 @@ if "%1" == "--config" (
if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes
- set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-applicationhistoryservice\target\classes
)
+
if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-applicationhistoryservice\target\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-applicationhistoryservice\target\classes
)
+
if exist %HADOOP_YARN_HOME%\build\test\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes
)
@@ -142,7 +143,8 @@ if "%1" == "--config" (
)
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
- application applicationattempt container node logs daemonlog historyserver
+ application applicationattempt container node logs daemonlog historyserver ^
+ timelineserver
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
)
@@ -204,6 +206,8 @@ goto :eof
goto :eof
:historyserver
+ @echo DEPRECATED: Use of this command to start the timeline server is deprecated. 1>&2
+ @echo Instead use the timelineserver command for it. 1>&2
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\ahs-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer
set YARN_OPTS=%YARN_OPTS% %HADOOP_HISTORYSERVER_OPTS%
@@ -212,6 +216,15 @@ goto :eof
)
goto :eof
+:timelineserver
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer
+ set YARN_OPTS=%YARN_OPTS% %HADOOP_TIMELINESERVER_OPTS%
+ if defined YARN_TIMELINESERVER_HEAPSIZE (
+ set JAVA_HEAP_MAX=-Xmx%YARN_TIMELINESERVER_HEAPSIZE%m
+ )
+ goto :eof
+
:nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
@@ -277,7 +290,7 @@ goto :eof
@echo where COMMAND is one of:
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
- @echo historyserver run the application history server
+ @echo timelineserver run the timeline server
@echo rmadmin admin tools
@echo version print the version
@echo jar ^<jar^> run a jar file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh Tue Aug 19 23:49:39 2014
@@ -13,118 +13,115 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+##
+## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL
+## WORK DONE BY THE yarn AND RELATED COMMANDS.
+##
+## Precedence rules:
+##
+## yarn-env.sh > hadoop-env.sh > hard-coded defaults
+##
+## YARN_xyz > HADOOP_xyz > hard-coded defaults
+##
+
+###
+# Generic settings for YARN
+###
+
# User for YARN daemons
export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-# resolve links - $0 may be a softlink
-export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+#
+# By default, YARN will use HADOOP_CONF_DIR. Specify a custom
+# YARN_CONF_DIR here
+# export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+#
+
+# Override Hadoop's log directory & file
+# export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+# export YARN_LOGFILE='yarn.log'
+
+# Need a custom-to-YARN service-level authorization policy file?
+# export YARN_POLICYFILE="yarn-policy.xml"
-# some Java parameters
-# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
-if [ "$JAVA_HOME" != "" ]; then
- #echo "run java in $JAVA_HOME"
- JAVA_HOME=$JAVA_HOME
-fi
-
-if [ "$JAVA_HOME" = "" ]; then
- echo "Error: JAVA_HOME is not set."
- exit 1
-fi
-
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
-
-# For setting YARN specific HEAP sizes please use this
-# Parameter and set appropriately
-# YARN_HEAPSIZE=1000
-
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
-fi
+#Override the log4j settings for all YARN apps
+# export YARN_ROOT_LOGGER="INFO,console"
+###
# Resource Manager specific parameters
+###
-# Specify the max Heapsize for the ResourceManager using a numerical value
+# Specify the max heapsize for the ResourceManager using a numerical value
# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_RESOURCEMANAGER_OPTS.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_RESOURCEMANAGER_OPTS.
# If not specified, the default value will be picked from either YARN_HEAPMAX
# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
#export YARN_RESOURCEMANAGER_HEAPSIZE=1000
-# Specify the max Heapsize for the HistoryManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_HISTORYSERVER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-#export YARN_HISTORYSERVER_HEAPSIZE=1000
-
# Specify the JVM options to be used when starting the ResourceManager.
# These options will be appended to the options specified as YARN_OPTS
# and therefore may override any similar flags set in YARN_OPTS
-#export YARN_RESOURCEMANAGER_OPTS=
+#
+# Examples for a Sun/Oracle JDK:
+# a) override the appsummary log file:
+# export YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log -Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY"
+#
+# b) Set JMX options
+# export YARN_RESOURCEMANAGER_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
+#
+# c) Set garbage collection logs from hadoop-env.sh
+# export YARN_RESOURCE_MANAGER_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+#
+# d) ... or set them directly
+# export YARN_RESOURCEMANAGER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+#
+#
+# export YARN_RESOURCEMANAGER_OPTS=
+###
# Node Manager specific parameters
+###
# Specify the max Heapsize for the NodeManager using a numerical value
# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_NODEMANAGER_OPTS.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_NODEMANAGER_OPTS.
# If not specified, the default value will be picked from either YARN_HEAPMAX
# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
#export YARN_NODEMANAGER_HEAPSIZE=1000
# Specify the JVM options to be used when starting the NodeManager.
# These options will be appended to the options specified as YARN_OPTS
# and therefore may override any similar flags set in YARN_OPTS
+#
+# See ResourceManager for some examples
+#
#export YARN_NODEMANAGER_OPTS=
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
+###
+# TimeLineServer specifc parameters
+###
+# Specify the max Heapsize for the timeline server using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_TIMELINESERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
+#export YARN_TIMELINESERVER_HEAPSIZE=1000
-# default log directory & file
-if [ "$YARN_LOG_DIR" = "" ]; then
- YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$YARN_LOGFILE" = "" ]; then
- YARN_LOGFILE='yarn.log'
-fi
-
-# default policy file for service-level authorization
-if [ "$YARN_POLICYFILE" = "" ]; then
- YARN_POLICYFILE="hadoop-policy.xml"
-fi
-
-# restore ordinary behaviour
-unset IFS
-
-MAC_OSX=false
-case "`uname`" in
-Darwin*) MAC_OSX=true;;
-esac
-
-if $MAC_OSX; then
- YARN_OPTS="$YARN_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc="
-fi
-
-
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
-YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
- YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi
-YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
-
+# Specify the JVM options to be used when starting the TimeLineServer.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_TIMELINESERVER_OPTS=
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Tue Aug 19 23:49:39 2014
@@ -142,6 +142,22 @@
<Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService" />
+ <Field name="allocFile" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <!-- Inconsistent sync warning - minimumAllocation is only initialized once and never changed -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler" />
+ <Field name="minimumAllocation" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode" />
+ <Method name="reserveResource" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
<!-- Inconsistent sync warning - reinitialize read from other queue does not need sync-->
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue" />
@@ -178,12 +194,19 @@
<Field name="scheduleAsynchronously" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
- <!-- Inconsistent sync warning - minimumAllocation is only initialized once and never changed -->
+ <!-- Inconsistent sync warning - updateInterval is only initialized once and never changed -->
<Match>
- <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler" />
- <Field name="minimumAllocation" />
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+ <Field name="updateInterval" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Inconsistent sync warning - callDurationMetrics is only initialized once and never changed -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+ <Field name="fsOpDurations" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+
<!-- Inconsistent sync warning - numRetries is only initialized once and never changed -->
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" />
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml Tue Aug 19 23:49:39 2014
@@ -51,24 +51,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java Tue Aug 19 23:49:39 2014
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.util.Recor
* <li>A list of nodes whose status has been updated.</li>
* <li>The number of available nodes in a cluster.</li>
* <li>A description of resources requested back by the cluster</li>
+ * <li>AMRMToken, if AMRMToken has been rolled over</li>
* </ul>
* </p>
*
@@ -102,6 +104,23 @@ public abstract class AllocateResponse {
return response;
}
+ @Private
+ @Unstable
+ public static AllocateResponse newInstance(int responseId,
+ List<ContainerStatus> completedContainers,
+ List<Container> allocatedContainers, List<NodeReport> updatedNodes,
+ Resource availResources, AMCommand command, int numClusterNodes,
+ PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
+ List<ContainerResourceIncrease> increasedContainers,
+ List<ContainerResourceDecrease> decreasedContainers) {
+ AllocateResponse response =
+ newInstance(responseId, completedContainers, allocatedContainers,
+ updatedNodes, availResources, command, numClusterNodes, preempt,
+ nmTokens, increasedContainers, decreasedContainers);
+ response.setAMRMToken(amRMToken);
+ return response;
+ }
+
/**
* If the <code>ResourceManager</code> needs the
* <code>ApplicationMaster</code> to take some action then it will send an
@@ -270,4 +289,17 @@ public abstract class AllocateResponse {
@Unstable
public abstract void setDecreasedContainers(
List<ContainerResourceDecrease> decreasedContainers);
+
+ /**
+ * The AMRMToken that belong to this attempt
+ *
+ * @return The AMRMToken that belong to this attempt
+ */
+ @Public
+ @Unstable
+ public abstract Token getAMRMToken();
+
+ @Private
+ @Unstable
+ public abstract void setAMRMToken(Token amRMToken);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java Tue Aug 19 23:49:39 2014
@@ -308,6 +308,15 @@ public abstract class GetApplicationsReq
/**
* Set the range of start times to filter applications on
*
+ * @param range
+ */
+ @Private
+ @Unstable
+ public abstract void setStartRange(LongRange range);
+
+ /**
+ * Set the range of start times to filter applications on
+ *
* @param begin beginning of the range
* @param end end of the range
* @throws IllegalArgumentException
@@ -329,6 +338,15 @@ public abstract class GetApplicationsReq
/**
* Set the range of finish times to filter applications on
*
+ * @param range
+ */
+ @Private
+ @Unstable
+ public abstract void setFinishRange(LongRange range);
+
+ /**
+ * Set the range of finish times to filter applications on
+ *
* @param begin beginning of the range
* @param end end of the range
* @throws IllegalArgumentException
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java Tue Aug 19 23:49:39 2014
@@ -47,7 +47,7 @@ public abstract class ApplicationResourc
}
/**
- * Get the number of used containers
+ * Get the number of used containers. -1 for invalid/inaccessible reports.
* @return the number of used containers
*/
@Public
@@ -63,7 +63,7 @@ public abstract class ApplicationResourc
public abstract void setNumUsedContainers(int num_containers);
/**
- * Get the number of reserved containers
+ * Get the number of reserved containers. -1 for invalid/inaccessible reports.
* @return the number of reserved containers
*/
@Private
@@ -79,7 +79,7 @@ public abstract class ApplicationResourc
public abstract void setNumReservedContainers(int num_reserved_containers);
/**
- * Get the used <code>Resource</code>
+ * Get the used <code>Resource</code>. -1 for invalid/inaccessible reports.
* @return the used <code>Resource</code>
*/
@Public
@@ -91,7 +91,7 @@ public abstract class ApplicationResourc
public abstract void setUsedResources(Resource resources);
/**
- * Get the reserved <code>Resource</code>
+ * Get the reserved <code>Resource</code>. -1 for invalid/inaccessible reports.
* @return the reserved <code>Resource</code>
*/
@Public
@@ -103,7 +103,7 @@ public abstract class ApplicationResourc
public abstract void setReservedResources(Resource reserved_resources);
/**
- * Get the needed <code>Resource</code>
+ * Get the needed <code>Resource</code>. -1 for invalid/inaccessible reports.
* @return the needed <code>Resource</code>
*/
@Public
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java Tue Aug 19 23:49:39 2014
@@ -88,7 +88,7 @@ public abstract class ApplicationSubmiss
int maxAppAttempts, Resource resource, String applicationType) {
return newInstance(applicationId, applicationName, queue, priority,
amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
- resource, null, false);
+ resource, applicationType, false);
}
@Public
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java Tue Aug 19 23:49:39 2014
@@ -46,4 +46,30 @@ public class ContainerExitStatus {
* Containers preempted by the framework.
*/
public static final int PREEMPTED = -102;
+
+ /**
+ * Container terminated because of exceeding allocated virtual memory.
+ */
+ public static final int KILLED_EXCEEDED_VMEM = -103;
+
+ /**
+ * Container terminated because of exceeding allocated physical memory.
+ */
+ public static final int KILLED_EXCEEDED_PMEM = -104;
+
+ /**
+ * Container was terminated by stop request by the app master.
+ */
+ public static final int KILLED_BY_APPMASTER = -105;
+
+ /**
+ * Container was terminated by the resource manager.
+ */
+ public static final int KILLED_BY_RESOURCEMANAGER = -106;
+
+ /**
+ * Container was terminated after the application finished.
+ */
+ public static final int KILLED_AFTER_APP_COMPLETION = -107;
+
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java Tue Aug 19 23:49:39 2014
@@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.util.Recor
* <li>Allocated Resources to the container.</li>
* <li>Assigned Node id.</li>
* <li>Assigned Priority.</li>
- * <li>Start Time.</li>
+ * <li>Creation Time.</li>
* <li>Finish Time.</li>
* <li>Container Exit Status.</li>
* <li>{@link ContainerState} of the container.</li>
@@ -53,14 +53,14 @@ public abstract class ContainerReport {
@Unstable
public static ContainerReport newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
- long startTime, long finishTime, String diagnosticInfo, String logUrl,
+ long creationTime, long finishTime, String diagnosticInfo, String logUrl,
int containerExitStatus, ContainerState containerState) {
ContainerReport report = Records.newRecord(ContainerReport.class);
report.setContainerId(containerId);
report.setAllocatedResource(allocatedResource);
report.setAssignedNode(assignedNode);
report.setPriority(priority);
- report.setStartTime(startTime);
+ report.setCreationTime(creationTime);
report.setFinishTime(finishTime);
report.setDiagnosticsInfo(diagnosticInfo);
report.setLogUrl(logUrl);
@@ -122,17 +122,17 @@ public abstract class ContainerReport {
public abstract void setPriority(Priority priority);
/**
- * Get the Start time of the container.
+ * Get the creation time of the container.
*
- * @return Start time of the container
+ * @return creation time of the container
*/
@Public
@Unstable
- public abstract long getStartTime();
+ public abstract long getCreationTime();
@Public
@Unstable
- public abstract void setStartTime(long startTime);
+ public abstract void setCreationTime(long creationTime);
/**
* Get the Finish time of the container.
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java Tue Aug 19 23:49:39 2014
@@ -31,6 +31,8 @@ public abstract class ResourceOption {
int overCommitTimeout){
ResourceOption resourceOption = Records.newRecord(ResourceOption.class);
resourceOption.setResource(resource);
+ resourceOption.setOverCommitTimeout(overCommitTimeout);
+ resourceOption.build();
return resourceOption;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java Tue Aug 19 23:49:39 2014
@@ -296,13 +296,7 @@ public abstract class ResourceRequest im
int capabilityComparison =
this.getCapability().compareTo(other.getCapability());
if (capabilityComparison == 0) {
- int numContainersComparison =
- this.getNumContainers() - other.getNumContainers();
- if (numContainersComparison == 0) {
- return 0;
- } else {
- return numContainersComparison;
- }
+ return this.getNumContainers() - other.getNumContainers();
} else {
return capabilityComparison;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java Tue Aug 19 23:49:39 2014
@@ -107,6 +107,17 @@ public class TimelinePutResponse {
*/
public static final int IO_EXCEPTION = 2;
+ /**
+ * Error code returned if the user specifies the timeline system reserved
+ * filter key
+ */
+ public static final int SYSTEM_FILTER_CONFLICT = 3;
+
+ /**
+ * Error code returned if the user is denied to access the timeline data
+ */
+ public static final int ACCESS_DENIED = 4;
+
private String entityId;
private String entityType;
private int errorCode;