You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by ya...@apache.org on 2019/04/02 11:27:10 UTC
[incubator-amaterasu] branch master updated (acdadb6 -> 6992ebf)
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.
from acdadb6 Merge pull request #47 from anonymousGiraffe/refactoring-JobParser
new 8b4c35c yarm containers resources are using the new SDK API
new 11d277e spark obs are being lunched
new f16ebf8 added loging for emr
new 6a0f901 change spark configuration initilization
new 4bcc33b moved data loader to kotlin
new 2b7b394 changed spark exec config to lazy
new dfde0f5 dependencies can be null
new 85287a6 support for spark 2.4.0
new ae7bcc6 fixed jeckson version
new 11ecb94 added zk path debug message
new 29e7be4 checking for spark-exec-env
new 85814bf removed try initjob
new 75d3ef5 better null handling for spark
new 1441e6f debugging appmaster command
new d5e4e3a fixed appmaster cli args
new 89ebd0d fixed new-job-id
new d2d0c12 fixed new-job-id
new 29a6f87 fixed job state
new d7c16ca jobManager fixes
new 0592eac jobManager fixes
new ff5f1fa checking registered actions
new f3d430f added loading of spark env
new 41421f5 making dist available on appMaster
new 9be621f making dist available on appMaster
new e14a4cf framework resources are copied as folder
new aa47f5e update hadoop version
new 36ee61a fixed yarn spark_home
new cc85424 fixed yarn spark_home
new 8b46f75 fixed dist barier
new 1493c84 removed appMaster consumer
new d13ec9a notifications are fixed
new e00e06a fixed jar execution
new c9c62c8 removed commented files
new c7cdc40 fixed tests
new b62775c commented python files
new 08df7db commented python files
new 6992ebf Merge pull request #48 from roadan/AMATERASU-62
The 626 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 2 +-
common/build.gradle | 17 +-
.../amaterasu/common/dataobjects/ExecData.kt} | 5 +-
.../amaterasu/common/dataobjects/TaskData.kt} | 4 +-
.../dependencies/Dependencies.kt} | 9 +-
.../execution/dependencies/PythonDependencies.kt | 3 +
.../common/execution/dependencies/PythonPackage.kt | 3 +
.../amaterasu/common/runtime/Environment.kt} | 18 +-
.../amaterasu/common/utils/ActiveNotifier.kt | 81 ++++
.../common/configuration/ClusterConfig.scala | 23 +-
.../amaterasu/common/dataobjects/TaskData.scala | 25 --
.../execution/dependencies/Dependencies.scala | 25 --
executor/build.gradle | 13 +-
.../executor/common/executors/ActiveNotifier.scala | 87 ----
.../mesos/executors/MesosActionsExecutor.scala | 6 +-
.../executor/yarn/executors/ActionsExecutor.scala | 16 +-
.../executor/yarn/executors/YarnNotifier.scala | 39 --
.../spark/dispatcher/SparkSetupProvider.scala | 28 +-
.../runners/providers/PySparkRunnerProvider.scala | 7 +-
.../providers/SparkScalaRunnerProvider.scala | 8 +-
.../providers/SparkSubmitScalaRunnerProvider.scala | 12 +-
.../spark/runner/SparkRunnersProvider.scala | 58 +--
.../spark/runner/pyspark/PySparkRunner.scala | 9 +-
.../spark/runner/repl/SparkRunnerHelper.scala | 32 +-
.../spark/runner/repl/SparkScalaRunner.scala | 2 +-
.../spark/runner/sparksql/SparkSqlRunner.scala | 2 +-
.../frameworks/spark/runner/SparkTestsSuite.scala | 28 +-
.../spark/runner/repl/SparkScalaRunnerTests.scala | 2 +-
.../runner/sparksql/SparkSqlRunnerTests.scala | 16 +-
.../frameworks/spark/runtime/AmaContext.scala | 2 +-
leader-common/build.gradle | 13 +-
.../leader/common/configuration/ConfigManager.kt | 4 +-
.../amaterasu/leader/common/execution/JobLoader.kt | 97 +++++
.../leader/common/execution/JobManager.kt | 14 +-
.../frameworls/FrameworkProvidersFactory.kt | 27 +-
.../amaterasu/leader/common/launcher/AmaOpts.kt} | 35 +-
.../amaterasu/leader/common/launcher/ArgsParser.kt | 35 ++
.../common/utilities/ActiveReportListener.kt | 20 +-
.../leader/common/utilities/DataLoader.kt | 125 ++++++
.../leader/common/utilities/MessagingClientUtil.kt | 2 +-
.../frameworks/FrameworkProvidersFactory.scala | 1 +
.../leader/common/utilities/DataLoader.scala | 111 -----
.../common/configuration/ConfigManagerTests.kt | 2 +-
leader-yarn/build.gradle | 89 ++++
.../amaterasu/leader/yarn/AppMasterArgsParser.kt | 31 +-
.../amaterasu/leader/yarn/ApplicationMaster.kt | 449 +++++++++++++++++++
.../org/apache/amaterasu/leader/yarn/Client.kt | 316 ++++++++++++++
.../amaterasu/leader/yarn/ClientArgsParser.kt | 20 +-
.../amaterasu/leader/yarn/YarnNMCallbackHandler.kt | 55 +++
leader/build.gradle | 8 +-
.../apache/amaterasu/leader/yarn/ArgsParser.java | 81 ----
.../org/apache/amaterasu/leader/yarn/Client.java | 336 --------------
.../org/apache/amaterasu/leader/yarn/JobOpts.java | 28 --
.../leader/mesos/schedulers/JobScheduler.scala | 14 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 483 ---------------------
.../leader/yarn/YarnNMCallbackHandler.scala | 53 ---
.../leader/yarn/YarnRMCallbackHandler.scala | 134 ------
leader/src/main/scripts/ama-start-yarn.sh | 17 +-
.../common/execution/JobParserTests.scala | 10 +-
sdk/build.gradle | 2 +-
60 files changed, 1543 insertions(+), 1651 deletions(-)
rename common/src/main/{scala/org/apache/amaterasu/common/dataobjects/ExecData.scala => kotlin/org/apache/amaterasu/common/dataobjects/ExecData.kt} (75%)
copy common/src/main/{scala/org/apache/amaterasu/common/dataobjects/JobData.scala => kotlin/org/apache/amaterasu/common/dataobjects/TaskData.kt} (81%)
mode change 100755 => 100644
copy common/src/main/kotlin/org/apache/amaterasu/common/{logging/KLogging.kt => execution/dependencies/Dependencies.kt} (76%)
create mode 100644 common/src/main/kotlin/org/apache/amaterasu/common/execution/dependencies/PythonDependencies.kt
create mode 100644 common/src/main/kotlin/org/apache/amaterasu/common/execution/dependencies/PythonPackage.kt
rename common/src/main/{scala/org/apache/amaterasu/common/runtime/Environment.scala => kotlin/org/apache/amaterasu/common/runtime/Environment.kt} (76%)
mode change 100755 => 100644
create mode 100644 common/src/main/kotlin/org/apache/amaterasu/common/utils/ActiveNotifier.kt
delete mode 100644 common/src/main/scala/org/apache/amaterasu/common/dataobjects/TaskData.scala
delete mode 100755 common/src/main/scala/org/apache/amaterasu/common/execution/dependencies/Dependencies.scala
delete mode 100644 executor/src/main/scala/org/apache/amaterasu/executor/common/executors/ActiveNotifier.scala
delete mode 100644 executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/YarnNotifier.scala
create mode 100644 leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/execution/JobLoader.kt
copy sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.kt => leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/execution/frameworls/FrameworkProvidersFactory.kt (52%)
copy leader-common/src/main/{scala/org/apache/amaterasu/leader/common/utilities/MemoryFormatParser.scala => kotlin/org/apache/amaterasu/leader/common/launcher/AmaOpts.kt} (54%)
create mode 100644 leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/launcher/ArgsParser.kt
create mode 100644 leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/DataLoader.kt
delete mode 100755 leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/DataLoader.scala
copy leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/MemoryFormatParser.scala => leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/AppMasterArgsParser.kt (54%)
create mode 100644 leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt
create mode 100644 leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/Client.kt
copy leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/dsl/GitUtil.kt => leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ClientArgsParser.kt (67%)
create mode 100644 leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/YarnNMCallbackHandler.kt
delete mode 100644 leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
delete mode 100644 leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
delete mode 100644 leader/src/main/java/org/apache/amaterasu/leader/yarn/JobOpts.java
delete mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
delete mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnNMCallbackHandler.scala
delete mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala