You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by na...@apache.org on 2019/02/20 09:36:28 UTC
[incubator-amaterasu] branch master updated (d722228 -> f88be0a)
This is an automated email from the ASF dual-hosted git repository.
nadavha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.
from d722228 Merge pull request #43 from anonymousGiraffe/fix-debug-test
new 812ebc4 started migrating the SDK to kotlin
new 382a58a moved the default action resource files to getActionResources
new ab02a64 moving from spark shell to spark jar implementation
new f7ec145 added artifact support to ActionData and SparkSubmitScalaRunnerProvider
new 435eaef added support for binary actions
new 51587ea fixed the command executed by the jar executor
new 7b78833 spark jars are executing correctly but spark is not loaded to the container
new 0ffb969 spark is now loaded to the container
new 7a36e3b hybrid executor mode for mesos
new da97347 TestNotifier fixed
new fa9b743 hybrid mode completed
new b71845e switched to the correct feature branch
new b1c1ea2 removed all hardcoded spark version
new 7023909 removed debug messages
new f88be0a Merge pull request #42 from roadan/AMATERASU-8
The 572 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
common/build.gradle | 9 +-
.../common/configuration/enums/ActionStatus.kt | 12 +-
.../amaterasu/common/dataobjects/ActionData.kt | 11 +-
.../amaterasu/common/dataobjects/Artifact.kt} | 2 +-
.../apache/amaterasu/common/dataobjects/Repo.kt} | 2 +-
.../common/execution/actions/Notification.kt} | 8 +-
.../common/execution/actions/Notifier.kt} | 15 +-
.../actions/enums/NotificationLevel.kt} | 10 +-
.../execution/actions/enums/NotificationType.kt} | 8 +-
.../apache/amaterasu/common/utils/ArtifactUtil.kt | 56 +++++
.../common/configuration/ClusterConfig.scala | 4 +-
.../common/dataobjects/ActionDataHelper.scala | 32 ---
.../common/execution/actions/Notifier.scala | 56 -----
.../executor/common/executors/ActiveNotifier.scala | 17 +-
.../mesos/executors/MesosActionsExecutor.scala | 4 +-
.../executor/mesos/executors/MesosNotifier.scala | 17 +-
.../executor/yarn/executors/YarnNotifier.scala | 6 +-
.../spark/dispatcher/SparkSetupProvider.scala | 4 +-
.../runners/providers/PySparkRunnerProvider.scala | 4 +-
.../providers/SparkScalaRunnerProvider.scala | 5 +-
.../providers/SparkShellScalaRunnerProvider.scala | 30 ---
.../providers/SparkSubmitScalaRunnerProvider.scala | 48 ++++
frameworks/spark/runner/build.gradle | 3 -
.../spark/runner/pyspark/PySparkEntryPoint.java | 2 +-
.../spark/runner/pyspark/PySparkRunner.scala | 2 +-
.../spark/runner/repl/SparkRunnerHelper.scala | 4 +-
.../spark/runner/repl/SparkScalaRunner.scala | 8 +-
.../spark/runner/sparksql/SparkSqlRunner.scala | 2 +-
.../runner/src/test/resources/amaterasu.properties | 14 ++
.../apache/amaterasu/utilities/TestNotifier.scala | 6 +-
leader-common/build.gradle | 7 +-
.../amaterasu/leader/common/dsl/JobParser.kt | 40 +++-
.../leader/common/execution/JobManager.kt | 33 +--
.../leader/common/execution/actions/Action.kt | 22 +-
.../leader/common/execution/actions/ErrorAction.kt | 4 +-
.../common/execution/actions/SequentialAction.kt | 4 +-
.../execution/actions/SequentialActionBase.kt | 12 +-
.../common/utilities/ActiveReportListener.kt | 64 ++++++
.../leader/common/utilities/MessagingClientUtil.kt | 57 +++++
.../common/utilities/ActiveReportListener.scala | 70 ------
.../leader/common/utilities/DataLoader.scala | 7 +-
.../common/configuration/ConfigManagerTests.kt | 1 -
.../leader/common/dsl/JobParserArtifactTests.kt | 69 ++++++
.../resources/{maki.yml => artifact-maki.yaml} | 37 +--
leader/build.gradle | 14 +-
.../scala/org/apache/amaterasu/leader/Kami.scala | 111 ---------
.../org/apache/amaterasu/leader/dsl/GitUtil.scala | 45 ----
.../amaterasu/leader/execution/JobLoader.scala | 4 +-
.../apache/amaterasu/leader/mesos/Launcher.scala | 63 -----
.../mesos/schedulers/AmaterasuScheduler.scala | 2 +-
.../leader/mesos/schedulers/ClusterScheduler.scala | 135 -----------
.../leader/mesos/schedulers/JobScheduler.scala | 254 +++++++++++----------
.../amaterasu/leader/utilities/HttpServer.scala | 4 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 64 +++---
.../leader/yarn/YarnNMCallbackHandler.scala | 2 +-
.../leader/yarn/YarnRMCallbackHandler.scala | 6 +-
leader/src/test/resources/simple-maki.yml | 3 +-
.../common/execution/ActionStatusTests.scala | 6 +-
.../common/execution/JobExecutionTests.scala | 48 ++--
.../common/execution/JobRestoreTests.scala | 12 +-
.../leader/mesos/ClusterSchedulerTests.scala | 58 -----
project/build.properties | 1 +
sdk/build.gradle | 68 +++++-
.../sdk/frameworks/RunnerSetupProvider.java | 31 ---
.../configuration/DriverConfiguration.java | 37 ---
.../sdk/frameworks/FrameworkSetupProvider.kt} | 25 +-
.../sdk/frameworks/RunnerSetupProvider.kt | 39 ++++
.../configuration/DriverConfiguration.kt | 6 +-
.../sdk/frameworks/RunnerSetupProviderTests.kt | 22 ++
.../amaterasu/sdk/frameworks/TestRunnerProvider.kt | 23 ++
70 files changed, 897 insertions(+), 1014 deletions(-)
copy common/src/main/{scala/org/apache/amaterasu/common/dataobjects/Export.scala => kotlin/org/apache/amaterasu/common/dataobjects/Artifact.kt} (90%)
copy common/src/main/{scala/org/apache/amaterasu/common/dataobjects/Export.scala => kotlin/org/apache/amaterasu/common/dataobjects/Repo.kt} (92%)
copy common/src/main/{scala/org/apache/amaterasu/common/dataobjects/ExecData.scala => kotlin/org/apache/amaterasu/common/execution/actions/Notification.kt} (70%)
copy common/src/main/{scala/org/apache/amaterasu/common/runtime/Environment.scala => kotlin/org/apache/amaterasu/common/execution/actions/Notifier.kt} (74%)
mode change 100755 => 100644
copy common/src/main/kotlin/org/apache/amaterasu/common/{logging/KLogging.kt => execution/actions/enums/NotificationLevel.kt} (82%)
copy common/src/main/{scala/org/apache/amaterasu/common/dataobjects/Export.scala => kotlin/org/apache/amaterasu/common/execution/actions/enums/NotificationType.kt} (82%)
create mode 100644 common/src/main/kotlin/org/apache/amaterasu/common/utils/ArtifactUtil.kt
delete mode 100644 common/src/main/scala/org/apache/amaterasu/common/dataobjects/ActionDataHelper.scala
delete mode 100755 common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
delete mode 100644 frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/SparkShellScalaRunnerProvider.scala
create mode 100644 frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/SparkSubmitScalaRunnerProvider.scala
create mode 100644 leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/ActiveReportListener.kt
create mode 100644 leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/MessagingClientUtil.kt
delete mode 100644 leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/ActiveReportListener.scala
create mode 100644 leader-common/src/test/kotlin/org/apache/amaterasu/leader/common/dsl/JobParserArtifactTests.kt
copy leader-common/src/test/resources/{maki.yml => artifact-maki.yaml} (64%)
delete mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/Kami.scala
delete mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/dsl/GitUtil.scala
delete mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/mesos/Launcher.scala
delete mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/ClusterScheduler.scala
delete mode 100755 leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
create mode 100644 project/build.properties
delete mode 100644 sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.java
delete mode 100644 sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/configuration/DriverConfiguration.java
rename sdk/src/main/{java/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.java => kotlin/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.kt} (62%)
create mode 100644 sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.kt
rename leader/src/main/scala/org/apache/amaterasu/leader/package.scala => sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/configuration/DriverConfiguration.kt (86%)
mode change 100755 => 100644
create mode 100644 sdk/src/test/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProviderTests.kt
create mode 100644 sdk/src/test/kotlin/org/apache/amaterasu/sdk/frameworks/TestRunnerProvider.kt