You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by ya...@apache.org on 2018/01/11 08:45:38 UTC
[incubator-amaterasu] 01/01: Merge pull request #3 from
eyalbenivri/feature/yarn-support-pyfix
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 8b07446a8619365f398759b1099320d8fc2cbc3f
Merge: f9306bc 55d4d5a
Author: Yaniv Rodenski <ro...@gmail.com>
AuthorDate: Thu Jan 11 19:45:35 2018 +1100
Merge pull request #3 from eyalbenivri/feature/yarn-support-pyfix
Feature/yarn support pyfix
common/build.gradle | 15 +-
.../common/configuration/ClusterConfig.scala | 102 ++-
.../common/configuration/SparkConfig.scala | 0
.../common/configuration/enums/ActionStatus.scala | 2 +-
.../common/dataobjects/ActionData.scala | 4 +-
.../common/dataobjects/ActionDataHelper.scala | 14 +
.../common/dataobjects/ExecData.scala | 0
.../common/dataobjects/Export.scala | 0
.../common/dataobjects/JobData.scala | 0
.../common/dataobjects/TaskData.scala | 0
.../common/execution/actions/Notifier.scala | 3 -
.../execution/dependencies/Dependencies.scala | 3 -
.../common/logging/Logging.scala | 0
.../common/runtime/Environment.scala | 3 -
executor/build.gradle | 16 +-
.../executors/ProvidersFactory.scala | 7 +-
.../spark/PySpark/PySparkExecutionQueue.scala | 6 +-
.../runners/spark/PySpark/PySparkResultQueue.scala | 3 -
.../runners/spark/PySpark/PySparkRunner.scala | 11 +-
.../runners/spark/PySpark/ResultQueue.scala | 6 +-
.../actions/runners/spark/SparkRRunner.scala | 4 +-
.../runners/spark/SparkRunnersProvider.scala | 29 +-
.../runners/spark/SparkSql/SparkSqlRunner.scala | 1 -
.../executor/mesos/executors/ActionsExecutor.scala | 11 +-
.../executor/mesos/executors/MesosNotifier.scala | 4 +-
.../executor/yarn/executors/ActionsExecutor.scala | 94 +++
.../executor/yarn/executors/YarnNotifier.scala | 23 +
.../spark/repl/amaterasu/AmaSparkILoop.scala | 2 +-
.../runners/spark/SparkRunnerHelper.scala | 79 ++-
.../amaterasu/runners/spark/SparkScalaRunner.scala | 24 +-
executor/src/test/resources/amaterasu.properties | 10 +-
executor/src/test/resources/py4j/backport.py | 233 -------
executor/src/test/resources/py4j/clientserver.py | 550 ---------------
executor/src/test/resources/py4j/signals.py | 123 ----
.../src/test/resources/py4j/tests/benchmark1.py | 146 ----
.../resources/py4j/tests/client_server_test.py | 537 ---------------
.../src/test/resources/py4j/tests/instrumented.py | 166 -----
.../src/test/resources/py4j/tests/java_tls_test.py | 100 ---
.../test/resources/py4j/tests/memory_leak_test.py | 767 ---------------------
.../py4j/tests/py4j_callback_listener_example.py | 25 -
.../py4j/tests/py4j_callback_recursive_example.py | 78 ---
.../test/resources/py4j/tests/py4j_signals_test.py | 151 ----
.../src/test/resources/py4j/tests/signals_test.py | 97 ---
.../src/test/resources/pyspark-with-amacontext.py | 5 +-
executor/src/test/resources/pyspark.zip | Bin 442927 -> 454461 bytes
executor/src/test/resources/runtime.py | 18 +-
executor/src/test/resources/simple-pyspark.py | 13 +-
executor/src/test/resources/spark_intp.py | 39 +-
.../RunnersTests/RunnersLoadingTests.scala | 19 -
.../RunnersTests/RunnersLoadingTests.scala | 2 +-
.../amaterasu/spark/PySparkRunnerTests.scala | 24 +-
.../amaterasu/spark/SparkSqlRunnerTests.scala | 8 +-
.../apache/amaterasu/spark/SparkTestsSuite.scala | 19 +-
.../apache/amaterasu/utilities/TestNotifier.scala | 4 +-
leader/build.gradle | 11 +-
.../apache/amaterasu/leader/yarn/ArgsParser.java | 61 ++
.../org/apache/amaterasu/leader/yarn/Client.java | 268 +++++++
.../org/apache/amaterasu/leader/yarn/JobOpts.java | 12 +
leader/src/main/resources/log4j.properties | 8 +
.../amaterasu/leader/execution/JobLoader.scala | 6 +-
.../amaterasu/leader/execution/JobManager.scala | 6 +-
.../leader/execution/actions/Action.scala | 2 +-
.../execution/actions/SequentialAction.scala | 6 +-
.../amaterasu/leader/mesos/JobLauncher.scala | 113 ---
.../amaterasu/leader/mesos/MesosJobLauncher.scala | 45 ++
.../leader/mesos/schedulers/JobScheduler.scala | 19 +-
.../apache/amaterasu/leader/utilities/Args.scala | 67 ++
.../leader/utilities/BaseJobLauncher.scala | 26 +
.../amaterasu/leader/utilities/DataLoader.scala | 29 +-
.../amaterasu/leader/utilities/HttpServer.scala | 2 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 387 +++++++++++
.../leader/yarn/YarnNMCallbackHandler.scala | 53 ++
.../leader/yarn/YarnRMCallbackHandler.scala | 135 ++++
.../scripts/{ama-start.sh => ama-start-mesos.sh} | 5 +
.../scripts/{ama-start.sh => ama-start-yarn.sh} | 46 +-
leader/src/main/scripts/amaterasu.properties | 18 +-
leader/src/main/scripts/log4j.properties | 8 +
leader/src/test/resources/amaterasu.properties | 1 +
.../amaterasu/common/execution/ActionTests.scala | 7 +-
.../common/execution/JobRestoreTests.scala | 8 +-
.../amaterasu/leader/mesos/MesosTestUtil.scala | 4 +-
.../amaterasu/utilities/HttpServerTests.scala | 31 +-
.../org/apache/amaterasu/sdk/AmaterasuRunner.java | 3 -
.../org/apache/amaterasu/sdk/RunnersProvider.java | 5 +-
84 files changed, 1614 insertions(+), 3378 deletions(-)
--
To stop receiving notification emails like this one, please contact
"commits@amaterasu.apache.org" <co...@amaterasu.apache.org>.