You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by ya...@apache.org on 2018/01/11 08:45:37 UTC
[incubator-amaterasu] branch master updated (f9306bc -> 8b07446)
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.
from f9306bc Merge pull request #1 from kirupagaran/jetty-sql-gradle-apache-upgrade
add 99b5472 yarn-support (copied from old fork) - preparing for merge.
add 81ba6a0 missing code
add 9560dc4 barely merged with origin/master, may the schwartz be with us
add 4856b83 yarn work
add b9af741 parameteriazed spark home
add 8e2708c filtered only pyfiles
add 1e7bf36 pyfiles moved to comma delimited
add 2efc722 added persistance log
add 9f28a32 added persistance log
add c52ab49 added persistance log
add 4f7cc9d test serielizing jeckson
add 77f6441 added logs
add f47ccbe Merge branch 'feature/yarn-support-pyfix' of github.com:eyalbenivri/incubator-amaterasu into feature/yarn-support-pyfix
add 486224b added yarn home dir
add 93652e7 added env YARN_CONF_DIR to container lounch
add 201d877 removed fs.hdfs.impl
add f57fb49 changed JVM maximum mem
add 598a04d log fix
add b580472 added logs again
add 1755db4 fixed env issues
add 254f7df readded anaconda
add c8f3e53 readded anaconda
add 86be76a mem fix
add 088115a added write logging and output
add a541679 Merge branch 'feature/yarn-support-pyfix' of github.com:eyalbenivri/incubator-amaterasu into feature/yarn-support-pyfix
add 0c5a4d4 added hadoop conf dir to classpath - moved us forward.
add 99f7aef added container tokens
add 9c46570 fixed scala version
add b23e0ce added scala compiler ref
add 42f7768 changed executor cp order
add fe1ba7d persistance error logging not throwing exception
add f0c023f persistance error logging not throwing exception
add f679321 logging executor classpath
add c1d595c logging executor classpath
add b8bf4d0 logging executor classpath
add eeab6de logging executor classpath
add a0c8e17 logging executor classpath
add 5bc4214 fixing executor classpath
add 91b69a0 fixing executor classpath
add 7c9b414 fixing executor classpath
add 1485a62 added driver opts to executor java command - solved issue with spark classpath
add fa2f234 fixed directory creation by spark
add f58a92b changed mem and added chmod to yarn script that should be removed.
add 09d9f38 changed mem and added chmod to yarn script that should be removed.
add 8c34847 fixed directory creation by spark
add 55d4d5a YARN jobs run end-to-end
new 8b07446 Merge pull request #3 from eyalbenivri/feature/yarn-support-pyfix
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
common/build.gradle | 15 +-
.../common/configuration/ClusterConfig.scala | 102 ++-
.../common/configuration/SparkConfig.scala | 0
.../common/configuration/enums/ActionStatus.scala | 2 +-
.../common/dataobjects/ActionData.scala | 4 +-
.../common/dataobjects/ActionDataHelper.scala | 14 +
.../common/dataobjects/ExecData.scala | 0
.../common/dataobjects/Export.scala | 0
.../common/dataobjects/JobData.scala | 0
.../common/dataobjects/TaskData.scala | 0
.../common/execution/actions/Notifier.scala | 3 -
.../execution/dependencies/Dependencies.scala | 3 -
.../common/logging/Logging.scala | 0
.../common/runtime/Environment.scala | 3 -
executor/build.gradle | 16 +-
.../executors/ProvidersFactory.scala | 7 +-
.../spark/PySpark/PySparkExecutionQueue.scala | 6 +-
.../runners/spark/PySpark/PySparkResultQueue.scala | 3 -
.../runners/spark/PySpark/PySparkRunner.scala | 11 +-
.../runners/spark/PySpark/ResultQueue.scala | 6 +-
.../actions/runners/spark/SparkRRunner.scala | 4 +-
.../runners/spark/SparkRunnersProvider.scala | 29 +-
.../runners/spark/SparkSql/SparkSqlRunner.scala | 1 -
.../executor/mesos/executors/ActionsExecutor.scala | 11 +-
.../executor/mesos/executors/MesosNotifier.scala | 4 +-
.../executor/yarn/executors/ActionsExecutor.scala | 94 +++
.../executor/yarn/executors/YarnNotifier.scala | 23 +
.../spark/repl/amaterasu/AmaSparkILoop.scala | 2 +-
.../runners/spark/SparkRunnerHelper.scala | 79 ++-
.../amaterasu/runners/spark/SparkScalaRunner.scala | 24 +-
executor/src/test/resources/amaterasu.properties | 10 +-
executor/src/test/resources/py4j/backport.py | 233 -------
executor/src/test/resources/py4j/clientserver.py | 550 ---------------
executor/src/test/resources/py4j/signals.py | 123 ----
.../src/test/resources/py4j/tests/benchmark1.py | 146 ----
.../resources/py4j/tests/client_server_test.py | 537 ---------------
.../src/test/resources/py4j/tests/instrumented.py | 166 -----
.../src/test/resources/py4j/tests/java_tls_test.py | 100 ---
.../test/resources/py4j/tests/memory_leak_test.py | 767 ---------------------
.../py4j/tests/py4j_callback_listener_example.py | 25 -
.../py4j/tests/py4j_callback_recursive_example.py | 78 ---
.../test/resources/py4j/tests/py4j_signals_test.py | 151 ----
.../src/test/resources/py4j/tests/signals_test.py | 97 ---
.../src/test/resources/pyspark-with-amacontext.py | 5 +-
executor/src/test/resources/pyspark.zip | Bin 442927 -> 454461 bytes
executor/src/test/resources/runtime.py | 18 +-
executor/src/test/resources/simple-pyspark.py | 13 +-
executor/src/test/resources/spark_intp.py | 39 +-
.../RunnersTests/RunnersLoadingTests.scala | 19 -
.../RunnersTests/RunnersLoadingTests.scala | 2 +-
.../amaterasu/spark/PySparkRunnerTests.scala | 24 +-
.../amaterasu/spark/SparkSqlRunnerTests.scala | 8 +-
.../apache/amaterasu/spark/SparkTestsSuite.scala | 19 +-
.../apache/amaterasu/utilities/TestNotifier.scala | 4 +-
leader/build.gradle | 11 +-
.../apache/amaterasu/leader/yarn/ArgsParser.java | 61 ++
.../org/apache/amaterasu/leader/yarn/Client.java | 268 +++++++
.../org/apache/amaterasu/leader/yarn/JobOpts.java | 12 +
leader/src/main/resources/log4j.properties | 8 +
.../amaterasu/leader/execution/JobLoader.scala | 6 +-
.../amaterasu/leader/execution/JobManager.scala | 6 +-
.../leader/execution/actions/Action.scala | 2 +-
.../execution/actions/SequentialAction.scala | 6 +-
.../amaterasu/leader/mesos/JobLauncher.scala | 113 ---
.../amaterasu/leader/mesos/MesosJobLauncher.scala | 45 ++
.../leader/mesos/schedulers/JobScheduler.scala | 19 +-
.../apache/amaterasu/leader/utilities/Args.scala | 67 ++
.../leader/utilities/BaseJobLauncher.scala | 26 +
.../amaterasu/leader/utilities/DataLoader.scala | 29 +-
.../amaterasu/leader/utilities/HttpServer.scala | 2 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 387 +++++++++++
.../leader/yarn/YarnNMCallbackHandler.scala | 53 ++
.../leader/yarn/YarnRMCallbackHandler.scala | 135 ++++
.../scripts/{ama-start.sh => ama-start-mesos.sh} | 5 +
.../scripts/{ama-start.sh => ama-start-yarn.sh} | 46 +-
leader/src/main/scripts/amaterasu.properties | 18 +-
leader/src/main/scripts/log4j.properties | 8 +
leader/src/test/resources/amaterasu.properties | 1 +
.../amaterasu/common/execution/ActionTests.scala | 7 +-
.../common/execution/JobRestoreTests.scala | 8 +-
.../amaterasu/leader/mesos/MesosTestUtil.scala | 4 +-
.../amaterasu/utilities/HttpServerTests.scala | 31 +-
.../org/apache/amaterasu/sdk/AmaterasuRunner.java | 3 -
.../org/apache/amaterasu/sdk/RunnersProvider.java | 5 +-
84 files changed, 1614 insertions(+), 3378 deletions(-)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/configuration/ClusterConfig.scala (62%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/configuration/SparkConfig.scala (100%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/configuration/enums/ActionStatus.scala (94%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/dataobjects/ActionData.scala (93%)
create mode 100644 common/src/main/scala/org/apache/amaterasu/common/dataobjects/ActionDataHelper.scala
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/dataobjects/ExecData.scala (100%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/dataobjects/Export.scala (100%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/dataobjects/JobData.scala (100%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/dataobjects/TaskData.scala (100%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/execution/actions/Notifier.scala (97%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/execution/dependencies/Dependencies.scala (94%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/logging/Logging.scala (100%)
rename common/src/main/scala/org/apache/{ameterasu => amaterasu}/common/runtime/Environment.scala (96%)
rename executor/src/main/scala/org/apache/amaterasu/executor/{mesos => common}/executors/ProvidersFactory.scala (93%)
create mode 100644 executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala
create mode 100644 executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/YarnNotifier.scala
delete mode 100755 executor/src/test/resources/py4j/backport.py
delete mode 100755 executor/src/test/resources/py4j/clientserver.py
delete mode 100755 executor/src/test/resources/py4j/signals.py
delete mode 100755 executor/src/test/resources/py4j/tests/benchmark1.py
delete mode 100755 executor/src/test/resources/py4j/tests/client_server_test.py
delete mode 100755 executor/src/test/resources/py4j/tests/instrumented.py
delete mode 100755 executor/src/test/resources/py4j/tests/java_tls_test.py
delete mode 100755 executor/src/test/resources/py4j/tests/memory_leak_test.py
delete mode 100755 executor/src/test/resources/py4j/tests/py4j_callback_listener_example.py
delete mode 100755 executor/src/test/resources/py4j/tests/py4j_callback_recursive_example.py
delete mode 100755 executor/src/test/resources/py4j/tests/py4j_signals_test.py
delete mode 100755 executor/src/test/resources/py4j/tests/signals_test.py
delete mode 100644 executor/src/test/scala/io/shinto/amaterasu/RunnersTests/RunnersLoadingTests.scala
create mode 100644 leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
create mode 100644 leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
create mode 100644 leader/src/main/java/org/apache/amaterasu/leader/yarn/JobOpts.java
create mode 100644 leader/src/main/resources/log4j.properties
delete mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/mesos/JobLauncher.scala
create mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/mesos/MesosJobLauncher.scala
create mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/utilities/Args.scala
create mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/utilities/BaseJobLauncher.scala
create mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
create mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnNMCallbackHandler.scala
create mode 100644 leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
copy leader/src/main/scripts/{ama-start.sh => ama-start-mesos.sh} (94%)
rename leader/src/main/scripts/{ama-start.sh => ama-start-yarn.sh} (63%)
create mode 100644 leader/src/main/scripts/log4j.properties
--
To stop receiving notification emails like this one, please contact
['"commits@amaterasu.apache.org" <co...@amaterasu.apache.org>'].
[incubator-amaterasu] 01/01: Merge pull request #3 from
eyalbenivri/feature/yarn-support-pyfix
Posted by ya...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 8b07446a8619365f398759b1099320d8fc2cbc3f
Merge: f9306bc 55d4d5a
Author: Yaniv Rodenski <ro...@gmail.com>
AuthorDate: Thu Jan 11 19:45:35 2018 +1100
Merge pull request #3 from eyalbenivri/feature/yarn-support-pyfix
Feature/yarn support pyfix
common/build.gradle | 15 +-
.../common/configuration/ClusterConfig.scala | 102 ++-
.../common/configuration/SparkConfig.scala | 0
.../common/configuration/enums/ActionStatus.scala | 2 +-
.../common/dataobjects/ActionData.scala | 4 +-
.../common/dataobjects/ActionDataHelper.scala | 14 +
.../common/dataobjects/ExecData.scala | 0
.../common/dataobjects/Export.scala | 0
.../common/dataobjects/JobData.scala | 0
.../common/dataobjects/TaskData.scala | 0
.../common/execution/actions/Notifier.scala | 3 -
.../execution/dependencies/Dependencies.scala | 3 -
.../common/logging/Logging.scala | 0
.../common/runtime/Environment.scala | 3 -
executor/build.gradle | 16 +-
.../executors/ProvidersFactory.scala | 7 +-
.../spark/PySpark/PySparkExecutionQueue.scala | 6 +-
.../runners/spark/PySpark/PySparkResultQueue.scala | 3 -
.../runners/spark/PySpark/PySparkRunner.scala | 11 +-
.../runners/spark/PySpark/ResultQueue.scala | 6 +-
.../actions/runners/spark/SparkRRunner.scala | 4 +-
.../runners/spark/SparkRunnersProvider.scala | 29 +-
.../runners/spark/SparkSql/SparkSqlRunner.scala | 1 -
.../executor/mesos/executors/ActionsExecutor.scala | 11 +-
.../executor/mesos/executors/MesosNotifier.scala | 4 +-
.../executor/yarn/executors/ActionsExecutor.scala | 94 +++
.../executor/yarn/executors/YarnNotifier.scala | 23 +
.../spark/repl/amaterasu/AmaSparkILoop.scala | 2 +-
.../runners/spark/SparkRunnerHelper.scala | 79 ++-
.../amaterasu/runners/spark/SparkScalaRunner.scala | 24 +-
executor/src/test/resources/amaterasu.properties | 10 +-
executor/src/test/resources/py4j/backport.py | 233 -------
executor/src/test/resources/py4j/clientserver.py | 550 ---------------
executor/src/test/resources/py4j/signals.py | 123 ----
.../src/test/resources/py4j/tests/benchmark1.py | 146 ----
.../resources/py4j/tests/client_server_test.py | 537 ---------------
.../src/test/resources/py4j/tests/instrumented.py | 166 -----
.../src/test/resources/py4j/tests/java_tls_test.py | 100 ---
.../test/resources/py4j/tests/memory_leak_test.py | 767 ---------------------
.../py4j/tests/py4j_callback_listener_example.py | 25 -
.../py4j/tests/py4j_callback_recursive_example.py | 78 ---
.../test/resources/py4j/tests/py4j_signals_test.py | 151 ----
.../src/test/resources/py4j/tests/signals_test.py | 97 ---
.../src/test/resources/pyspark-with-amacontext.py | 5 +-
executor/src/test/resources/pyspark.zip | Bin 442927 -> 454461 bytes
executor/src/test/resources/runtime.py | 18 +-
executor/src/test/resources/simple-pyspark.py | 13 +-
executor/src/test/resources/spark_intp.py | 39 +-
.../RunnersTests/RunnersLoadingTests.scala | 19 -
.../RunnersTests/RunnersLoadingTests.scala | 2 +-
.../amaterasu/spark/PySparkRunnerTests.scala | 24 +-
.../amaterasu/spark/SparkSqlRunnerTests.scala | 8 +-
.../apache/amaterasu/spark/SparkTestsSuite.scala | 19 +-
.../apache/amaterasu/utilities/TestNotifier.scala | 4 +-
leader/build.gradle | 11 +-
.../apache/amaterasu/leader/yarn/ArgsParser.java | 61 ++
.../org/apache/amaterasu/leader/yarn/Client.java | 268 +++++++
.../org/apache/amaterasu/leader/yarn/JobOpts.java | 12 +
leader/src/main/resources/log4j.properties | 8 +
.../amaterasu/leader/execution/JobLoader.scala | 6 +-
.../amaterasu/leader/execution/JobManager.scala | 6 +-
.../leader/execution/actions/Action.scala | 2 +-
.../execution/actions/SequentialAction.scala | 6 +-
.../amaterasu/leader/mesos/JobLauncher.scala | 113 ---
.../amaterasu/leader/mesos/MesosJobLauncher.scala | 45 ++
.../leader/mesos/schedulers/JobScheduler.scala | 19 +-
.../apache/amaterasu/leader/utilities/Args.scala | 67 ++
.../leader/utilities/BaseJobLauncher.scala | 26 +
.../amaterasu/leader/utilities/DataLoader.scala | 29 +-
.../amaterasu/leader/utilities/HttpServer.scala | 2 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 387 +++++++++++
.../leader/yarn/YarnNMCallbackHandler.scala | 53 ++
.../leader/yarn/YarnRMCallbackHandler.scala | 135 ++++
.../scripts/{ama-start.sh => ama-start-mesos.sh} | 5 +
.../scripts/{ama-start.sh => ama-start-yarn.sh} | 46 +-
leader/src/main/scripts/amaterasu.properties | 18 +-
leader/src/main/scripts/log4j.properties | 8 +
leader/src/test/resources/amaterasu.properties | 1 +
.../amaterasu/common/execution/ActionTests.scala | 7 +-
.../common/execution/JobRestoreTests.scala | 8 +-
.../amaterasu/leader/mesos/MesosTestUtil.scala | 4 +-
.../amaterasu/utilities/HttpServerTests.scala | 31 +-
.../org/apache/amaterasu/sdk/AmaterasuRunner.java | 3 -
.../org/apache/amaterasu/sdk/RunnersProvider.java | 5 +-
84 files changed, 1614 insertions(+), 3378 deletions(-)
--
To stop receiving notification emails like this one, please contact
"commits@amaterasu.apache.org" <co...@amaterasu.apache.org>.