You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amaterasu.apache.org by ya...@apache.org on 2018/07/07 08:28:55 UTC
[incubator-amaterasu] branch master updated (172f60a -> ae84d8e)
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.
from 172f60a Merge pull request #28 from apache/version-0.2.0-incubating-rc4
add 47e1c11 AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project (Master)
add f8fb7a2 fixed mesos classpath
add 967c670 fixed miniconda typo
new ae84d8e Merge pull request #30 from arunma/Amaterasu-24-Master
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 2 +
build.gradle | 4 -
.../common/execution/actions/Notifier.scala | 4 +-
executor/build.gradle | 14 +---
.../mesos/executors/MesosActionsExecutor.scala | 4 +-
.../executor/yarn/executors/ActionsExecutor.scala | 8 +-
{executor => frameworks/spark/runner}/build.gradle | 14 ++--
.../spark/runner/pyspark}/PySparkEntryPoint.java | 11 +--
.../spark/runner/src/main}/resources/codegen.py | 0
.../spark/runner/src/main}/resources/runtime.py | 0
.../main/resources/spark-version-info.properties | 11 +++
.../spark/runner}/src/main/resources/spark_intp.py | 0
.../spark/runner}/SparkRunnersProvider.scala | 10 +--
.../runner/pyspark}/PySparkExecutionQueue.scala | 2 +-
.../spark/runner/pyspark}/PySparkResultQueue.scala | 4 +-
.../spark/runner/pyspark}/PySparkRunner.scala | 10 +--
.../spark/runner/pyspark}/ResultQueue.scala | 2 +-
.../spark/runner/repl}/AmaSparkILoop.scala | 2 +-
.../spark/runner/repl}/SparkRunnerHelper.scala | 14 ++--
.../spark/runner/repl}/SparkScalaRunner.scala | 6 +-
.../spark/runner/sparkr}/SparkRRunner.scala | 28 +++----
.../spark/runner/sparksql}/SparkSqlRunner.scala | 9 ++-
.../resources/SparkSql/csv/SparkSqlTestCsv.csv | 0
.../resources/SparkSql/json/SparkSqlTestData.json | 0
...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
.../src/test/resources/SparkSql/parquet/_SUCCESS | 0
.../resources/SparkSql/parquet/_common_metadata | Bin
.../src/test/resources/SparkSql/parquet/_metadata | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
.../src/test/resources/amaterasu.properties | 0
.../spark/runner/src/test}/resources/codegen.py | 0
.../runner}/src/test/resources/py4j-0.10.4-src.zip | Bin
.../spark/runner}/src/test/resources/py4j.tar.gz | Bin
.../src/test/resources/pyspark-with-amacontext.py | 0
.../runner}/src/test/resources/pyspark.tar.gz | Bin
.../spark/runner}/src/test/resources/pyspark.zip | Bin
.../spark/runner/src/test}/resources/runtime.py | 0
.../runner}/src/test/resources/simple-pyspark.py | 0
.../src/test/resources/simple-python-err.py | 0
.../runner}/src/test/resources/simple-python.py | 0
.../runner}/src/test/resources/simple-spark.scala | 5 +-
.../test}/resources/spark-version-info.properties | 0
.../spark/runner}/src/test/resources/spark_intp.py | 0
.../spark/runner}/src/test/resources/step-2.scala | 3 +-
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
.../framework}/spark/SparkTestsSuite.scala | 10 +--
.../spark/runner}/RunnersLoadingTests.scala | 2 +-
.../spark/runner/pyspark}/PySparkRunnerTests.scala | 3 +-
.../spark/runner/repl}/SparkScalaRunnerTests.scala | 9 +--
.../runner/sparksql}/SparkSqlRunnerTests.scala | 9 +--
.../apache/amaterasu/utilities/TestNotifier.scala | 0
frameworks/spark/runtime/build.gradle | 89 +++++++++++++++++++++
.../framework/spark}/runtime/AmaContext.scala | 8 +-
.../apache/amaterasu/leader/yarn/ArgsParser.java | 6 +-
.../org/apache/amaterasu/leader/yarn/Client.java | 23 +++++-
.../apache/amaterasu/leader/dsl/JobParser.scala | 4 +-
.../frameworks/spark/SparkSetupProvider.scala | 1 -
.../mesos/schedulers/AmaterasuScheduler.scala | 1 -
.../leader/mesos/schedulers/JobScheduler.scala | 17 +++-
.../leader/utilities/ActiveReportListener.scala | 2 -
.../amaterasu/leader/utilities/HttpServer.scala | 8 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 39 ++++++---
.../leader/yarn/YarnRMCallbackHandler.scala | 5 +-
leader/src/main/scripts/ama-start-mesos.sh | 4 +-
leader/src/main/scripts/ama-start-yarn.sh | 4 +-
.../leader/mesos/ClusterSchedulerTests.scala | 2 +-
.../amaterasu/utilities/HttpServerTests.scala | 6 --
settings.gradle | 14 +++-
71 files changed, 267 insertions(+), 166 deletions(-)
copy {executor => frameworks/spark/runner}/build.gradle (93%)
rename {executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark => frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkEntryPoint.java (92%)
rename {executor/src/test => frameworks/spark/runner/src/main}/resources/codegen.py (100%)
rename {executor/src/test => frameworks/spark/runner/src/main}/resources/runtime.py (100%)
create mode 100644 frameworks/spark/runner/src/main/resources/spark-version-info.properties
rename {executor => frameworks/spark/runner}/src/main/resources/spark_intp.py (100%)
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner}/SparkRunnersProvider.scala (93%)
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkExecutionQueue.scala (94%)
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkResultQueue.scala (85%)
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkRunner.scala (95%)
mode change 100755 => 100644
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/ResultQueue.scala (94%)
rename {executor/src/main/scala/org/apache/spark/repl/amaterasu => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl}/AmaSparkILoop.scala (95%)
rename {executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl}/SparkRunnerHelper.scala (94%)
rename {executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl}/SparkScalaRunner.scala (97%)
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr}/SparkRRunner.scala (69%)
rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql}/SparkSqlRunner.scala (96%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/json/SparkSqlTestData.json (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/_SUCCESS (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/_common_metadata (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/_metadata (100%)
rename {executor/src/test/resources/tmp/job/start/x => frameworks/spark/runner/src/test/resources/SparkSql/parquet}/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet (100%)
rename {executor/src/test/resources/tmp/job/start/x => frameworks/spark/runner/src/test/resources/SparkSql/parquet}/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/amaterasu.properties (100%)
rename {executor/src/main => frameworks/spark/runner/src/test}/resources/codegen.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/py4j-0.10.4-src.zip (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/py4j.tar.gz (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/pyspark-with-amacontext.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/pyspark.tar.gz (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/pyspark.zip (100%)
rename {executor/src/main => frameworks/spark/runner/src/test}/resources/runtime.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/simple-pyspark.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/simple-python-err.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/simple-python.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/simple-spark.scala (83%)
rename {executor/src/main => frameworks/spark/runner/src/test}/resources/spark-version-info.properties (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/spark_intp.py (100%)
rename {executor => frameworks/spark/runner}/src/test/resources/step-2.scala (94%)
rename {executor/src/test/resources/SparkSql/parquet => frameworks/spark/runner/src/test/resources/tmp/job/start/x}/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet (100%)
rename {executor/src/test/resources/SparkSql/parquet => frameworks/spark/runner/src/test/resources/tmp/job/start/x}/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet (100%)
rename {executor/src/test/scala/org/apache/amaterasu => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework}/spark/SparkTestsSuite.scala (90%)
rename {executor/src/test/scala/org/apache/amaterasu/RunnersTests => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner}/RunnersLoadingTests.scala (96%)
rename {executor/src/test/scala/org/apache/amaterasu/spark => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkRunnerTests.scala (96%)
rename {executor/src/test/scala/org/apache/amaterasu/spark => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl}/SparkScalaRunnerTests.scala (92%)
rename {executor/src/test/scala/org/apache/amaterasu/spark => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql}/SparkSqlRunnerTests.scala (96%)
rename {executor => frameworks/spark/runner}/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala (100%)
create mode 100644 frameworks/spark/runtime/build.gradle
rename {executor/src/main/scala/org/apache/amaterasu/executor => frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark}/runtime/AmaContext.scala (93%)
mode change 100755 => 100644
[incubator-amaterasu] 01/01: Merge pull request #30 from
arunma/Amaterasu-24-Master
Posted by ya...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit ae84d8e54039c4ef829792ae5fb309028be130a6
Merge: 172f60a 967c670
Author: Yaniv Rodenski <ro...@gmail.com>
AuthorDate: Sat Jul 7 18:28:53 2018 +1000
Merge pull request #30 from arunma/Amaterasu-24-Master
AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own pro…
.gitignore | 2 +
build.gradle | 4 -
.../common/execution/actions/Notifier.scala | 4 +-
executor/build.gradle | 14 +---
.../mesos/executors/MesosActionsExecutor.scala | 4 +-
.../executor/yarn/executors/ActionsExecutor.scala | 8 +-
{executor => frameworks/spark/runner}/build.gradle | 14 ++--
.../spark/runner/pyspark}/PySparkEntryPoint.java | 11 +--
.../spark/runner/src/main}/resources/codegen.py | 0
.../spark/runner/src/main}/resources/runtime.py | 0
.../main/resources/spark-version-info.properties | 11 +++
.../spark/runner}/src/main/resources/spark_intp.py | 0
.../spark/runner}/SparkRunnersProvider.scala | 10 +--
.../runner/pyspark}/PySparkExecutionQueue.scala | 2 +-
.../spark/runner/pyspark}/PySparkResultQueue.scala | 4 +-
.../spark/runner/pyspark}/PySparkRunner.scala | 10 +--
.../spark/runner/pyspark}/ResultQueue.scala | 2 +-
.../spark/runner/repl}/AmaSparkILoop.scala | 2 +-
.../spark/runner/repl}/SparkRunnerHelper.scala | 14 ++--
.../spark/runner/repl}/SparkScalaRunner.scala | 6 +-
.../spark/runner/sparkr}/SparkRRunner.scala | 28 +++----
.../spark/runner/sparksql}/SparkSqlRunner.scala | 9 ++-
.../resources/SparkSql/csv/SparkSqlTestCsv.csv | 0
.../resources/SparkSql/json/SparkSqlTestData.json | 0
...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
.../src/test/resources/SparkSql/parquet/_SUCCESS | 0
.../resources/SparkSql/parquet/_common_metadata | Bin
.../src/test/resources/SparkSql/parquet/_metadata | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
.../src/test/resources/amaterasu.properties | 0
.../spark/runner/src/test}/resources/codegen.py | 0
.../runner}/src/test/resources/py4j-0.10.4-src.zip | Bin
.../spark/runner}/src/test/resources/py4j.tar.gz | Bin
.../src/test/resources/pyspark-with-amacontext.py | 0
.../runner}/src/test/resources/pyspark.tar.gz | Bin
.../spark/runner}/src/test/resources/pyspark.zip | Bin
.../spark/runner/src/test}/resources/runtime.py | 0
.../runner}/src/test/resources/simple-pyspark.py | 0
.../src/test/resources/simple-python-err.py | 0
.../runner}/src/test/resources/simple-python.py | 0
.../runner}/src/test/resources/simple-spark.scala | 5 +-
.../test}/resources/spark-version-info.properties | 0
.../spark/runner}/src/test/resources/spark_intp.py | 0
.../spark/runner}/src/test/resources/step-2.scala | 3 +-
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
.../framework}/spark/SparkTestsSuite.scala | 10 +--
.../spark/runner}/RunnersLoadingTests.scala | 2 +-
.../spark/runner/pyspark}/PySparkRunnerTests.scala | 3 +-
.../spark/runner/repl}/SparkScalaRunnerTests.scala | 9 +--
.../runner/sparksql}/SparkSqlRunnerTests.scala | 9 +--
.../apache/amaterasu/utilities/TestNotifier.scala | 0
frameworks/spark/runtime/build.gradle | 89 +++++++++++++++++++++
.../framework/spark}/runtime/AmaContext.scala | 8 +-
.../apache/amaterasu/leader/yarn/ArgsParser.java | 6 +-
.../org/apache/amaterasu/leader/yarn/Client.java | 23 +++++-
.../apache/amaterasu/leader/dsl/JobParser.scala | 4 +-
.../frameworks/spark/SparkSetupProvider.scala | 1 -
.../mesos/schedulers/AmaterasuScheduler.scala | 1 -
.../leader/mesos/schedulers/JobScheduler.scala | 17 +++-
.../leader/utilities/ActiveReportListener.scala | 2 -
.../amaterasu/leader/utilities/HttpServer.scala | 8 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 39 ++++++---
.../leader/yarn/YarnRMCallbackHandler.scala | 5 +-
leader/src/main/scripts/ama-start-mesos.sh | 4 +-
leader/src/main/scripts/ama-start-yarn.sh | 4 +-
.../leader/mesos/ClusterSchedulerTests.scala | 2 +-
.../amaterasu/utilities/HttpServerTests.scala | 6 --
settings.gradle | 14 +++-
71 files changed, 267 insertions(+), 166 deletions(-)