You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@amaterasu.apache.org by GitBox <gi...@apache.org> on 2018/09/15 09:06:01 UTC
[GitHub] roadan closed pull request #26: Amaterasu 23
roadan closed pull request #26: Amaterasu 23
URL: https://github.com/apache/incubator-amaterasu/pull/26
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/build.gradle b/build.gradle
index 79b926f..2008f3c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -62,4 +62,4 @@ rat {
tasks.withType(Test) {
maxParallelForks = 1
-}
+}
\ No newline at end of file
diff --git a/executor/build.gradle b/executor/build.gradle
index 21bc2b0..9b7294e 100644
--- a/executor/build.gradle
+++ b/executor/build.gradle
@@ -16,7 +16,7 @@
*/
plugins {
id 'com.github.johnrengelman.shadow' version '1.2.4'
- id 'com.github.maiflai.scalatest' version '0.6-5-g9065d91'
+ id 'com.github.maiflai.scalatest' version '0.22'
id 'scala'
id 'java'
}
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 4b125b8..38185b0 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
-#Fri Jan 27 12:21:51 AEDT 2017
+#Sun Jun 10 14:41:17 AEST 2018
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip
diff --git a/gradlew b/gradlew
index 3efb0e9..9aa616c 100755
--- a/gradlew
+++ b/gradlew
@@ -1,20 +1,4 @@
#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
##############################################################################
##
diff --git a/gradlew.bat b/gradlew.bat
index 718266c..e95643d 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -1,102 +1,84 @@
-rem
-rem
-rem Licensed to the Apache Software Foundation (ASF) under one or more
-rem contributor license agreements. See the NOTICE file distributed with
-rem this work for additional information regarding copyright ownership.
-rem The ASF licenses this file to You under the Apache License, Version 2.0
-rem (the "License"); you may not use this file except in compliance with
-rem the License. You may obtain a copy of the License at
-rem
-rem http://www.apache.org/licenses/LICENSE-2.0
-rem
-rem Unless required by applicable law or agreed to in writing, software
-rem distributed under the License is distributed on an "AS IS" BASIS,
-rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-rem See the License for the specific language governing permissions and
-rem limitations under the License.
-rem
-
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/leader/build.gradle b/leader/build.gradle
index da29397..3f9276e 100644
--- a/leader/build.gradle
+++ b/leader/build.gradle
@@ -16,7 +16,7 @@
*/
plugins {
id "com.github.johnrengelman.shadow" version "1.2.4"
- id 'com.github.maiflai.scalatest' version '0.6-5-g9065d91'
+ id 'com.github.maiflai.scalatest' version '0.22'
id 'scala'
id 'java'
}
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala b/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
index 8c487c1..21b7be4 100644
--- a/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
@@ -19,7 +19,6 @@ package org.apache.amaterasu.leader.frameworks.spark
import java.io.File
import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.common.dataobjects.ExecData
import org.apache.amaterasu.leader.utilities.{DataLoader, MemoryFormatParser}
import org.apache.amaterasu.sdk.frameworks.FrameworkSetupProvider
import org.apache.amaterasu.sdk.frameworks.configuration.DriverConfiguration
@@ -28,21 +27,11 @@ import scala.collection.mutable
class SparkSetupProvider extends FrameworkSetupProvider {
-
- private var env: String = _
- private var conf: ClusterConfig = _
- private val runnersResources = mutable.Map[String, Array[File]]()
//private var execData: ExecData = _
private lazy val sparkExecConfigurations: mutable.Map[String, Any] = loadSparkConfig
-
- private def loadSparkConfig: mutable.Map[String, Any] = {
- val execData = DataLoader.getExecutorData(env, conf)
- val sparkExecConfigurationsurations = execData.configurations.get("spark")
- if (sparkExecConfigurationsurations.isEmpty) {
- throw new Exception(s"Spark configuration files could not be loaded for the environment ${env}")
- }
- collection.mutable.Map(sparkExecConfigurationsurations.get.toSeq: _*)
- }
+ private val runnersResources = mutable.Map[String, Array[File]]()
+ private var env: String = _
+ private var conf: ClusterConfig = _
override def init(env: String, conf: ClusterConfig): Unit = {
this.env = env
@@ -98,4 +87,24 @@ class SparkSetupProvider extends FrameworkSetupProvider {
new DriverConfiguration(mem, cpu)
}
+
+ override def getRuntimePaths: Array[String] = {
+ Array[String](
+ s"executor-${conf.version}-all.jar",
+ s"spark-${conf.Webserver.sparkVersion}/jars/*"
+ )
+ }
+
+ override def getRuntimeCommand = {
+ "java -cp"
+ }
+
+ private def loadSparkConfig: mutable.Map[String, Any] = {
+ val execData = DataLoader.getExecutorData(env, conf)
+ val sparkExecConfigurationsurations = execData.configurations.get("spark")
+ if (sparkExecConfigurationsurations.isEmpty) {
+ throw new Exception(s"Spark configuration files could not be loaded for the environment ${env}")
+ }
+ collection.mutable.Map(sparkExecConfigurationsurations.get.toSeq: _*)
+ }
}
\ No newline at end of file
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
index 87a8f5d..f47c7e4 100755
--- a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
@@ -16,11 +16,10 @@
*/
package org.apache.amaterasu.leader.mesos.schedulers
-import java.io.File
import java.util
import java.util.concurrent.locks.ReentrantLock
import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue}
-import java.util.{Collections, Properties, UUID}
+import java.util.{Collections, UUID}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -63,6 +62,7 @@ class JobScheduler extends AmaterasuScheduler {
private var branch: String = _
private var resume: Boolean = false
private var reportLevel: NotificationLevel = _
+ private var frameworkFactory: FrameworkProvidersFactory = _
val slavesExecutors = new TrieMap[String, ExecutorInfo]
private var awsEnv: String = ""
@@ -130,6 +130,7 @@ class JobScheduler extends AmaterasuScheduler {
}
def resourceOffers(driver: SchedulerDriver, offers: util.List[Offer]): Unit = {
+
for (offer <- offers.asScala) {
if (validateOffer(offer)) {
@@ -159,6 +160,9 @@ class JobScheduler extends AmaterasuScheduler {
//TODO: move to .getOrElseUpdate when migrting to scala 2.11
var executor: ExecutorInfo = null
val slaveId = offer.getSlaveId.getValue
+
+ val frameworkProvider = frameworkFactory.providers(actionData.groupId)
+
slavesExecutors.synchronized {
if (slavesExecutors.contains(slaveId) &&
offer.getExecutorIdsList.contains(slavesExecutors(slaveId).getExecutorId)) {
@@ -170,10 +174,8 @@ class JobScheduler extends AmaterasuScheduler {
val command = CommandInfo
.newBuilder
.setValue(
- s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} env MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz java -cp executor-${config.version}-all.jar:spark-${config.Webserver.sparkVersion}/jars/* -Dscala.usejavacp=true -Djava.library.path=/usr/lib org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor ${jobManager.jobId} ${config.master} ${actionData.name}""".stripMargin
+ s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} env MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz ${frameworkProvider.getRuntimeCommand} ${frameworkProvider.getRuntimePaths.mkString(":")} -Dscala.usejavacp=true -Djava.library.path=/usr/lib org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor ${jobManager.jobId} ${config.master} ${actionData.name}""".stripMargin
)
- // HttpServer.getFilesInDirectory(sys.env("AMA_NODE"), config.Webserver.Port).foreach(f=>
- // )
.addUris(URI.newBuilder
.setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/executor-${config.version}-all.jar")
.setExecutable(false)
@@ -221,8 +223,6 @@ class JobScheduler extends AmaterasuScheduler {
}
}
- val frameworkFactory = FrameworkProvidersFactory.apply(env, config)
- val frameworkProvider = frameworkFactory.providers(actionData.groupId)
val driverConfiguration = frameworkProvider.getDriverConfiguration
val actionTask = TaskInfo
@@ -347,6 +347,7 @@ object JobScheduler {
scheduler.client = CuratorFrameworkFactory.newClient(config.zk, retryPolicy)
scheduler.client.start()
scheduler.config = config
+ scheduler.frameworkFactory = FrameworkProvidersFactory(env, config)
scheduler
}
diff --git a/leader/src/main/scripts/ama-start-mesos.sh b/leader/src/main/scripts/ama-start-mesos.sh
index 18dbed9..d7a7659 100755
--- a/leader/src/main/scripts/ama-start-mesos.sh
+++ b/leader/src/main/scripts/ama-start-mesos.sh
@@ -123,7 +123,6 @@ fi
if ! ls ${BASEDIR}/dist/spark*.tgz 1> /dev/null 2>&1; then
echo "${bold} Fetching spark distributable ${NC}"
- #wget https://d3kbcqa49mib13.cloudfront.net/spark-2.2.1-bin-hadoop2.7.tgz -P ${BASEDIR}/dist
wget http://apache.mirror.digitalpacific.com.au/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz -P ${BASEDIR}/dist
fi
if [ ! -f ${BASEDIR}/dist/Miniconda2-latest-Linux-x86_64.sh ]; then
@@ -131,6 +130,7 @@ if [ ! -f ${BASEDIR}/dist/Miniconda2-latest-Linux-x86_64.sh ]; then
wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -P ${BASEDIR}/dist
fi
cp ${BASEDIR}/amaterasu.properties ${BASEDIR}/dist
+echo $CMD
eval $CMD | grep "===>"
echo ""
diff --git a/leader/src/main/scripts/log4j.properties b/leader/src/main/scripts/log4j.properties
index 742eb59..14ce344 100644
--- a/leader/src/main/scripts/log4j.properties
+++ b/leader/src/main/scripts/log4j.properties
@@ -22,4 +22,5 @@ log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-log4j.logger.reflections.Reflections=OFF
\ No newline at end of file
+log4j.logger.reflections.Reflections=OFF
+log4j.logger.org.reflections.Reflections=OFF
\ No newline at end of file
diff --git a/sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.java b/sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.java
index ef53fa9..71933a4 100644
--- a/sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.java
+++ b/sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.java
@@ -21,7 +21,7 @@
import java.io.File;
-public interface FrameworkSetupProvider {
+public interface FrameworkSetupProvider {
void init(String env, ClusterConfig conf);
@@ -33,4 +33,8 @@
DriverConfiguration getDriverConfiguration();
+ String[] getRuntimePaths();
+
+ String getRuntimeCommand();
+
}
\ No newline at end of file
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services