You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ra...@apache.org on 2017/06/23 16:39:56 UTC
[26/50] [abbrv] incubator-spot git commit: Reverting changes to
spot-ml/ml_test.sh (--num-executors). Adding new parameters / updated to
match last version of ml_ops.sh.
Reverting changes to spot-ml/ml_test.sh (--num-executors). Adding new parameters / updated to match last version of ml_ops.sh.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/19002d5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/19002d5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/19002d5a
Branch: refs/heads/SPOT-35_graphql_api
Commit: 19002d5a5f9579dd16ba324541726ea96cf4ad0b
Parents: 7d647f1
Author: Ricardo Barona <ri...@intel.com>
Authored: Tue Jun 13 15:50:16 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Tue Jun 13 15:50:16 2017 -0500
----------------------------------------------------------------------
spot-ml/ml_test.sh | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/19002d5a/spot-ml/ml_test.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_test.sh b/spot-ml/ml_test.sh
index 8fed837..801cedd 100755
--- a/spot-ml/ml_test.sh
+++ b/spot-ml/ml_test.sh
@@ -55,17 +55,21 @@ hdfs dfs -rm -R -f ${HDFS_SCORED_CONNECTS}
time spark-submit --class "org.apache.spot.SuspiciousConnects" \
--master yarn-client \
--driver-memory ${SPK_DRIVER_MEM} \
- --num-executors ${SPK_EXEC} \
--conf spark.driver.maxResultSize=${SPK_DRIVER_MAX_RESULTS} \
--conf spark.driver.maxPermSize=512m \
+ --conf spark.driver.cores=1 \
--conf spark.dynamicAllocation.enabled=true \
+ --conf spark.dynamicAllocation.minExecutors=1 \
+ --conf spark.dynamicAllocation.maxExecutors=${SPK_EXEC} \
--conf spark.executor.cores=${SPK_EXEC_CORES} \
--conf spark.executor.memory=${SPK_EXEC_MEM} \
--conf spark.sql.autoBroadcastJoinThreshold=${SPK_AUTO_BRDCST_JOIN_THR} \
--conf "spark.executor.extraJavaOptions=-XX:MaxPermSize=512M -XX:PermSize=512M" \
+ --conf spark.shuffle.io.preferDirectBufs=false \
--conf spark.kryoserializer.buffer.max=512m \
- --conf spark.yarn.am.waitTime=100s \
- --conf spark.yarn.am.memoryOverhead=${SPK_DRIVER_MEM_OVERHEAD} \
+ --conf spark.shuffle.service.enabled=true \
+ --conf spark.yarn.am.waitTime=1000000 \
+ --conf spark.yarn.driver.memoryOverhead=${SPK_DRIVER_MEM_OVERHEAD} \
--conf spark.yarn.executor.memoryOverhead=${SPK_EXEC_MEM_OVERHEAD} target/scala-2.10/spot-ml-assembly-1.1.jar \
--analysis ${DSOURCE} \
--input ${RAWDATA_PATH} \