You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ra...@apache.org on 2017/06/23 16:39:56 UTC

[26/50] [abbrv] incubator-spot git commit: Reverting changes to spot-ml/ml_test.sh (--num-executors). Adding new parameters / updated to match last version of ml_ops.sh.

Reverting changes to spot-ml/ml_test.sh (--num-executors). Adding new parameters / updated to match last version of ml_ops.sh.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/19002d5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/19002d5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/19002d5a

Branch: refs/heads/SPOT-35_graphql_api
Commit: 19002d5a5f9579dd16ba324541726ea96cf4ad0b
Parents: 7d647f1
Author: Ricardo Barona <ri...@intel.com>
Authored: Tue Jun 13 15:50:16 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Tue Jun 13 15:50:16 2017 -0500

----------------------------------------------------------------------
 spot-ml/ml_test.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/19002d5a/spot-ml/ml_test.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_test.sh b/spot-ml/ml_test.sh
index 8fed837..801cedd 100755
--- a/spot-ml/ml_test.sh
+++ b/spot-ml/ml_test.sh
@@ -55,17 +55,21 @@ hdfs dfs -rm -R -f ${HDFS_SCORED_CONNECTS}
 time spark-submit --class "org.apache.spot.SuspiciousConnects" \
   --master yarn-client \
   --driver-memory ${SPK_DRIVER_MEM} \
-  --num-executors ${SPK_EXEC} \
   --conf spark.driver.maxResultSize=${SPK_DRIVER_MAX_RESULTS} \
   --conf spark.driver.maxPermSize=512m \
+  --conf spark.driver.cores=1 \
   --conf spark.dynamicAllocation.enabled=true \
+  --conf spark.dynamicAllocation.minExecutors=1 \
+  --conf spark.dynamicAllocation.maxExecutors=${SPK_EXEC} \
   --conf spark.executor.cores=${SPK_EXEC_CORES} \
   --conf spark.executor.memory=${SPK_EXEC_MEM} \
   --conf spark.sql.autoBroadcastJoinThreshold=${SPK_AUTO_BRDCST_JOIN_THR} \
   --conf "spark.executor.extraJavaOptions=-XX:MaxPermSize=512M -XX:PermSize=512M" \
+  --conf spark.shuffle.io.preferDirectBufs=false    \
   --conf spark.kryoserializer.buffer.max=512m \
-  --conf spark.yarn.am.waitTime=100s \
-  --conf spark.yarn.am.memoryOverhead=${SPK_DRIVER_MEM_OVERHEAD} \
+  --conf spark.shuffle.service.enabled=true \
+  --conf spark.yarn.am.waitTime=1000000 \
+  --conf spark.yarn.driver.memoryOverhead=${SPK_DRIVER_MEM_OVERHEAD} \
   --conf spark.yarn.executor.memoryOverhead=${SPK_EXEC_MEM_OVERHEAD} target/scala-2.10/spot-ml-assembly-1.1.jar \
   --analysis ${DSOURCE} \
   --input ${RAWDATA_PATH}  \