You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ac...@apache.org on 2016/09/22 05:36:45 UTC
[5/6] incubator-systemml git commit: [SYSTEMML-930] Warn for result
size when call through MLContext
[SYSTEMML-930] Warn for result size when call through MLContext
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c8076d04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c8076d04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c8076d04
Branch: refs/heads/master
Commit: c8076d04a2b2512614a91c22b05c8c5599922f5f
Parents: 4bc6601
Author: Arvind Surve <ac...@yahoo.com>
Authored: Wed Sep 21 22:22:38 2016 -0700
Committer: Arvind Surve <ac...@yahoo.com>
Committed: Wed Sep 21 22:26:59 2016 -0700
----------------------------------------------------------------------
.../context/SparkExecutionContext.java | 10 +++++++++-
.../apache/sysml/runtime/util/UtilFunctions.java | 18 ++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8076d04/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index f95e0d4..a6f99e8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -233,7 +233,15 @@ public class SparkExecutionContext extends ExecutionContext
_spctx = new JavaSparkContext(conf);
}
}
-
+
+ // Set warning if spark.driver.maxResultSize is not set. It needs to be set before starting Spark Context for CP collect
+ String strDriverMaxResSize = _spctx.getConf().get("spark.driver.maxResultSize", "1g");
+ long driverMaxResSize = UtilFunctions.parseMemorySize(strDriverMaxResSize);
+ if (driverMaxResSize != 0 && driverMaxResSize<OptimizerUtils.getLocalMemBudget())
+ LOG.warn("Configuration parameter spark.driver.maxResultSize set to " + UtilFunctions.formatMemorySize(driverMaxResSize) + "."
+ + " You can set it through Spark default configuration setting either to 0 (unlimited) or to available memory budget of size "
+ + UtilFunctions.formatMemorySize((long)OptimizerUtils.getLocalMemBudget()) + ".");
+
//globally add binaryblock serialization framework for all hdfs read/write operations
//TODO if spark context passed in from outside (mlcontext), we need to clean this up at the end
if( MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8076d04/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index 81a1e8f..e5a792a 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -556,6 +556,24 @@ public class UtilFunctions
}
/**
+ * Format a memory size with g/m/k quantifiers into its
+ * number representation.
+ *
+ * @param arg
+ * @return
+ */
+ public static String formatMemorySize(long arg) {
+ if (arg >= 1024 * 1024 * 1024)
+ return String.format("%d GB", arg/(1024*1024*1024));
+ else if (arg >= 1024 * 1024)
+ return String.format("%d MB", arg/(1024*1024));
+ else if (arg >= 1024)
+ return String.format("%d KB", arg/(1024));
+ else
+ return String.format("%d", arg);
+ }
+
+ /**
*
* @param low lower bound (inclusive)
* @param up upper bound (inclusive)