You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/04/18 07:12:49 UTC
[2/5] systemml git commit: [SYSTEMML-2249] Reduced bufferpool thread
contention (min thresholds)
[SYSTEMML-2249] Reduced bufferpool thread contention (min thresholds)
In multi-threaded environment with large degree of parallelism many
small operations, the bufferpool shows - despite the small critical
region - unnecessary contention. So far we use a min threshold of 4KB
(objects below that threshold are not subject to bufferpool eviction),
which still stems from a time when we ran CP programs in very small
drivers of 400MB. In modern scale-up environments with heap sizes of
~1TB, such a small threshold causes unnecessary contention. This patch
resolves this by using a conservative threshold of max(4KB, 1e-5 * MEM)
relative to the max heap size MEM.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/fdc55118
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/fdc55118
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/fdc55118
Branch: refs/heads/master
Commit: fdc551183b46d99944d9fb42b91ec35ee5156ceb
Parents: d1389c3
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue Apr 17 20:26:28 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue Apr 17 20:26:28 2018 -0700
----------------------------------------------------------------------
.../controlprogram/caching/CacheableData.java | 27 ++++++++++----------
1 file changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/fdc55118/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 3c001bc..1ba6c4c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -76,21 +76,22 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
/** Global logging instance for all subclasses of CacheableData */
protected static final Log LOG = LogFactory.getLog(CacheableData.class.getName());
-
+
// global constant configuration parameters
- public static final long CACHING_THRESHOLD = 4*1024; //obj not s.t. caching if below threshold [in bytes]
- public static final double CACHING_BUFFER_SIZE = 0.15;
+ public static final long CACHING_THRESHOLD = (long)Math.max(4*1024, //obj not s.t. caching
+ 1e-5 * InfrastructureAnalyzer.getLocalMaxMemory()); //if below threshold [in bytes]
+ public static final double CACHING_BUFFER_SIZE = 0.15;
public static final RPolicy CACHING_BUFFER_POLICY = RPolicy.FIFO;
public static final boolean CACHING_BUFFER_PAGECACHE = false;
public static final boolean CACHING_WRITE_CACHE_ON_READ = false;
public static final String CACHING_COUNTER_GROUP_NAME = "SystemML Caching Counters";
public static final String CACHING_EVICTION_FILEEXTENSION = ".dat";
public static final boolean CACHING_ASYNC_FILECLEANUP = true;
-
+
/**
* Defines all possible cache status types for a data blob.
- * An object of class {@link CacheableData} can be in one of the following
- * five status types:
+ * An object of class {@link CacheableData} can be in one of the following
+ * five status types:
*
* <code>EMPTY</code>: Either there is no data blob at all, or the data blob
* resides in a specified import file and has never been downloaded yet.
@@ -102,13 +103,13 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
* <code>CACHED</code>: The data blob is in main memory, and nobody is using nor referencing it.
* There is always an persistent recovery object for it
**/
- protected enum CacheStatus {
- EMPTY,
- READ,
- MODIFY,
- CACHED,
- CACHED_NOWRITE,
- }
+ protected enum CacheStatus {
+ EMPTY,
+ READ,
+ MODIFY,
+ CACHED,
+ CACHED_NOWRITE,
+ }
/** Global flag indicating if caching is enabled (controls eviction) */
private static volatile boolean _activeFlag = false;