You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/07/20 23:20:23 UTC
systemml git commit: [SYSTEMML-1791] Performance frame cache release
(avoid scans)
Repository: systemml
Updated Branches:
refs/heads/master 3d4c21b01 -> d45b84863
[SYSTEMML-1791] Performance frame cache release (avoid scans)
This patch removes unnecessary frame cache release overhead for JMLC
deployments. On cache release, we update the size of currently pinned
objects, which is only used in the hybrid spark execution mode. For
frame with string schema this creates unnecessary overhead because for
string column this is not a meta data operation but requires a full scan
over the individual strings to determine their lengths. We now disable
this in forced CP execution mode, which significantly improved
performance in the context of column indexing, which are meanwhile pure
meta data operations.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d45b8486
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d45b8486
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d45b8486
Branch: refs/heads/master
Commit: d45b84863e889f6fb2cacc31722147ce8409bf22
Parents: 3d4c21b
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Jul 20 15:39:24 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Jul 20 16:21:47 2017 -0700
----------------------------------------------------------------------
.../runtime/controlprogram/caching/CacheableData.java | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d45b8486/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 4084eb9..a9f604f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -595,7 +596,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
//update meta data
refreshMetaData();
}
-
+
//compact empty in-memory block
_data.compactEmptyBlock();
@@ -1242,11 +1243,11 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
}
protected void updateStatusPinned(boolean add) {
- if( _data != null ) { //data should never be null
- long size = sizePinned.get();
- size += (add ? 1 : -1) * _data.getInMemorySize();
- sizePinned.set( Math.max(size,0) );
- }
+ if( _data == null || !OptimizerUtils.isHybridExecutionMode() )
+ return; //avoid size computation for string frames
+ long size = sizePinned.get();
+ size += (add ? 1 : -1) * _data.getInMemorySize();
+ sizePinned.set( Math.max(size,0) );
}
protected long getPinnedSize() {