[systemds] branch master updated: [MINOR] Improve lineage cache spilling

[ar...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

arnabp20 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 1967f8b  [MINOR] Improve lineage cache spilling
1967f8b is described below

commit 1967f8bb23109b2d3c6b0692fbcbf22324295594
Author: arnabp <ar...@tugraz.at>
AuthorDate: Thu Nov 5 00:38:08 2020 +0100

    [MINOR] Improve lineage cache spilling
    
    This patch:
     - adds lineage tracing for frame indexing,
     - reduces starting computation time for spilling from 100 to 10ms
    Allowing more entries to be spilled to disk increases peroformance,
    and makes the difference between the policies smaller.
---
 .../runtime/instructions/cp/FrameIndexingCPInstruction.java      | 9 +++++++++
 .../instructions/cp/ParameterizedBuiltinCPInstruction.java       | 2 +-
 .../org/apache/sysds/runtime/lineage/LineageCacheConfig.java     | 4 ++--
 .../org/apache/sysds/runtime/lineage/LineageCacheEviction.java   | 2 +-
 4 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/FrameIndexingCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/FrameIndexingCPInstruction.java
index f5f2b93..5c2ef3c 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/FrameIndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/FrameIndexingCPInstruction.java
@@ -21,9 +21,12 @@ package org.apache.sysds.runtime.instructions.cp;
 
 import org.apache.sysds.lops.LeftIndex;
 import org.apache.sysds.lops.RightIndex;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.sysds.common.Types.DataType;
 import org.apache.sysds.runtime.DMLRuntimeException;
 import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysds.runtime.lineage.LineageItem;
+import org.apache.sysds.runtime.lineage.LineageItemUtils;
 import org.apache.sysds.runtime.matrix.data.FrameBlock;
 import org.apache.sysds.runtime.util.IndexRange;
 
@@ -83,4 +86,10 @@ public final class FrameIndexingCPInstruction extends IndexingCPInstruction {
 		else
 			throw new DMLRuntimeException("Invalid opcode (" + opcode +") encountered in FrameIndexingCPInstruction.");		
 	}
+
+	@Override
+	public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) {
+		return Pair.of(output.getName(), new LineageItem(getOpcode(),
+			LineageItemUtils.getLineage(ec, input1,input2,input3,rowLower,rowUpper,colLower,colUpper)));
+	}
 }
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index c082f3c..e87ee1e 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -446,7 +446,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
 		}
 		else if (opcode.equalsIgnoreCase("transformdecode") ||
 				opcode.equalsIgnoreCase("transformapply")) {
-			CPOperand target = getTargetOperand();
+			CPOperand target = new CPOperand(params.get("target"), ValueType.FP64, DataType.FRAME);
 			CPOperand meta = getLiteral("meta", ValueType.UNKNOWN, DataType.FRAME);
 			CPOperand spec = getStringLiteral("spec");
 			return Pair.of(output.getName(), new LineageItem(getOpcode(),
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
index 8e922d8..66972c4 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
@@ -75,9 +75,9 @@ public class LineageCacheConfig
 
 	private static boolean _allowSpill = false;
 	// Minimum reliable spilling estimate in milliseconds.
-	public static final double MIN_SPILL_TIME_ESTIMATE = 100;
+	public static final double MIN_SPILL_TIME_ESTIMATE = 10;
 	// Minimum reliable data size for spilling estimate in MB.
-	public static final double MIN_SPILL_DATA = 20;
+	public static final double MIN_SPILL_DATA = 2;
 	// Default I/O in MB per second for binary blocks
 	public static double FSREAD_DENSE = 200;
 	public static double FSREAD_SPARSE = 100;
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheEviction.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheEviction.java
index 31fccc7..553ca03 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheEviction.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheEviction.java
@@ -216,7 +216,7 @@ public class LineageCacheEviction
 				if (exectime > LineageCacheConfig.MIN_SPILL_TIME_ESTIMATE) {
 					System.out.print("LI " + e._key.getOpcode());
 					System.out.print(" exec time " + ((double) e._computeTime) / 1000000);
-					System.out.print(" estimate time " + getDiskSpillEstimate(e) * 1000);
+					System.out.print(" spill time " + getDiskSpillEstimate(e) * 1000);
 					System.out.print(" dim " + e.getMBValue().getNumRows() + " " + e.getMBValue().getNumColumns());
 					System.out.println(" size " + getDiskSizeEstimate(e));
 				}