You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2021/07/16 13:32:23 UTC
[systemds] branch master updated: [SYSTEMDS-3062] Fix runtime size
propagation spark sample
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new c7a5de0 [SYSTEMDS-3062] Fix runtime size propagation spark sample
c7a5de0 is described below
commit c7a5de08ff72f700533815ed453831355d98e77a
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Fri Jul 16 15:31:59 2021 +0200
[SYSTEMDS-3062] Fix runtime size propagation spark sample
This patch fixes a missing propagation of the number of rows on spark
sample operations in case the compiler could not infer them, which led
to exceptions if following operations need this info but are forced to
spark operations as well.
---
.../sysds/runtime/instructions/spark/RandSPInstruction.java | 3 ++-
.../sysds/test/functions/builtin/BuiltinMatrixProfileTest.java | 8 ++++----
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
index 91511d4..421b365 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
@@ -756,7 +756,8 @@ public class RandSPInstruction extends UnarySPInstruction {
JavaPairRDD<MatrixIndexes, MatrixBlock> mbRDD =
RDDConverterUtils.binaryCellToBinaryBlock(sec.getSparkContext(), miRDD, mcOut, true);
- sec.getDataCharacteristics(output.getName()).setNonZeros(lrows);
+ //step 5: output handling, incl meta data
+ sec.getDataCharacteristics(output.getName()).set(mcOut);
sec.setRDDHandleForVariable(output.getName(), mbRDD);
}
diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
index f0c227f..8664ab8 100644
--- a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
@@ -61,10 +61,10 @@ public class BuiltinMatrixProfileTest extends AutomatedTestBase
runMatrixProfileTest(4, 0.6, "TRUE", ExecType.CP);
}
-// @Test
-// public void testMatrixProfileSPARK() throws IOException {
-// runMatrixProfileTest(4, 1.0, "FALSE", ExecType.SPARK);
-// }
+ @Test
+ public void testMatrixProfileSPARK() throws IOException {
+ runMatrixProfileTest(4, 0.02, "FALSE", ExecType.SPARK);
+ }
private void runMatrixProfileTest(Integer window_size, Double sample_percent, String is_verbose, ExecType instType) throws IOException