You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/23 04:31:28 UTC
[1/2] systemml git commit: [MINOR] Fix test resource leaks to address
intermittent test failures
Repository: systemml
Updated Branches:
refs/heads/master 6bd2d8808 -> 723acfc09
[MINOR] Fix test resource leaks to address intermittent test failures
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/38162d23
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/38162d23
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/38162d23
Branch: refs/heads/master
Commit: 38162d232e22aa189003428d0afbcc04712b608d
Parents: 6bd2d88
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Nov 22 15:16:42 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed Nov 22 20:32:04 2017 -0800
----------------------------------------------------------------------
.../functions/jmlc/JMLCInputStreamReadTest.java | 2 ++
.../integration/mlcontext/MLContextTest.java | 20 +++++++++++---------
.../org/apache/sysml/test/utils/TestUtils.java | 8 +-------
3 files changed, 14 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/38162d23/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputStreamReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputStreamReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputStreamReadTest.java
index 05ee456..6411410 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputStreamReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputStreamReadTest.java
@@ -142,6 +142,7 @@ public class JMLCInputStreamReadTest extends AutomatedTestBase
//read matrix from input stream
FileInputStream fis = new FileInputStream(output("X"));
double[][] data2 = conn.convertToDoubleMatrix(fis, rows, cols, format);
+ fis.close();
//compare matrix result
TestUtils.compareMatrices(data, data2, rows, cols, 0);
@@ -164,6 +165,7 @@ public class JMLCInputStreamReadTest extends AutomatedTestBase
//read frame from input stream
FileInputStream fis = new FileInputStream(output("X"));
String[][] fdata2 = conn.convertToStringFrame(fis, rows, cols, format);
+ fis.close();
//compare frame result
TestUtils.compareFrames(fdata, fdata2, rows, cols);
http://git-wip-us.apache.org/repos/asf/systemml/blob/38162d23/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
index 9e4cfac..6dc3053 100644
--- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
@@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
+import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
@@ -123,23 +123,25 @@ public class MLContextTest extends MLContextTestBase {
}
@Test
- public void testCreateDMLScriptBasedOnInputStreamAndExecute() throws FileNotFoundException {
+ public void testCreateDMLScriptBasedOnInputStreamAndExecute() throws IOException {
System.out.println("MLContextTest - create DML script based on InputStream and execute");
setExpectedStdOut("hello world");
File file = new File(baseDirectory + File.separator + "hello-world.dml");
- InputStream is = new FileInputStream(file);
- Script script = dmlFromInputStream(is);
- ml.execute(script);
+ try( InputStream is = new FileInputStream(file) ) {
+ Script script = dmlFromInputStream(is);
+ ml.execute(script);
+ }
}
@Test
- public void testCreatePYDMLScriptBasedOnInputStreamAndExecute() throws FileNotFoundException {
+ public void testCreatePYDMLScriptBasedOnInputStreamAndExecute() throws IOException {
System.out.println("MLContextTest - create PYDML script based on InputStream and execute");
setExpectedStdOut("hello world");
File file = new File(baseDirectory + File.separator + "hello-world.pydml");
- InputStream is = new FileInputStream(file);
- Script script = pydmlFromInputStream(is);
- ml.execute(script);
+ try( InputStream is = new FileInputStream(file) ) {
+ Script script = pydmlFromInputStream(is);
+ ml.execute(script);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/systemml/blob/38162d23/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index 0ea4489..aaaade23 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -570,12 +570,8 @@ public class TestUtils
}
else if ( count > 1 ) {
File tmp = new File(csvFile+"_temp.csv");
- OutputStreamWriter out = null;
-
- try {
- out = new OutputStreamWriter(new FileOutputStream(tmp),
- "UTF-8");
+ try( OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(tmp), "UTF-8") ) {
// Directory listing may contain .crc files or may be in the
// wrong order. Sanitize the list of names.
ArrayList<String> partNames = new ArrayList<String>();
@@ -594,8 +590,6 @@ public class TestUtils
"UTF-8");
out.append(fileContents);
}
- } finally {
- IOUtilFunctions.closeSilently(out);
}
csvFile = tmp.getCanonicalPath();
[2/2] systemml git commit: [SYSTEMML-2025] Improved codegen optimizer
(search space layout)
Posted by mb...@apache.org.
[SYSTEMML-2025] Improved codegen optimizer (search space layout)
This patch makes a minor improvement to the codegen optimizer. So far we
linearized the search space by sorting cutsets by their scores and
appending all other interesting points. Now, we sort the remaining
interesting points decreasing according to their size, which can improve
the pruning efficiency by skipping larger sub spaces.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/723acfc0
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/723acfc0
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/723acfc0
Branch: refs/heads/master
Commit: 723acfc09da95f9a2d490b4c48452799383e54fc
Parents: 38162d2
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Nov 22 15:37:48 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed Nov 22 20:32:05 2017 -0800
----------------------------------------------------------------------
.../hops/codegen/opt/ReachabilityGraph.java | 24 ++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/723acfc0/src/main/java/org/apache/sysml/hops/codegen/opt/ReachabilityGraph.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/ReachabilityGraph.java b/src/main/java/org/apache/sysml/hops/codegen/opt/ReachabilityGraph.java
index 90079a3..c6ca0a9 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/opt/ReachabilityGraph.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/opt/ReachabilityGraph.java
@@ -69,7 +69,9 @@ public class ReachabilityGraph
//short-cut for partitions without cutsets
if( tmpCS.isEmpty() ) {
_cutSets = new CutSet[0];
- _searchSpace = part.getMatPointsExt();
+ //sort materialization points in decreasing order of their sizes
+ //which can improve the pruning efficiency by skipping larger sub-spaces.
+ _searchSpace = sortBySize(part.getMatPointsExt(), memo, false);
return;
}
@@ -120,7 +122,7 @@ public class ReachabilityGraph
.map(p -> p.getLeft()).toArray(CutSet[]::new);
//created sorted order of materialization points
- //(cut sets in predetermined order, all other points appended)
+ //(cut sets in predetermined order, other points sorted by size)
HashMap<InterestingPoint, Integer> probe = new HashMap<>();
ArrayList<InterestingPoint> lsearchSpace = new ArrayList<>();
for( CutSet cs : _cutSets ) {
@@ -128,7 +130,9 @@ public class ReachabilityGraph
for( InterestingPoint p : cs.cut )
probe.put(p, probe.size());
}
- for( InterestingPoint p : part.getMatPointsExt() )
+ //sort materialization points in decreasing order of their sizes
+ //which can improve the pruning efficiency by skipping larger sub-spaces.
+ for( InterestingPoint p : sortBySize(part.getMatPointsExt(), memo, false) )
if( !probe.containsKey(p) ) {
lsearchSpace.add(p);
probe.put(p, probe.size());
@@ -258,7 +262,19 @@ public class ReachabilityGraph
return cutSets;
}
-
+
+ private InterestingPoint[] sortBySize(InterestingPoint[] points, CPlanMemoTable memo, boolean asc) {
+ return Arrays.stream(points)
+ .sorted(Comparator.comparing(p -> (asc ? 1 : -1) *
+ getSize(memo.getHopRefs().get(p.getToHopID()))))
+ .toArray(InterestingPoint[]::new);
+ }
+
+ private static long getSize(Hop hop) {
+ return Math.max(hop.getDim1(),1)
+ * Math.max(hop.getDim2(),1);
+ }
+
public static class SubProblem {
public int offset;
public int[] freePos;