You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/02/12 19:32:19 UTC
[5/5] incubator-systemml git commit: [HOTFIX] Fix csv frame readers
(robust size computation), tests
[HOTFIX] Fix csv frame readers (robust size computation), tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/10007504
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/10007504
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/10007504
Branch: refs/heads/master
Commit: 1000750464d470a7b56db71d896c6ca25e11e0bd
Parents: b3ba991
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Feb 12 20:15:00 2017 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Feb 12 20:15:00 2017 +0100
----------------------------------------------------------------------
.../sysml/runtime/io/IOUtilFunctions.java | 10 ++--
.../TransformCSVFrameEncodeReadTest.java | 48 ++++++++++++++++----
.../transform/TransformCSVFrameEncodeRead.dml | 3 +-
3 files changed, 48 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 3f0ea56..9d3b0ab 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -382,19 +382,21 @@ public class IOUtilFunctions
{
LongWritable key = new LongWritable();
Text value = new Text();
- int ncol = -1;
+ int ncol = -1;
for( int i=0; i<splits.length && ncol<=0; i++ ) {
RecordReader<LongWritable, Text> reader =
informat.getRecordReader(splits[i], job, Reporter.NULL);
try {
if( reader.next(key, value) ) {
+ boolean hasValue = true;
if( value.toString().startsWith(TfUtils.TXMTD_MVPREFIX) )
- reader.next(key, value);
+ hasValue = reader.next(key, value);
if( value.toString().startsWith(TfUtils.TXMTD_NDPREFIX) )
- reader.next(key, value);
+ hasValue = reader.next(key, value);
String row = value.toString().trim();
- if( !row.isEmpty() )
+ if( hasValue && !row.isEmpty() ) {
ncol = IOUtilFunctions.countTokensCSV(row, delim);
+ }
}
}
finally {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
index b28c2df..b35f2ac 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
@@ -51,33 +51,64 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
@Test
public void testFrameReadMetaSingleNodeCSV() {
- runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false);
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false, false);
}
@Test
public void testFrameReadMetaSparkCSV() {
- runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false);
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false, false);
}
@Test
public void testFrameReadMetaHybridCSV() {
- runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false);
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false, false);
}
@Test
public void testFrameParReadMetaSingleNodeCSV() {
- runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true);
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false, true);
}
@Test
public void testFrameParReadMetaSparkCSV() {
- runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true);
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false, true);
}
@Test
public void testFrameParReadMetaHybridCSV() {
- runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true);
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false, true);
}
+
+ @Test
+ public void testFrameReadSubMetaSingleNodeCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true, false);
+ }
+
+ @Test
+ public void testFrameReadSubMetaSparkCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true, false);
+ }
+
+ @Test
+ public void testFrameReadSubMetaHybridCSV() {
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true, false);
+ }
+
+ @Test
+ public void testFrameParReadSubMetaSingleNodeCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true, true);
+ }
+
+ @Test
+ public void testFrameParReadSubMetaSparkCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true, true);
+ }
+
+ @Test
+ public void testFrameParReadSubMetaHybridCSV() {
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true, true);
+ }
+
/**
*
@@ -85,7 +116,7 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
* @param ofmt
* @param dataset
*/
- private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean parRead )
+ private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean subset, boolean parRead )
{
//set runtime platform
RUNTIME_PLATFORM rtold = rtplatform;
@@ -104,9 +135,10 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
getAndLoadTestConfiguration(TEST_NAME1);
String HOME = SCRIPT_DIR + TEST_DIR;
+ int nrows = subset ? 4 : 13;
fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
programArgs = new String[]{"-explain", "-stats","-args",
- HOME + "input/" + DATASET, output("R") };
+ HOME + "input/" + DATASET, String.valueOf(nrows), output("R") };
OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true;
runTest(true, false, null, -1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml b/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
index 9da935f..d75cf8a 100644
--- a/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
+++ b/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
@@ -24,6 +24,7 @@ jspec = "{\"ids\": true, \"recode\": [1,2,3]}";
[X, M] = transformencode(target=F1, spec=jspec);
+M = M[1:$2,]
print(toString(M))
-write(M, $2, format="csv");
+write(M, $3, format="csv");