You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/07/14 04:31:42 UTC
[3/4] systemml git commit: [SYSTEMML-2442] Compiler/runtime
integration matrix market properties
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
index 911f35f..3ec542e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
@@ -32,6 +32,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.io.FileFormatPropertiesMM.MMField;
+import org.apache.sysml.runtime.io.FileFormatPropertiesMM.MMFormat;
+import org.apache.sysml.runtime.io.FileFormatPropertiesMM.MMSymmetry;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -48,24 +51,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
private final static int dim = 1200;
private final static double sparsity = 0.1;
- private enum MMFormat {
- COORDINATE,
- ARRAY,
- }
-
- private enum MMField {
- REAL,
- INTEGER,
- COMPLEX,
- PATTERN,
- }
-
- private enum MMSymmetry {
- GENERAL,
- SYMMETRIC,
- SKEW_SYMMETRIC, //- instead _
- }
-
@Override
public void setUp() {
TestUtils.clearAssertionInformation();
@@ -337,10 +322,8 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
MatrixBlock tmp = MatrixBlock.randOperations(
rows, cols, sparsity, -10, 10, "uniform", 7);
- String header = "%%MatrixMarket matrix "
- + fmt.name().toLowerCase() + " "
- + field.name().toLowerCase() + " "
- + symmetry.name().toLowerCase().replace("_", "-") + "\n";
+ String header = "%%MatrixMarket matrix " + fmt.toString() + " "
+ + field.toString() + " " + symmetry.toString() + "\n";
String meta = rows + " " + cols + ((fmt == MMFormat.COORDINATE) ?
" " + tmp.getNonZeros() : "") + "\n";
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
index e055b38..6dcb673 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
@@ -42,6 +42,7 @@ import org.apache.sysml.runtime.instructions.spark.functions.CopyFrameBlockPairF
import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongFrameToLongWritableFrameFunction;
import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongWritableFrameToLongFrameFunction;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
import org.apache.sysml.runtime.io.FrameWriter;
@@ -51,7 +52,6 @@ import org.apache.sysml.runtime.io.MatrixReaderFactory;
import org.apache.sysml.runtime.io.MatrixWriter;
import org.apache.sysml.runtime.io.MatrixWriterFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -441,7 +441,7 @@ public class FrameConverterTest extends AutomatedTestBase
InputInfo iinfo = InputInfo.BinaryBlockInputInfo;
JavaPairRDD<LongWritable, FrameBlock> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, LongWritable.class, FrameBlock.class);
JavaPairRDD<Long, FrameBlock> rddIn2 = rddIn.mapToPair(new CopyFrameBlockPairFunction(false));
- CSVFileFormatProperties fprop = new CSVFileFormatProperties();
+ FileFormatPropertiesCSV fprop = new FileFormatPropertiesCSV();
JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToCsv(rddIn2, mc, fprop, true);
rddOut.saveAsTextFile(fnameOut);
break;
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
index 77ebf10..7f7e089 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
@@ -24,11 +24,11 @@ import java.io.IOException;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
import org.apache.sysml.runtime.io.FrameWriter;
import org.apache.sysml.runtime.io.FrameWriterFactory;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.MapReduceTool;
@@ -179,7 +179,7 @@ public class FrameReadWriteTest extends AutomatedTestBase
initFrameData(frame2, B, schema2);
//Write frame data to disk
- CSVFileFormatProperties fprop = new CSVFileFormatProperties();
+ FileFormatPropertiesCSV fprop = new FileFormatPropertiesCSV();
fprop.setDelim(DELIMITER);
fprop.setHeader(HEADER);
@@ -218,7 +218,7 @@ public class FrameReadWriteTest extends AutomatedTestBase
}
}
- void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, CSVFileFormatProperties fprop)
+ void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, FileFormatPropertiesCSV fprop)
throws IOException
{
String fname1 = SCRIPT_DIR + TEST_DIR + "/frameData1";
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
index 4fd825b..0f34622 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
@@ -24,8 +24,8 @@ import org.junit.Test;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
@@ -114,7 +114,7 @@ public class CSVReadUnknownSizeTest extends AutomatedTestBase {
double[][] X = getRandomMatrix(rows, cols, -1, 1, 1.0d, 7);
MatrixBlock mb = DataConverter.convertToMatrixBlock(X);
MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, 1000, 1000);
- CSVFileFormatProperties fprop = new CSVFileFormatProperties();
+ FileFormatPropertiesCSV fprop = new FileFormatPropertiesCSV();
DataConverter.writeMatrixToHDFS(mb, input("X"), OutputInfo.CSVOutputInfo, mc, -1, fprop);
mc.set(-1, -1, -1, -1);
MapReduceTool.writeMetaDataFile(input("X.mtd"), ValueType.DOUBLE, mc, OutputInfo.CSVOutputInfo, fprop);
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java
index 056e619..04684a5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java
@@ -22,9 +22,9 @@ package org.apache.sysml.test.integration.functions.transform;
import org.junit.Test;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.util.DataConverter;
@@ -96,7 +96,7 @@ public class FrameCSVReadWriteTest extends AutomatedTestBase
//read input/output and compare
FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
- new CSVFileFormatProperties(false, ",", false));
+ new FileFormatPropertiesCSV(false, ",", false));
FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L);
FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo);
FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L);
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java
index 059bdb8..86a0cfd 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java
@@ -23,9 +23,9 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.util.DataConverter;
@@ -96,7 +96,7 @@ public class TransformCSVFrameEncodeDecodeTest extends AutomatedTestBase
//read input/output and compare
FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
- new CSVFileFormatProperties(false, ",", false));
+ new FileFormatPropertiesCSV(false, ",", false));
FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L);
FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo);
FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L);
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
index 535d88d..d63890f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
@@ -22,10 +22,10 @@ package org.apache.sysml.test.integration.functions.transform;
import org.junit.Test;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderTextCSV;
import org.apache.sysml.runtime.io.FrameReaderTextCSVParallel;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.test.integration.AutomatedTestBase;
@@ -142,8 +142,8 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
//read input/output and compare
FrameReader reader2 = parRead ?
- new FrameReaderTextCSVParallel( new CSVFileFormatProperties() ) :
- new FrameReaderTextCSV( new CSVFileFormatProperties() );
+ new FrameReaderTextCSVParallel( new FileFormatPropertiesCSV() ) :
+ new FrameReaderTextCSV( new FileFormatPropertiesCSV() );
FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L);
System.out.println(DataConverter.toString(fb2));
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
index 790f661..45605df 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
@@ -21,9 +21,9 @@ package org.apache.sysml.test.integration.functions.transform;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.util.DataConverter;
@@ -125,7 +125,7 @@ public class TransformFrameEncodeColmapTest extends AutomatedTestBase
//read input/output and compare
FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
- new CSVFileFormatProperties(true, ",", false));
+ new FileFormatPropertiesCSV(true, ",", false));
FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L);
FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo);
FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L);
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
index 3af3149..302ac2e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
@@ -21,9 +21,9 @@ package org.apache.sysml.test.integration.functions.transform;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.util.DataConverter;
@@ -172,7 +172,7 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase
//read input/output and compare
FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
- new CSVFileFormatProperties(true, ",", false));
+ new FileFormatPropertiesCSV(true, ",", false));
FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L);
FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo);
FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L);
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java
index f9dbe06..8d01d60 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java
@@ -23,9 +23,9 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FileFormatPropertiesCSV;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
-import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.util.DataConverter;
@@ -105,10 +105,10 @@ public class TransformFrameEncodeDecodeTokenTest extends AutomatedTestBase
//read input/output and compare
FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
- new CSVFileFormatProperties(false, " ", false));
+ new FileFormatPropertiesCSV(false, " ", false));
FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET1, -1L, -1L);
FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
- new CSVFileFormatProperties(false, " ", false));
+ new FileFormatPropertiesCSV(false, " ", false));
FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L);
String[][] R1 = DataConverter.convertToStringFrame(fb1);
String[][] R2 = DataConverter.convertToStringFrame(fb2);