You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ac...@apache.org on 2016/04/05 06:13:54 UTC
incubator-systemml git commit: [SYSTEMML-573] Frame read-write test
Repository: incubator-systemml
Updated Branches:
refs/heads/master 6608d0ffc -> 5096defa7
[SYSTEMML-573] Frame read-write test
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5096defa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5096defa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5096defa
Branch: refs/heads/master
Commit: 5096defa718c8d50ac9a5ab8b89929d02132e7c3
Parents: 6608d0f
Author: Arvind Surve <ac...@yahoo.com>
Authored: Mon Apr 4 21:12:54 2016 -0700
Committer: Arvind Surve <ac...@yahoo.com>
Committed: Mon Apr 4 21:12:55 2016 -0700
----------------------------------------------------------------------
.../sysml/runtime/io/FrameReaderFactory.java | 37 ++++
.../functions/frame/FrameReadWriteTest.java | 184 +++++++++++++++++++
2 files changed, 221 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5096defa/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
index 535e7a7..58f2490 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -21,7 +21,9 @@ package org.apache.sysml.runtime.io;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
/**
*
@@ -91,4 +93,39 @@ public class FrameReaderFactory
return reader;
}
+
+
+ /**
+ *
+ * @param props
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static FrameReader createFrameReader( InputInfo iinfo, FileFormatProperties props )
+ throws DMLRuntimeException
+ {
+ //check valid read properties
+ if( props == null )
+ throw new DMLRuntimeException("Failed to create frame reader with empty properties.");
+
+ FrameReader reader = null;
+
+ if( iinfo == InputInfo.TextCellInputInfo ) {
+ reader = new FrameReaderTextCell();
+ }
+ else if( iinfo == InputInfo.CSVInputInfo ) {
+ if( props!=null && !(props instanceof CSVFileFormatProperties) )
+ throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
+ reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props);
+ }
+ else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
+ reader = new FrameReaderBinaryBlock();
+ }
+ else {
+ throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
+ + InputInfo.inputInfoToString(iinfo));
+ }
+
+ return reader;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5096defa/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
new file mode 100644
index 0000000..757a418
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.cp.AppendCPInstruction.AppendType;
+import org.apache.sysml.runtime.io.FrameReader;
+import org.apache.sysml.runtime.io.FrameReaderFactory;
+import org.apache.sysml.runtime.io.FrameWriter;
+import org.apache.sysml.runtime.io.FrameWriterFactory;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FrameReadWriteTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/frame/io/";
+
+ private final static int rows = 1593;
+ private final static ValueType[] schemaStrings = new ValueType[]{ValueType.STRING, ValueType.STRING, ValueType.STRING};
+ private final static ValueType[] schemaMixed = new ValueType[]{ValueType.STRING, ValueType.DOUBLE, ValueType.INT, ValueType.BOOLEAN};
+
+ private final static String DELIMITER = "::";
+ private final static boolean HEADER = true;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testFrameStringsStringsCBind() {
+ runFrameCopyTest(schemaStrings, schemaStrings, AppendType.CBIND);
+ }
+
+ @Test
+ public void testFrameStringsStringsRBind() { //note: ncol(A)=ncol(B)
+ runFrameCopyTest(schemaStrings, schemaStrings, AppendType.RBIND);
+ }
+
+ @Test
+ public void testFrameMixedStringsCBind() {
+ runFrameCopyTest(schemaMixed, schemaStrings, AppendType.CBIND);
+ }
+
+ @Test
+ public void testFrameStringsMixedCBind() {
+ runFrameCopyTest(schemaStrings, schemaMixed, AppendType.CBIND);
+ }
+
+ @Test
+ public void testFrameMixedMixedCBind() {
+ runFrameCopyTest(schemaMixed, schemaMixed, AppendType.CBIND);
+ }
+
+ @Test
+ public void testFrameMixedMixedRBind() { //note: ncol(A)=ncol(B)
+ runFrameCopyTest(schemaMixed, schemaMixed, AppendType.RBIND);
+ }
+
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ */
+ private void runFrameCopyTest( ValueType[] schema1, ValueType[] schema2, AppendType atype)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, schema1.length, -10, 10, 0.9, 2373);
+ double[][] B = getRandomMatrix(rows, schema2.length, -10, 10, 0.9, 129);
+
+ //Initialize the frame data.
+ //init data frame 1
+ List<ValueType> lschema1 = Arrays.asList(schema1);
+ FrameBlock frame1 = new FrameBlock(lschema1);
+ initFrameData(frame1, A, lschema1);
+
+ //init data frame 2
+ List<ValueType> lschema2 = Arrays.asList(schema2);
+ FrameBlock frame2 = new FrameBlock(lschema2);
+ initFrameData(frame2, B, lschema2);
+
+ //Write frame data to disk
+ CSVFileFormatProperties fprop = new CSVFileFormatProperties();
+ fprop.setDelim(DELIMITER);
+ fprop.setHeader(HEADER);
+
+ writeAndVerifyData(OutputInfo.TextCellOutputInfo, frame1, frame2, fprop);
+ writeAndVerifyData(OutputInfo.CSVOutputInfo, frame1, frame2, fprop);
+ writeAndVerifyData(OutputInfo.BinaryBlockOutputInfo, frame1, frame2, fprop);
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema)
+ {
+ Object[] row1 = new Object[lschema.size()];
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<lschema.size(); j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
+ row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ frame.appendRow(row1);
+ }
+ }
+
+ void verifyFrameData(FrameBlock frame1, FrameBlock frame2)
+ {
+ List<ValueType> lschema = frame1.getSchema();
+ for ( int i=0; i<frame1.getNumRows(); ++i )
+ for( int j=0; j<lschema.size(); j++ ) {
+ if( UtilFunctions.compareTo(lschema.get(j), frame1.get(i, j), frame2.get(i, j)) != 0)
+ Assert.fail("Target value for cell ("+ i + "," + j + ") is " + frame1.get(i, j) +
+ ", is not same as original value " + frame2.get(i, j));
+ }
+ }
+
+ /**
+ *
+ * @param frame1
+ * @param frame2
+ * @param fprop
+ * @return
+ * @throws DMLUnsupportedOperationException, DMLRuntimeException, IOException
+ */
+
+ void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, CSVFileFormatProperties fprop)
+ throws DMLUnsupportedOperationException, DMLRuntimeException, IOException
+ {
+ String fname1 = TEST_DIR + "/frameData1";
+ String fname2 = TEST_DIR + "/frameData2";
+
+ //Create reader/writer
+ FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo, fprop);
+ FrameReader reader = FrameReaderFactory.createFrameReader(OutputInfo.getMatchingInputInfo(oinfo), fprop);
+
+ //Write frame data to disk
+ writer.writeFrameToHDFS(frame1, fname1, frame1.getNumRows(), frame1.getNumColumns());
+ writer.writeFrameToHDFS(frame2, fname2, frame2.getNumRows(), frame2.getNumColumns());
+
+ //Read frame data from disk
+ FrameBlock frame1Read = reader.readFrameFromHDFS(fname1, frame1.getSchema(), frame1.getNumRows(), frame1.getNumColumns());
+ FrameBlock frame2Read = reader.readFrameFromHDFS(fname2, frame2.getSchema(), frame2.getNumRows(), frame2.getNumColumns());
+
+ // Verify that data read with original frames
+ verifyFrameData(frame1, frame1Read);
+ verifyFrameData(frame2, frame2Read);
+ }
+
+}