You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/09/23 05:42:46 UTC
[2/4] incubator-systemml git commit: [SYSTEMML-557] Memory efficiency
frame block (array-based schema/meta)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
index eff0a38..e8d212f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -92,14 +89,13 @@ public class FrameGetSetTest extends AutomatedTestBase
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
//init data frame
- List<ValueType> lschema = Arrays.asList(schema);
- FrameBlock frame = new FrameBlock(lschema);
+ FrameBlock frame = new FrameBlock(schema);
//init data frame
if( itype == InitType.COLUMN )
{
- for( int j=0; j<lschema.size(); j++ ) {
- ValueType vt = lschema.get(j);
+ for( int j=0; j<schema.length; j++ ) {
+ ValueType vt = schema[j];
switch( vt ) {
case STRING:
String[] tmp1 = new String[rows];
@@ -131,20 +127,20 @@ public class FrameGetSetTest extends AutomatedTestBase
}
}
else if( itype == InitType.ROW_OBJ ) {
- Object[] row = new Object[lschema.size()];
+ Object[] row = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row[j] = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]));
+ for( int j=0; j<schema.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema[j],
+ row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
frame.appendRow(row);
}
}
else if( itype == InitType.ROW_STRING ) {
- String[] row = new String[lschema.size()];
+ String[] row = new String[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ ) {
- Object obj = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]);
- A[i][j] = UtilFunctions.objectToDouble(lschema.get(j), obj);
+ for( int j=0; j<schema.length; j++ ) {
+ Object obj = UtilFunctions.doubleToObject(schema[j], A[i][j]);
+ A[i][j] = UtilFunctions.objectToDouble(schema[j], obj);
row[j] = (obj!=null) ? obj.toString() : null;
}
frame.appendRow(row);
@@ -154,7 +150,7 @@ public class FrameGetSetTest extends AutomatedTestBase
//some updates via set
for( int i=7; i<13; i++ )
for( int j=0; j<=2; j++ ) {
- frame.set(i, j, UtilFunctions.doubleToObject(lschema.get(j), (double)i*j));
+ frame.set(i, j, UtilFunctions.doubleToObject(schema[j], (double)i*j));
A[i][j] = (double)i*j;
}
@@ -164,8 +160,8 @@ public class FrameGetSetTest extends AutomatedTestBase
//check correct values
for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j));
+ for( int j=0; j<schema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
if( tmp != A[i][j] )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
index 86dee49..c1f12e7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
@@ -180,20 +180,19 @@ public class FrameIndexingDistTest extends AutomatedTestBase
inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
//initialize the frame data.
- List<ValueType> lschema = Arrays.asList(schema);
-
+
double sparsity=sparsity1;//rand.nextDouble();
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
- writeInputFrameWithMTD("A", A, true, lschema, OutputInfo.BinaryBlockOutputInfo);
+ writeInputFrameWithMTD("A", A, true, schema, OutputInfo.BinaryBlockOutputInfo);
sparsity=sparsity3;//rand.nextDouble();
double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, 2345 /*System.currentTimeMillis()*/);
- List<ValueType> lschemaB = lschema.subList((int)colstart-1, (int)colend);
+ ValueType[] lschemaB = Arrays.copyOfRange(schema, (int)colstart-1, (int)colend);
writeInputFrameWithMTD("B", B, true, lschemaB, OutputInfo.BinaryBlockOutputInfo);
sparsity=sparsity2;//rand.nextDouble();
double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, 3267 /*System.currentTimeMillis()*/);
- List<ValueType> lschemaC = lschema.subList((int)colstart-1, (int)cols);
+ ValueType[] lschemaC = Arrays.copyOfRange(schema, (int)colstart-1, (int)cols);
writeInputFrameWithMTD("C", C, true, lschemaC, OutputInfo.BinaryBlockOutputInfo);
sparsity=sparsity4;//rand.nextDoublBe();
@@ -222,11 +221,10 @@ public class FrameIndexingDistTest extends AutomatedTestBase
inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
//initialize the frame data.
- List<ValueType> lschema = Arrays.asList(schema);
double sparsity = bSparse ? sparsity4 : sparsity2;
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
- writeInputFrameWithMTD("A", A, true, lschema, OutputInfo.BinaryBlockOutputInfo);
+ writeInputFrameWithMTD("A", A, true, schema, OutputInfo.BinaryBlockOutputInfo);
ValueType[] schemaB = new ValueType[(int) (colend-colstart+1)];
System.arraycopy(schema, (int)(colstart-1), schemaB, 0, (int)(colend-colstart+1));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
index 36bafea..858c9d6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
@@ -19,10 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -91,13 +87,12 @@ public class FrameIndexingTest extends AutomatedTestBase
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2412);
//init data frame 1
- List<ValueType> lschema1 = Arrays.asList(schema);
- FrameBlock frame1 = new FrameBlock(lschema1);
- Object[] row1 = new Object[lschema1.size()];
+ FrameBlock frame1 = new FrameBlock(schema);
+ Object[] row1 = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema1.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema1.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema1.get(j), A[i][j]));
+ for( int j=0; j<schema.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema[j],
+ row1[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
frame1.appendRow(row1);
}
@@ -119,15 +114,15 @@ public class FrameIndexingTest extends AutomatedTestBase
double[][] B = getRandomMatrix(ru-rl+1, cu-cl+1, -10, 10, 0.9, 7);
//init data frame 2
- List<ValueType> lschema2 = new ArrayList<ValueType>();
+ ValueType[] lschema2 = new ValueType[cu-cl+1];
for( int j=cl; j<=cu; j++ )
- lschema2.add(schema[j]);
+ lschema2[j-cl] = schema[j];
FrameBlock frame2 = new FrameBlock(lschema2);
- Object[] row2 = new Object[lschema2.size()];
+ Object[] row2 = new Object[lschema2.length];
for( int i=0; i<ru-rl+1; i++ ) {
- for( int j=0; j<lschema2.size(); j++ )
- B[i][j] = UtilFunctions.objectToDouble(lschema2.get(j),
- row2[j] = UtilFunctions.doubleToObject(lschema2.get(j), B[i][j]));
+ for( int j=0; j<lschema2.length; j++ )
+ B[i][j] = UtilFunctions.objectToDouble(lschema2[j],
+ row2[j] = UtilFunctions.doubleToObject(lschema2[j], B[i][j]));
frame2.appendRow(row2);
}
@@ -145,10 +140,10 @@ public class FrameIndexingTest extends AutomatedTestBase
Assert.fail("Wrong number of rows: "+frame3.getNumRows()+", expected: "+mbC.getNumRows());
//check correct values
- List<ValueType> lschema = frame3.getSchema();
+ ValueType[] lschema = frame3.getSchema();
for( int i=0; i<ru-rl+1; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame3.get(i, j));
+ for( int j=0; j<lschema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(lschema[j], frame3.get(i, j));
if( tmp != mbC.quickGetValue(i, j) )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+mbC.quickGetValue(i, j));
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
index d46c11f..692f1ec 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
@@ -20,8 +20,6 @@
package org.apache.sysml.test.integration.functions.frame;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
@@ -174,14 +172,12 @@ public class FrameReadWriteTest extends AutomatedTestBase
//Initialize the frame data.
//init data frame 1
- List<ValueType> lschema1 = Arrays.asList(schema1);
- FrameBlock frame1 = new FrameBlock(lschema1);
- initFrameData(frame1, A, lschema1);
+ FrameBlock frame1 = new FrameBlock(schema1);
+ initFrameData(frame1, A, schema1);
//init data frame 2
- List<ValueType> lschema2 = Arrays.asList(schema2);
- FrameBlock frame2 = new FrameBlock(lschema2);
- initFrameData(frame2, B, lschema2);
+ FrameBlock frame2 = new FrameBlock(schema2);
+ initFrameData(frame2, B, schema2);
//Write frame data to disk
CSVFileFormatProperties fprop = new CSVFileFormatProperties();
@@ -201,23 +197,23 @@ public class FrameReadWriteTest extends AutomatedTestBase
}
}
- void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema)
+ void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema)
{
- Object[] row1 = new Object[lschema.size()];
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ for( int j=0; j<lschema.length; j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
frame.appendRow(row1);
}
}
void verifyFrameData(FrameBlock frame1, FrameBlock frame2)
{
- List<ValueType> lschema = frame1.getSchema();
+ ValueType[] lschema = frame1.getSchema();
for ( int i=0; i<frame1.getNumRows(); i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- if( UtilFunctions.compareTo(lschema.get(j), frame1.get(i, j), frame2.get(i, j)) != 0)
+ for( int j=0; j<lschema.length; j++ ) {
+ if( UtilFunctions.compareTo(lschema[j], frame1.get(i, j), frame2.get(i, j)) != 0)
Assert.fail("Target value for cell ("+ i + "," + j + ") is " + frame1.get(i, j) +
", is not same as original value " + frame2.get(i, j));
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
index b179a42..dec63c4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-
-import java.util.Arrays;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.io.FrameReaderFactory;
import org.apache.sysml.runtime.io.FrameWriterFactory;
@@ -133,7 +130,7 @@ public class FrameScalarCastingTest extends AutomatedTestBase
}
else {
retval = FrameReaderFactory.createFrameReader(InputInfo.TextCellInputInfo)
- .readFrameFromHDFS(output("B"), Arrays.asList(vt), 1, 1)
+ .readFrameFromHDFS(output("B"), new ValueType[]{vt}, 1, 1)
.get(0, 0);
}
Assert.assertEquals("Wrong output: "+retval+" (expected: "+inval+")", inval, retval);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
index b37b3fb..1bf215a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
@@ -19,9 +19,7 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.List;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.ValueType;
@@ -120,20 +118,17 @@ public class FrameSchemaReadTest extends AutomatedTestBase
TestConfiguration config = getTestConfiguration(testname);
loadTestConfiguration(config);
- List<ValueType> lschema = Arrays.asList(schema);
-
-
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain","-args", input("A"), getSchemaString(lschema, wildcard),
+ programArgs = new String[]{"-explain","-args", input("A"), getSchemaString(schema, wildcard),
Integer.toString(rows), Integer.toString(schema.length), output("B") };
//data generation
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2373);
//prepare input/output infos
- FrameBlock frame1 = new FrameBlock(lschema);
- initFrameData(frame1, A, lschema);
+ FrameBlock frame1 = new FrameBlock(schema);
+ initFrameData(frame1, A, schema);
//write frame data to hdfs
FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo);
@@ -147,11 +142,11 @@ public class FrameSchemaReadTest extends AutomatedTestBase
FrameBlock frame2 = ((FrameReaderBinaryBlock)reader).readFirstBlock(output("B"));
//verify output schema
- List<ValueType> schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ?
- Collections.nCopies(schema.length, ValueType.STRING) : lschema;
- for( int i=0; i<schemaExpected.size(); i++ ) {
- Assert.assertEquals("Wrong result: "+frame2.getSchema().get(i)+".",
- schemaExpected.get(i), frame2.getSchema().get(i));
+ ValueType[] schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ?
+ Collections.nCopies(schema.length, ValueType.STRING).toArray(new ValueType[0]) : schema;
+ for( int i=0; i<schemaExpected.length; i++ ) {
+ Assert.assertEquals("Wrong result: "+frame2.getSchema()[i]+".",
+ schemaExpected[i], frame2.getSchema()[i]);
}
}
catch(Exception ex) {
@@ -166,12 +161,12 @@ public class FrameSchemaReadTest extends AutomatedTestBase
* @param data
* @param lschema
*/
- private void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema) {
- Object[] row1 = new Object[lschema.size()];
+ private void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema) {
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ for( int j=0; j<lschema.length; j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
frame.appendRow(row1);
}
}
@@ -182,7 +177,7 @@ public class FrameSchemaReadTest extends AutomatedTestBase
* @param wildcard
* @return
*/
- private String getSchemaString( List<ValueType> lschema, boolean wildcard ) {
+ private String getSchemaString( ValueType[] lschema, boolean wildcard ) {
if( wildcard )
return "*";
StringBuilder ret = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
index f36d076..954abc3 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
@@ -25,8 +25,6 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
-import java.util.Arrays;
-import java.util.List;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -87,15 +85,14 @@ public class FrameSerializationTest extends AutomatedTestBase
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
//init data frame
- List<ValueType> lschema = Arrays.asList(schema);
- FrameBlock frame = new FrameBlock(lschema);
+ FrameBlock frame = new FrameBlock(schema);
//init data frame
- Object[] row = new Object[lschema.size()];
+ Object[] row = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row[j] = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]));
+ for( int j=0; j<schema.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema[j],
+ row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
frame.appendRow(row);
}
@@ -130,8 +127,8 @@ public class FrameSerializationTest extends AutomatedTestBase
//check correct values
for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j));
+ for( int j=0; j<schema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
if( tmp != A[i][j] )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
index a26cfe8..ac3035f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.mlcontext;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
@@ -38,6 +35,7 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
@@ -212,7 +210,7 @@ public class DataFrameFrameConversionTest extends AutomatedTestBase
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
- List<ValueType> schema = Collections.nCopies(cols, vt);
+ ValueType[] schema = UtilFunctions.nCopies(cols, vt);
//setup spark context
sec = (SparkExecutionContext) ExecutionContextFactory.createContext();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
index 11f3f02..e76f044 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
@@ -201,7 +201,7 @@ public class FrameTest extends AutomatedTestBase
double sparsity=sparsity1;//rand.nextDouble();
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
- writeInputFrameWithMTD("A", A, true, lschema, oinfo);
+ writeInputFrameWithMTD("A", A, true, schema, oinfo);
sparsity=sparsity2;//rand.nextDouble();
double[][] B = getRandomMatrix((int)(bRows), (int)(bCols), min, max, sparsity, 2345 /*System.currentTimeMillis()*/);
@@ -211,7 +211,7 @@ public class FrameTest extends AutomatedTestBase
for (int i = 0; i < bCols; ++i)
schemaB[i] = schema[colstart-1+i];
List<ValueType> lschemaB = Arrays.asList(schemaB);
- writeInputFrameWithMTD("B", B, true, lschemaB, oinfo);
+ writeInputFrameWithMTD("B", B, true, schemaB, oinfo);
ValueType[] schemaC = new ValueType[colendC-colstartC+1];
for (int i = 0; i < cCols; ++i)
@@ -226,13 +226,13 @@ public class FrameTest extends AutomatedTestBase
{
//Create DataFrame for input A
SQLContext sqlContext = new SQLContext(sc);
- StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschema, false);
- JavaRDD<Row> rowRDDA = FrameRDDConverterUtils.csvToRowRDD(jsc, input("A"), DataExpression.DEFAULT_DELIM_DELIMITER, lschema);
+ StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schema, false);
+ JavaRDD<Row> rowRDDA = FrameRDDConverterUtils.csvToRowRDD(jsc, input("A"), DataExpression.DEFAULT_DELIM_DELIMITER, schema);
dfA = sqlContext.createDataFrame(rowRDDA, dfSchemaA);
//Create DataFrame for input B
- StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschemaB, false);
- JavaRDD<Row> rowRDDB = FrameRDDConverterUtils.csvToRowRDD(jsc, input("B"), DataExpression.DEFAULT_DELIM_DELIMITER, lschemaB);
+ StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaB, false);
+ JavaRDD<Row> rowRDDB = FrameRDDConverterUtils.csvToRowRDD(jsc, input("B"), DataExpression.DEFAULT_DELIM_DELIMITER, schemaB);
dfB = sqlContext.createDataFrame(rowRDDB, dfSchemaB);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
index 972e6ea..df434f1 100644
--- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
@@ -183,9 +183,11 @@ public class MLContextFrameTest extends AutomatedTestBase {
List<String> listB = new ArrayList<String>();
FrameMetadata fmA = null, fmB = null;
Script script = null;
- List<ValueType> lschemaA = Arrays.asList(ValueType.INT, ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN);
+ ValueType[] schemaA = { ValueType.INT, ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN };
+ List<ValueType> lschemaA = Arrays.asList(schemaA);
FrameSchema fschemaA = new FrameSchema(lschemaA);
- List<ValueType> lschemaB = Arrays.asList(ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN);
+ ValueType[] schemaB = { ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN };
+ List<ValueType> lschemaB = Arrays.asList(schemaB);
FrameSchema fschemaB = new FrameSchema(lschemaB);
if (inputType != IO_TYPE.FILE) {
@@ -232,9 +234,9 @@ public class MLContextFrameTest extends AutomatedTestBase {
// Create DataFrame
SQLContext sqlContext = new SQLContext(sc);
- StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschemaA, false);
+ StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaA, false);
DataFrame dataFrameA = sqlContext.createDataFrame(javaRddRowA, dfSchemaA);
- StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschemaB, false);
+ StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaB, false);
DataFrame dataFrameB = sqlContext.createDataFrame(javaRddRowB, dfSchemaB);
if (script_type == SCRIPT_TYPE.DML)
script = dml("A[2:3,2:4]=B;C=A[2:3,2:3]").in("A", dataFrameA, fmA).in("B", dataFrameB, fmB).out("A")
@@ -305,8 +307,8 @@ public class MLContextFrameTest extends AutomatedTestBase {
MLResults mlResults = ml.execute(script);
//Validate output schema
- List<ValueType> lschemaOutA = mlResults.getFrameObject("A").getSchema();
- List<ValueType> lschemaOutC = mlResults.getFrameObject("C").getSchema();
+ List<ValueType> lschemaOutA = Arrays.asList(mlResults.getFrameObject("A").getSchema());
+ List<ValueType> lschemaOutC = Arrays.asList(mlResults.getFrameObject("C").getSchema());
if(inputType != IO_TYPE.FILE) {
Assert.assertEquals(ValueType.INT, lschemaOutA.get(0));
Assert.assertEquals(ValueType.STRING, lschemaOutA.get(1));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index 85557bf..2c387d2 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -41,7 +41,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Locale;
import java.util.Random;
import java.util.StringTokenizer;
@@ -1637,13 +1636,13 @@ public class TestUtils
* @throws IOException
* @throws DMLRuntimeException
*/
- public static void writeTestFrame(String file, double[][] data, List<ValueType> schema, OutputInfo oi, boolean isR)
+ public static void writeTestFrame(String file, double[][] data, ValueType[] schema, OutputInfo oi, boolean isR)
throws DMLRuntimeException, IOException
{
FrameWriter writer = FrameWriterFactory.createFrameWriter(oi);
FrameBlock frame = new FrameBlock(schema);
initFrameData(frame, data, schema, data.length);
- writer.writeFrameToHDFS(frame, file, data.length, schema.size());
+ writer.writeFrameToHDFS(frame, file, data.length, schema.length);
}
/**
@@ -1658,7 +1657,7 @@ public class TestUtils
* @throws IOException
* @throws DMLRuntimeException
*/
- public static void writeTestFrame(String file, double[][] data, List<ValueType> schema, OutputInfo oi)
+ public static void writeTestFrame(String file, double[][] data, ValueType[] schema, OutputInfo oi)
throws DMLRuntimeException, IOException
{
writeTestFrame(file, data, schema, oi, false);
@@ -1670,13 +1669,13 @@ public class TestUtils
* @param data
* @param lschema
*/
- public static void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema, int rows) {
- Object[] row1 = new Object[lschema.size()];
+ public static void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema, int rows) {
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ ) {
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
- if(row1[j] != null && lschema.get(j) == ValueType.STRING)
+ for( int j=0; j<lschema.length; j++ ) {
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
+ if(row1[j] != null && lschema[j] == ValueType.STRING)
row1[j] = "Str" + row1[j];
}
frame.appendRow(row1);