You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/09/23 05:42:45 UTC
[1/4] incubator-systemml git commit: [SYSTEMML-555] Error handling
common transformencode invocation issue
Repository: incubator-systemml
Updated Branches:
refs/heads/master 0b9d4e06f -> eb988781d
[SYSTEMML-555] Error handling common transformencode invocation issue
Transformencode is requires a multi-assignment statement (e.g.,
[X,M]=transformencode()) - this patch adds a more meaningful error
handling for the case where transformencode is called with a
single-assignment statement (e.g., X=transformencode(...)).
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/3c3d1a40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/3c3d1a40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/3c3d1a40
Branch: refs/heads/master
Commit: 3c3d1a40e2c98bd132febf2960b862e054073da3
Parents: 0b9d4e0
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Sep 22 12:58:11 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Sep 22 21:43:25 2016 -0700
----------------------------------------------------------------------
.../sysml/parser/ParameterizedBuiltinFunctionExpression.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3c3d1a40/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
index 90364d7..cce1d77 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -259,7 +259,13 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
break;
default: //always unconditional (because unsupported operation)
- raiseValidateError("Unsupported parameterized function "+ getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
+ //handle common issue of transformencode
+ if( getOpCode()==ParameterizedBuiltinFunctionOp.TRANSFORMENCODE )
+ raiseValidateError("Parameterized function "+ getOpCode() +" requires a multi-assignment statement "
+ + "for data and metadata.", false, LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
+ else
+ raiseValidateError("Unsupported parameterized function "+ getOpCode(),
+ false, LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
}
return;
}
[2/4] incubator-systemml git commit: [SYSTEMML-557] Memory efficiency
frame block (array-based schema/meta)
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
index eff0a38..e8d212f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -92,14 +89,13 @@ public class FrameGetSetTest extends AutomatedTestBase
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
//init data frame
- List<ValueType> lschema = Arrays.asList(schema);
- FrameBlock frame = new FrameBlock(lschema);
+ FrameBlock frame = new FrameBlock(schema);
//init data frame
if( itype == InitType.COLUMN )
{
- for( int j=0; j<lschema.size(); j++ ) {
- ValueType vt = lschema.get(j);
+ for( int j=0; j<schema.length; j++ ) {
+ ValueType vt = schema[j];
switch( vt ) {
case STRING:
String[] tmp1 = new String[rows];
@@ -131,20 +127,20 @@ public class FrameGetSetTest extends AutomatedTestBase
}
}
else if( itype == InitType.ROW_OBJ ) {
- Object[] row = new Object[lschema.size()];
+ Object[] row = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row[j] = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]));
+ for( int j=0; j<schema.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema[j],
+ row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
frame.appendRow(row);
}
}
else if( itype == InitType.ROW_STRING ) {
- String[] row = new String[lschema.size()];
+ String[] row = new String[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ ) {
- Object obj = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]);
- A[i][j] = UtilFunctions.objectToDouble(lschema.get(j), obj);
+ for( int j=0; j<schema.length; j++ ) {
+ Object obj = UtilFunctions.doubleToObject(schema[j], A[i][j]);
+ A[i][j] = UtilFunctions.objectToDouble(schema[j], obj);
row[j] = (obj!=null) ? obj.toString() : null;
}
frame.appendRow(row);
@@ -154,7 +150,7 @@ public class FrameGetSetTest extends AutomatedTestBase
//some updates via set
for( int i=7; i<13; i++ )
for( int j=0; j<=2; j++ ) {
- frame.set(i, j, UtilFunctions.doubleToObject(lschema.get(j), (double)i*j));
+ frame.set(i, j, UtilFunctions.doubleToObject(schema[j], (double)i*j));
A[i][j] = (double)i*j;
}
@@ -164,8 +160,8 @@ public class FrameGetSetTest extends AutomatedTestBase
//check correct values
for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j));
+ for( int j=0; j<schema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
if( tmp != A[i][j] )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
index 86dee49..c1f12e7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingDistTest.java
@@ -180,20 +180,19 @@ public class FrameIndexingDistTest extends AutomatedTestBase
inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
//initialize the frame data.
- List<ValueType> lschema = Arrays.asList(schema);
-
+
double sparsity=sparsity1;//rand.nextDouble();
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
- writeInputFrameWithMTD("A", A, true, lschema, OutputInfo.BinaryBlockOutputInfo);
+ writeInputFrameWithMTD("A", A, true, schema, OutputInfo.BinaryBlockOutputInfo);
sparsity=sparsity3;//rand.nextDouble();
double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, 2345 /*System.currentTimeMillis()*/);
- List<ValueType> lschemaB = lschema.subList((int)colstart-1, (int)colend);
+ ValueType[] lschemaB = Arrays.copyOfRange(schema, (int)colstart-1, (int)colend);
writeInputFrameWithMTD("B", B, true, lschemaB, OutputInfo.BinaryBlockOutputInfo);
sparsity=sparsity2;//rand.nextDouble();
double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, 3267 /*System.currentTimeMillis()*/);
- List<ValueType> lschemaC = lschema.subList((int)colstart-1, (int)cols);
+ ValueType[] lschemaC = Arrays.copyOfRange(schema, (int)colstart-1, (int)cols);
writeInputFrameWithMTD("C", C, true, lschemaC, OutputInfo.BinaryBlockOutputInfo);
sparsity=sparsity4;//rand.nextDoublBe();
@@ -222,11 +221,10 @@ public class FrameIndexingDistTest extends AutomatedTestBase
inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
//initialize the frame data.
- List<ValueType> lschema = Arrays.asList(schema);
double sparsity = bSparse ? sparsity4 : sparsity2;
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
- writeInputFrameWithMTD("A", A, true, lschema, OutputInfo.BinaryBlockOutputInfo);
+ writeInputFrameWithMTD("A", A, true, schema, OutputInfo.BinaryBlockOutputInfo);
ValueType[] schemaB = new ValueType[(int) (colend-colstart+1)];
System.arraycopy(schema, (int)(colstart-1), schemaB, 0, (int)(colend-colstart+1));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
index 36bafea..858c9d6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java
@@ -19,10 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -91,13 +87,12 @@ public class FrameIndexingTest extends AutomatedTestBase
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2412);
//init data frame 1
- List<ValueType> lschema1 = Arrays.asList(schema);
- FrameBlock frame1 = new FrameBlock(lschema1);
- Object[] row1 = new Object[lschema1.size()];
+ FrameBlock frame1 = new FrameBlock(schema);
+ Object[] row1 = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema1.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema1.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema1.get(j), A[i][j]));
+ for( int j=0; j<schema.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema[j],
+ row1[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
frame1.appendRow(row1);
}
@@ -119,15 +114,15 @@ public class FrameIndexingTest extends AutomatedTestBase
double[][] B = getRandomMatrix(ru-rl+1, cu-cl+1, -10, 10, 0.9, 7);
//init data frame 2
- List<ValueType> lschema2 = new ArrayList<ValueType>();
+ ValueType[] lschema2 = new ValueType[cu-cl+1];
for( int j=cl; j<=cu; j++ )
- lschema2.add(schema[j]);
+ lschema2[j-cl] = schema[j];
FrameBlock frame2 = new FrameBlock(lschema2);
- Object[] row2 = new Object[lschema2.size()];
+ Object[] row2 = new Object[lschema2.length];
for( int i=0; i<ru-rl+1; i++ ) {
- for( int j=0; j<lschema2.size(); j++ )
- B[i][j] = UtilFunctions.objectToDouble(lschema2.get(j),
- row2[j] = UtilFunctions.doubleToObject(lschema2.get(j), B[i][j]));
+ for( int j=0; j<lschema2.length; j++ )
+ B[i][j] = UtilFunctions.objectToDouble(lschema2[j],
+ row2[j] = UtilFunctions.doubleToObject(lschema2[j], B[i][j]));
frame2.appendRow(row2);
}
@@ -145,10 +140,10 @@ public class FrameIndexingTest extends AutomatedTestBase
Assert.fail("Wrong number of rows: "+frame3.getNumRows()+", expected: "+mbC.getNumRows());
//check correct values
- List<ValueType> lschema = frame3.getSchema();
+ ValueType[] lschema = frame3.getSchema();
for( int i=0; i<ru-rl+1; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame3.get(i, j));
+ for( int j=0; j<lschema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(lschema[j], frame3.get(i, j));
if( tmp != mbC.quickGetValue(i, j) )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+mbC.quickGetValue(i, j));
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
index d46c11f..692f1ec 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
@@ -20,8 +20,6 @@
package org.apache.sysml.test.integration.functions.frame;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
@@ -174,14 +172,12 @@ public class FrameReadWriteTest extends AutomatedTestBase
//Initialize the frame data.
//init data frame 1
- List<ValueType> lschema1 = Arrays.asList(schema1);
- FrameBlock frame1 = new FrameBlock(lschema1);
- initFrameData(frame1, A, lschema1);
+ FrameBlock frame1 = new FrameBlock(schema1);
+ initFrameData(frame1, A, schema1);
//init data frame 2
- List<ValueType> lschema2 = Arrays.asList(schema2);
- FrameBlock frame2 = new FrameBlock(lschema2);
- initFrameData(frame2, B, lschema2);
+ FrameBlock frame2 = new FrameBlock(schema2);
+ initFrameData(frame2, B, schema2);
//Write frame data to disk
CSVFileFormatProperties fprop = new CSVFileFormatProperties();
@@ -201,23 +197,23 @@ public class FrameReadWriteTest extends AutomatedTestBase
}
}
- void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema)
+ void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema)
{
- Object[] row1 = new Object[lschema.size()];
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ for( int j=0; j<lschema.length; j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
frame.appendRow(row1);
}
}
void verifyFrameData(FrameBlock frame1, FrameBlock frame2)
{
- List<ValueType> lschema = frame1.getSchema();
+ ValueType[] lschema = frame1.getSchema();
for ( int i=0; i<frame1.getNumRows(); i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- if( UtilFunctions.compareTo(lschema.get(j), frame1.get(i, j), frame2.get(i, j)) != 0)
+ for( int j=0; j<lschema.length; j++ ) {
+ if( UtilFunctions.compareTo(lschema[j], frame1.get(i, j), frame2.get(i, j)) != 0)
Assert.fail("Target value for cell ("+ i + "," + j + ") is " + frame1.get(i, j) +
", is not same as original value " + frame2.get(i, j));
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
index b179a42..dec63c4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-
-import java.util.Arrays;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.io.FrameReaderFactory;
import org.apache.sysml.runtime.io.FrameWriterFactory;
@@ -133,7 +130,7 @@ public class FrameScalarCastingTest extends AutomatedTestBase
}
else {
retval = FrameReaderFactory.createFrameReader(InputInfo.TextCellInputInfo)
- .readFrameFromHDFS(output("B"), Arrays.asList(vt), 1, 1)
+ .readFrameFromHDFS(output("B"), new ValueType[]{vt}, 1, 1)
.get(0, 0);
}
Assert.assertEquals("Wrong output: "+retval+" (expected: "+inval+")", inval, retval);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
index b37b3fb..1bf215a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSchemaReadTest.java
@@ -19,9 +19,7 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.List;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.ValueType;
@@ -120,20 +118,17 @@ public class FrameSchemaReadTest extends AutomatedTestBase
TestConfiguration config = getTestConfiguration(testname);
loadTestConfiguration(config);
- List<ValueType> lschema = Arrays.asList(schema);
-
-
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain","-args", input("A"), getSchemaString(lschema, wildcard),
+ programArgs = new String[]{"-explain","-args", input("A"), getSchemaString(schema, wildcard),
Integer.toString(rows), Integer.toString(schema.length), output("B") };
//data generation
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2373);
//prepare input/output infos
- FrameBlock frame1 = new FrameBlock(lschema);
- initFrameData(frame1, A, lschema);
+ FrameBlock frame1 = new FrameBlock(schema);
+ initFrameData(frame1, A, schema);
//write frame data to hdfs
FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo);
@@ -147,11 +142,11 @@ public class FrameSchemaReadTest extends AutomatedTestBase
FrameBlock frame2 = ((FrameReaderBinaryBlock)reader).readFirstBlock(output("B"));
//verify output schema
- List<ValueType> schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ?
- Collections.nCopies(schema.length, ValueType.STRING) : lschema;
- for( int i=0; i<schemaExpected.size(); i++ ) {
- Assert.assertEquals("Wrong result: "+frame2.getSchema().get(i)+".",
- schemaExpected.get(i), frame2.getSchema().get(i));
+ ValueType[] schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ?
+ Collections.nCopies(schema.length, ValueType.STRING).toArray(new ValueType[0]) : schema;
+ for( int i=0; i<schemaExpected.length; i++ ) {
+ Assert.assertEquals("Wrong result: "+frame2.getSchema()[i]+".",
+ schemaExpected[i], frame2.getSchema()[i]);
}
}
catch(Exception ex) {
@@ -166,12 +161,12 @@ public class FrameSchemaReadTest extends AutomatedTestBase
* @param data
* @param lschema
*/
- private void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema) {
- Object[] row1 = new Object[lschema.size()];
+ private void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema) {
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ for( int j=0; j<lschema.length; j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
frame.appendRow(row1);
}
}
@@ -182,7 +177,7 @@ public class FrameSchemaReadTest extends AutomatedTestBase
* @param wildcard
* @return
*/
- private String getSchemaString( List<ValueType> lschema, boolean wildcard ) {
+ private String getSchemaString( ValueType[] lschema, boolean wildcard ) {
if( wildcard )
return "*";
StringBuilder ret = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
index f36d076..954abc3 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java
@@ -25,8 +25,6 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
-import java.util.Arrays;
-import java.util.List;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -87,15 +85,14 @@ public class FrameSerializationTest extends AutomatedTestBase
double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
//init data frame
- List<ValueType> lschema = Arrays.asList(schema);
- FrameBlock frame = new FrameBlock(lschema);
+ FrameBlock frame = new FrameBlock(schema);
//init data frame
- Object[] row = new Object[lschema.size()];
+ Object[] row = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row[j] = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]));
+ for( int j=0; j<schema.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema[j],
+ row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
frame.appendRow(row);
}
@@ -130,8 +127,8 @@ public class FrameSerializationTest extends AutomatedTestBase
//check correct values
for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j));
+ for( int j=0; j<schema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
if( tmp != A[i][j] )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
index a26cfe8..ac3035f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameFrameConversionTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.mlcontext;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
@@ -38,6 +35,7 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
@@ -212,7 +210,7 @@ public class DataFrameFrameConversionTest extends AutomatedTestBase
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
- List<ValueType> schema = Collections.nCopies(cols, vt);
+ ValueType[] schema = UtilFunctions.nCopies(cols, vt);
//setup spark context
sec = (SparkExecutionContext) ExecutionContextFactory.createContext();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
index 11f3f02..e76f044 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java
@@ -201,7 +201,7 @@ public class FrameTest extends AutomatedTestBase
double sparsity=sparsity1;//rand.nextDouble();
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
- writeInputFrameWithMTD("A", A, true, lschema, oinfo);
+ writeInputFrameWithMTD("A", A, true, schema, oinfo);
sparsity=sparsity2;//rand.nextDouble();
double[][] B = getRandomMatrix((int)(bRows), (int)(bCols), min, max, sparsity, 2345 /*System.currentTimeMillis()*/);
@@ -211,7 +211,7 @@ public class FrameTest extends AutomatedTestBase
for (int i = 0; i < bCols; ++i)
schemaB[i] = schema[colstart-1+i];
List<ValueType> lschemaB = Arrays.asList(schemaB);
- writeInputFrameWithMTD("B", B, true, lschemaB, oinfo);
+ writeInputFrameWithMTD("B", B, true, schemaB, oinfo);
ValueType[] schemaC = new ValueType[colendC-colstartC+1];
for (int i = 0; i < cCols; ++i)
@@ -226,13 +226,13 @@ public class FrameTest extends AutomatedTestBase
{
//Create DataFrame for input A
SQLContext sqlContext = new SQLContext(sc);
- StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschema, false);
- JavaRDD<Row> rowRDDA = FrameRDDConverterUtils.csvToRowRDD(jsc, input("A"), DataExpression.DEFAULT_DELIM_DELIMITER, lschema);
+ StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schema, false);
+ JavaRDD<Row> rowRDDA = FrameRDDConverterUtils.csvToRowRDD(jsc, input("A"), DataExpression.DEFAULT_DELIM_DELIMITER, schema);
dfA = sqlContext.createDataFrame(rowRDDA, dfSchemaA);
//Create DataFrame for input B
- StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschemaB, false);
- JavaRDD<Row> rowRDDB = FrameRDDConverterUtils.csvToRowRDD(jsc, input("B"), DataExpression.DEFAULT_DELIM_DELIMITER, lschemaB);
+ StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaB, false);
+ JavaRDD<Row> rowRDDB = FrameRDDConverterUtils.csvToRowRDD(jsc, input("B"), DataExpression.DEFAULT_DELIM_DELIMITER, schemaB);
dfB = sqlContext.createDataFrame(rowRDDB, dfSchemaB);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
index 972e6ea..df434f1 100644
--- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextFrameTest.java
@@ -183,9 +183,11 @@ public class MLContextFrameTest extends AutomatedTestBase {
List<String> listB = new ArrayList<String>();
FrameMetadata fmA = null, fmB = null;
Script script = null;
- List<ValueType> lschemaA = Arrays.asList(ValueType.INT, ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN);
+ ValueType[] schemaA = { ValueType.INT, ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN };
+ List<ValueType> lschemaA = Arrays.asList(schemaA);
FrameSchema fschemaA = new FrameSchema(lschemaA);
- List<ValueType> lschemaB = Arrays.asList(ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN);
+ ValueType[] schemaB = { ValueType.STRING, ValueType.DOUBLE, ValueType.BOOLEAN };
+ List<ValueType> lschemaB = Arrays.asList(schemaB);
FrameSchema fschemaB = new FrameSchema(lschemaB);
if (inputType != IO_TYPE.FILE) {
@@ -232,9 +234,9 @@ public class MLContextFrameTest extends AutomatedTestBase {
// Create DataFrame
SQLContext sqlContext = new SQLContext(sc);
- StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschemaA, false);
+ StructType dfSchemaA = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaA, false);
DataFrame dataFrameA = sqlContext.createDataFrame(javaRddRowA, dfSchemaA);
- StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschemaB, false);
+ StructType dfSchemaB = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schemaB, false);
DataFrame dataFrameB = sqlContext.createDataFrame(javaRddRowB, dfSchemaB);
if (script_type == SCRIPT_TYPE.DML)
script = dml("A[2:3,2:4]=B;C=A[2:3,2:3]").in("A", dataFrameA, fmA).in("B", dataFrameB, fmB).out("A")
@@ -305,8 +307,8 @@ public class MLContextFrameTest extends AutomatedTestBase {
MLResults mlResults = ml.execute(script);
//Validate output schema
- List<ValueType> lschemaOutA = mlResults.getFrameObject("A").getSchema();
- List<ValueType> lschemaOutC = mlResults.getFrameObject("C").getSchema();
+ List<ValueType> lschemaOutA = Arrays.asList(mlResults.getFrameObject("A").getSchema());
+ List<ValueType> lschemaOutC = Arrays.asList(mlResults.getFrameObject("C").getSchema());
if(inputType != IO_TYPE.FILE) {
Assert.assertEquals(ValueType.INT, lschemaOutA.get(0));
Assert.assertEquals(ValueType.STRING, lschemaOutA.get(1));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index 85557bf..2c387d2 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -41,7 +41,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Locale;
import java.util.Random;
import java.util.StringTokenizer;
@@ -1637,13 +1636,13 @@ public class TestUtils
* @throws IOException
* @throws DMLRuntimeException
*/
- public static void writeTestFrame(String file, double[][] data, List<ValueType> schema, OutputInfo oi, boolean isR)
+ public static void writeTestFrame(String file, double[][] data, ValueType[] schema, OutputInfo oi, boolean isR)
throws DMLRuntimeException, IOException
{
FrameWriter writer = FrameWriterFactory.createFrameWriter(oi);
FrameBlock frame = new FrameBlock(schema);
initFrameData(frame, data, schema, data.length);
- writer.writeFrameToHDFS(frame, file, data.length, schema.size());
+ writer.writeFrameToHDFS(frame, file, data.length, schema.length);
}
/**
@@ -1658,7 +1657,7 @@ public class TestUtils
* @throws IOException
* @throws DMLRuntimeException
*/
- public static void writeTestFrame(String file, double[][] data, List<ValueType> schema, OutputInfo oi)
+ public static void writeTestFrame(String file, double[][] data, ValueType[] schema, OutputInfo oi)
throws DMLRuntimeException, IOException
{
writeTestFrame(file, data, schema, oi, false);
@@ -1670,13 +1669,13 @@ public class TestUtils
* @param data
* @param lschema
*/
- public static void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema, int rows) {
- Object[] row1 = new Object[lschema.size()];
+ public static void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema, int rows) {
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ ) {
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
- if(row1[j] != null && lschema.get(j) == ValueType.STRING)
+ for( int j=0; j<lschema.length; j++ ) {
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
+ if(row1[j] != null && lschema[j] == ValueType.STRING)
row1[j] = "Str" + row1[j];
}
frame.appendRow(row1);
[3/4] incubator-systemml git commit: [SYSTEMML-557] Memory efficiency
frame block (array-based schema/meta)
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 005b254..67674fe 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -29,12 +29,11 @@ import java.io.Serializable;
import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.io.Writable;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.Expression.ValueType;
@@ -61,29 +60,32 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
private int _numRows = -1;
/** The schema of the data frame as an ordered list of value types */
- private List<ValueType> _schema = null;
+ private ValueType[] _schema = null;
/** The column names of the data frame as an ordered list of strings */
- private List<String> _colnames = null;
+ private String[] _colnames = null;
- private List<ColumnMetadata> _colmeta = null;
+ private ColumnMetadata[] _colmeta = null;
/** The data frame data as an ordered list of columns */
- private List<Array> _coldata = null;
+ private Array[] _coldata = null;
/** Cache for recode maps from frame meta data, indexed by column 0-based */
private Map<Integer, SoftReference<HashMap<String,Long>>> _rcdMapCache = null;
public FrameBlock() {
_numRows = 0;
- _schema = new ArrayList<ValueType>();
- _colnames = new ArrayList<String>();
- _colmeta = new ArrayList<ColumnMetadata>();
- _coldata = new ArrayList<Array>();
if( REUSE_RECODE_MAPS )
_rcdMapCache = new HashMap<Integer, SoftReference<HashMap<String,Long>>>();
}
+ /**
+ * Copy constructor for frame blocks, which uses a shallow copy for
+ * the schema (column types and names) but a deep copy for meta data
+ * and actual column data.
+ *
+ * @param that
+ */
public FrameBlock(FrameBlock that) {
this(that.getSchema(), that.getColumnNames());
copy(that);
@@ -92,32 +94,32 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
public FrameBlock(int ncols, ValueType vt) {
this();
- _schema.addAll(Collections.nCopies(ncols, vt));
+ _schema = UtilFunctions.nCopies(ncols, vt);
_colnames = createColNames(ncols);
+ _colmeta = new ColumnMetadata[ncols];
for( int j=0; j<ncols; j++ )
- _colmeta.add(new ColumnMetadata(0));
+ _colmeta[j] = new ColumnMetadata(0);
}
- public FrameBlock(List<ValueType> schema) {
+ public FrameBlock(ValueType[] schema) {
this(schema, new String[0][]);
}
- public FrameBlock(List<ValueType> schema, List<String> names) {
+ public FrameBlock(ValueType[] schema, String[] names) {
this(schema, names, new String[0][]);
}
- public FrameBlock(List<ValueType> schema, String[][] data) {
- this(schema, createColNames(schema.size()), data);
+ public FrameBlock(ValueType[] schema, String[][] data) {
+ this(schema, createColNames(schema.length), data);
}
- public FrameBlock(List<ValueType> schema, List<String> names, String[][] data) {
+ public FrameBlock(ValueType[] schema, String[] names, String[][] data) {
_numRows = 0; //maintained on append
- _schema = new ArrayList<ValueType>(schema);
- _colnames = new ArrayList<String>(names);
- _colmeta = new ArrayList<ColumnMetadata>();
- for( int j=0; j<_schema.size(); j++ )
- _colmeta.add(new ColumnMetadata(0));
- _coldata = new ArrayList<Array>();
+ _schema = schema;
+ _colnames = names;
+ _colmeta = new ColumnMetadata[_schema.length];
+ for( int j=0; j<_schema.length; j++ )
+ _colmeta[j] = new ColumnMetadata(0);
for( int i=0; i<data.length; i++ )
appendRow(data[i]);
if( REUSE_RECODE_MAPS )
@@ -148,7 +150,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public int getNumColumns() {
- return _schema.size();
+ return _schema.length;
}
/**
@@ -156,7 +158,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
* @return
*/
- public List<ValueType> getSchema() {
+ public ValueType[] getSchema() {
return _schema;
}
@@ -165,9 +167,9 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
* @return
*/
- public void setSchema(List<ValueType> schema) {
+ public void setSchema(ValueType[] schema) {
_schema = schema;
- _colnames = createColNames(schema.size());
+ _colnames = createColNames(schema.length);
}
/**
@@ -175,7 +177,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
* @return
*/
- public List<String> getColumnNames() {
+ public String[] getColumnNames() {
return _colnames;
}
@@ -183,7 +185,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
* @param colnames
*/
- public void setColumnNames(List<String> colnames) {
+ public void setColumnNames(String[] colnames) {
_colnames = colnames;
}
@@ -191,7 +193,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
* @return
*/
- public List<ColumnMetadata> getColumnMetadata() {
+ public ColumnMetadata[] getColumnMetadata() {
return _colmeta;
}
@@ -201,7 +203,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public ColumnMetadata getColumnMetadata(int c) {
- return _colmeta.get(c);
+ return _colmeta[c];
}
/**
@@ -221,16 +223,16 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public boolean isColumnMetadataDefault(int c) {
- return _colmeta.get(c).getMvValue() == null
- && _colmeta.get(c).getNumDistinct() == 0;
+ return _colmeta[c].getMvValue() == null
+ && _colmeta[c].getNumDistinct() == 0;
}
/**
*
* @param colmeta
*/
- public void setColumnMetadata(List<ColumnMetadata> colmeta) {
- _colmeta = new ArrayList<FrameBlock.ColumnMetadata>(colmeta);
+ public void setColumnMetadata(ColumnMetadata[] colmeta) {
+ System.arraycopy(colmeta, 0, _colmeta, 0, _colmeta.length);
}
/**
@@ -239,7 +241,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param colmeta
*/
public void setColumnMetadata(int c, ColumnMetadata colmeta) {
- _colmeta.set(c, colmeta);
+ _colmeta[c] = colmeta;
}
/**
@@ -251,7 +253,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
public Map<String,Integer> getColumnNameIDMap() {
Map<String, Integer> ret = new HashMap<String, Integer>();
for( int j=0; j<getNumColumns(); j++ )
- ret.put(_colnames.get(j), j+1);
+ ret.put(_colnames[j], j+1);
return ret;
}
@@ -261,21 +263,24 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*/
public void ensureAllocatedColumns(int numRows) {
//early abort if already allocated
- if( _schema.size() == _coldata.size() )
+ if( _coldata != null && _schema.length == _coldata.length )
return;
- //allocate column meta data
- for( int j=0; j<_schema.size(); j++ )
- _colmeta.add(new ColumnMetadata(0));
+ //allocate column meta data if necessary
+ if( _colmeta == null || _schema.length != _colmeta.length ) {
+ _colmeta = new ColumnMetadata[_schema.length];
+ for( int j=0; j<_schema.length; j++ )
+ _colmeta[j] = new ColumnMetadata(0);
+ }
//allocate columns if necessary
- for( int j=0; j<_schema.size(); j++ ) {
- if( j >= _coldata.size() )
- switch( _schema.get(j) ) {
- case STRING: _coldata.add(new StringArray(new String[numRows])); break;
- case BOOLEAN: _coldata.add(new BooleanArray(new boolean[numRows])); break;
- case INT: _coldata.add(new LongArray(new long[numRows])); break;
- case DOUBLE: _coldata.add(new DoubleArray(new double[numRows])); break;
- default: throw new RuntimeException("Unsupported value type: "+_schema.get(j));
- }
+ _coldata = new Array[_schema.length];
+ for( int j=0; j<_schema.length; j++ ) {
+ switch( _schema[j] ) {
+ case STRING: _coldata[j] = new StringArray(new String[numRows]); break;
+ case BOOLEAN: _coldata[j] = new BooleanArray(new boolean[numRows]); break;
+ case INT: _coldata[j] = new LongArray(new long[numRows]); break;
+ case DOUBLE: _coldata[j] = new DoubleArray(new double[numRows]); break;
+ default: throw new RuntimeException("Unsupported value type: "+_schema[j]);
+ }
}
_numRows = numRows;
}
@@ -286,7 +291,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param newlen
*/
public void ensureColumnCompatibility(int newlen) {
- if( _coldata.size() > 0 && _numRows != newlen )
+ if( _coldata!=null && _coldata.length > 0 && _numRows != newlen )
throw new RuntimeException("Mismatch in number of rows: "+newlen+" (expected: "+_numRows+")");
}
@@ -295,10 +300,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param size
* @return
*/
- public static List<String> createColNames(int size) {
- ArrayList<String> ret = new ArrayList<String>(size);
+ public static String[] createColNames(int size) {
+ String[] ret = new String[size];
for( int i=1; i<=size; i++ )
- ret.add(createColName(i));
+ ret[i-1] = createColName(i);
return ret;
}
@@ -328,7 +333,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public boolean isColNameDefault(int i) {
- return _colnames.get(i).equals("C"+i);
+ return _colnames[i].equals("C"+(i+1));
}
/**
@@ -339,7 +344,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
int card = 0;
for( int i=0; i<getNumRows(); i++ )
card += (get(i, j) != null) ? 1 : 0;
- _colmeta.get(j).setNumDistinct(card);
+ _colmeta[j].setNumDistinct(card);
}
}
@@ -354,7 +359,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public Object get(int r, int c) {
- return _coldata.get(c).get(r);
+ return _coldata[c].get(r);
}
/**
@@ -366,7 +371,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param val
*/
public void set(int r, int c, Object val) {
- _coldata.get(c).set(r, UtilFunctions.objectToObject(_schema.get(c), val));
+ _coldata[c].set(r, UtilFunctions.objectToObject(_schema[c], val));
}
/**
@@ -376,17 +381,17 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*/
public void reset(int nrow, boolean clearMeta) {
if( clearMeta ) {
- getSchema().clear();
- getColumnNames().clear();
+ _schema = null;
+ _colnames = null;
if( _colmeta != null ) {
- for( int i=0; i<_colmeta.size(); i++ )
+ for( int i=0; i<_colmeta.length; i++ )
if( !isColumnMetadataDefault(i) )
- _colmeta.set(i, new ColumnMetadata(0));
+ _colmeta[i] = new ColumnMetadata(0);
}
}
if(_coldata != null) {
- for( int i=0; i < _coldata.size(); i++ )
- _coldata.get(i)._size = nrow;
+ for( int i=0; i < _coldata.length; i++ )
+ _coldata[i]._size = nrow;
}
}
@@ -407,7 +412,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
public void appendRow(Object[] row) {
ensureAllocatedColumns(0);
for( int j=0; j<row.length; j++ )
- _coldata.get(j).append(row[j]);
+ _coldata[j].append(row[j]);
_numRows++;
}
@@ -420,7 +425,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
public void appendRow(String[] row) {
ensureAllocatedColumns(0);
for( int j=0; j<row.length; j++ )
- _coldata.get(j).append(row[j]);
+ _coldata[j].append(row[j]);
_numRows++;
}
@@ -433,9 +438,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*/
public void appendColumn(String[] col) {
ensureColumnCompatibility(col.length);
- _schema.add(ValueType.STRING);
- _colnames.add(createColName(_schema.size()));
- _coldata.add(new StringArray(col));
+ _schema = ArrayUtils.add(_schema, ValueType.STRING);
+ _colnames = ArrayUtils.add(_colnames, createColName(_schema.length));
+ _coldata = (_coldata==null) ? new Array[]{new StringArray(col)} :
+ ArrayUtils.add(_coldata, new StringArray(col));
_numRows = col.length;
}
@@ -448,9 +454,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*/
public void appendColumn(boolean[] col) {
ensureColumnCompatibility(col.length);
- _schema.add(ValueType.BOOLEAN);
- _colnames.add(createColName(_schema.size()));
- _coldata.add(new BooleanArray(col));
+ _schema = ArrayUtils.add(_schema, ValueType.BOOLEAN);
+ _colnames = ArrayUtils.add(_colnames, createColName(_schema.length));
+ _coldata = (_coldata==null) ? new Array[]{new BooleanArray(col)} :
+ ArrayUtils.add(_coldata, new BooleanArray(col));
_numRows = col.length;
}
@@ -463,9 +470,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*/
public void appendColumn(long[] col) {
ensureColumnCompatibility(col.length);
- _schema.add(ValueType.INT);
- _colnames.add(createColName(_schema.size()));
- _coldata.add(new LongArray(col));
+ _schema = ArrayUtils.add(_schema, ValueType.INT);
+ _colnames = ArrayUtils.add(_colnames, createColName(_schema.length));
+ _coldata = (_coldata==null) ? new Array[]{new LongArray(col)} :
+ ArrayUtils.add(_coldata, new LongArray(col));
_numRows = col.length;
}
@@ -478,9 +486,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*/
public void appendColumn(double[] col) {
ensureColumnCompatibility(col.length);
- _schema.add(ValueType.DOUBLE);
- _colnames.add(createColName(_schema.size()));
- _coldata.add(new DoubleArray(col));
+ _schema = ArrayUtils.add(_schema, ValueType.DOUBLE);
+ _colnames = ArrayUtils.add(_colnames, createColName(_schema.length));
+ _coldata = (_coldata==null) ? new Array[]{new DoubleArray(col)} :
+ ArrayUtils.add(_coldata, new DoubleArray(col));
_numRows = col.length;
}
@@ -490,11 +499,11 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public Object getColumn(int c) {
- switch(_schema.get(c)) {
- case STRING: return ((StringArray)_coldata.get(c))._data;
- case BOOLEAN: return ((BooleanArray)_coldata.get(c))._data;
- case INT: return ((LongArray)_coldata.get(c))._data;
- case DOUBLE: return ((DoubleArray)_coldata.get(c))._data;
+ switch(_schema[c]) {
+ case STRING: return ((StringArray)_coldata[c])._data;
+ case BOOLEAN: return ((BooleanArray)_coldata[c])._data;
+ case INT: return ((LongArray)_coldata[c])._data;
+ case DOUBLE: return ((DoubleArray)_coldata[c])._data;
default: return null;
}
}
@@ -556,14 +565,14 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
out.writeBoolean(isDefaultMeta);
//write columns (value type, data)
for( int j=0; j<getNumColumns(); j++ ) {
- out.writeByte(_schema.get(j).ordinal());
+ out.writeByte(_schema[j].ordinal());
if( !isDefaultMeta ) {
- out.writeUTF(_colnames.get(j));
- out.writeLong(_colmeta.get(j).getNumDistinct());
- out.writeUTF( (_colmeta.get(j).getMvValue()!=null) ?
- _colmeta.get(j).getMvValue() : "" );
+ out.writeUTF(_colnames[j]);
+ out.writeLong(_colmeta[j].getNumDistinct());
+ out.writeUTF( (_colmeta[j].getMvValue()!=null) ?
+ _colmeta[j].getMvValue() : "" );
}
- _coldata.get(j).write(out);
+ _coldata[j].write(out);
}
}
@@ -573,10 +582,16 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
_numRows = in.readInt();
int numCols = in.readInt();
boolean isDefaultMeta = in.readBoolean();
+ //allocate schema/meta data arrays
+ _schema = (_schema!=null && _schema.length==numCols) ?
+ _schema : new ValueType[numCols];
+ _colnames = (_colnames != null && _colnames.length==numCols) ?
+ _colnames : new String[numCols];
+ _colmeta = (_colmeta != null && _colmeta.length==numCols) ?
+ _colmeta : new ColumnMetadata[numCols];
+ _coldata = (_coldata!=null && _coldata.length==numCols) ?
+ _coldata : new Array[numCols];
//read columns (value type, meta, data)
- _schema.clear();
- _colmeta.clear();
- _coldata.clear();
for( int j=0; j<numCols; j++ ) {
ValueType vt = ValueType.values()[in.readByte()];
String name = isDefaultMeta ? createColName(j) : in.readUTF();
@@ -591,11 +606,11 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
default: throw new IOException("Unsupported value type: "+vt);
}
arr.readFields(in);
- _schema.add(vt);
- _colnames.add(name);
- _colmeta.add(new ColumnMetadata(ndistinct,
- (mvvalue==null || mvvalue.isEmpty()) ? null : mvvalue));
- _coldata.add(arr);
+ _schema[j] = vt;
+ _colnames[j] = name;
+ _colmeta[j] = new ColumnMetadata(ndistinct,
+ (mvvalue==null || mvvalue.isEmpty()) ? null : mvvalue);
+ _coldata[j] = arr;
}
}
@@ -678,16 +693,17 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
if( ret == null )
ret = new FrameBlock();
ret._numRows = _numRows;
- ret._schema = new ArrayList<ValueType>(_schema);
- ret._colnames = new ArrayList<String>(_colnames);
- ret._colmeta = new ArrayList<ColumnMetadata>(_colmeta);
+ ret._schema = _schema.clone();
+ ret._colnames = _colnames.clone();
+ ret._colmeta = _colmeta.clone();
+ ret._coldata = new Array[getNumColumns()];
//copy data to output and partial overwrite w/ rhs
for( int j=0; j<getNumColumns(); j++ ) {
- Array tmp = _coldata.get(j).clone();
+ Array tmp = _coldata[j].clone();
if( j>=cl && j<=cu )
- tmp.set(rl, ru, rhsFrame._coldata.get(j-cl));
- ret._coldata.add(tmp);
+ tmp.set(rl, ru, rhsFrame._coldata[j-cl]);
+ ret._coldata[j] = tmp;
}
return ret;
@@ -737,20 +753,27 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
ret.reset(ru-rl+1, true);
//copy output schema and colnames
+ int numCols = cu-cl+1;
+ ret._schema = new ValueType[numCols];
+ ret._colnames = new String[numCols];
+ ret._colmeta = new ColumnMetadata[numCols];
+
for( int j=cl; j<=cu; j++ ) {
- ret._schema.add(_schema.get(j));
- ret._colnames.add(_colnames.get(j));
- ret._colmeta.add(_colmeta.get(j));
+ ret._schema[j-cl] = _schema[j];
+ ret._colnames[j-cl] = _colnames[j];
+ ret._colmeta[j-cl] = _colmeta[j];
}
ret._numRows = ru-rl+1;
//copy output data
- if(ret._coldata.size() == 0)
+ if(ret._coldata == null ) {
+ ret._coldata = new Array[numCols];
for( int j=cl; j<=cu; j++ )
- ret._coldata.add(_coldata.get(j).slice(rl,ru));
+ ret._coldata[j-cl] = _coldata[j].slice(rl,ru);
+ }
else
for( int j=cl; j<=cu; j++ )
- ret._coldata.get(j-cl).set(0, ru-rl, _coldata.get(j), rl);
+ ret._coldata[j-cl].set(0, ru-rl, _coldata[j], rl);
return ret;
}
@@ -818,18 +841,14 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
ret._numRows = _numRows;
//concatenate schemas (w/ deep copy to prevent side effects)
- ret._schema = new ArrayList<ValueType>(_schema);
- ret._schema.addAll(that._schema);
- ret._colnames = new ArrayList<String>(_colnames);
- ret._colnames.addAll(that._colnames);
- ret._colmeta = new ArrayList<ColumnMetadata>(_colmeta);
- ret._colmeta.addAll(that._colmeta);
+ ret._schema = ArrayUtils.addAll(_schema, that._schema);
+ ret._colnames = ArrayUtils.addAll(_colnames, that._colnames);
+ ret._colmeta = ArrayUtils.addAll(_colmeta, that._colmeta);
//concatenate column data (w/ deep copy to prevent side effects)
- for( Array tmp : _coldata )
- ret._coldata.add(tmp.clone());
- for( Array tmp : that._coldata )
- ret._coldata.add(tmp.clone());
+ ret._coldata = ArrayUtils.addAll(_coldata, that._coldata);
+ for( int i=0; i<ret._coldata.length; i++ )
+ ret._coldata[i] = ret._coldata[i].clone();
}
else //ROW APPEND
{
@@ -843,12 +862,13 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
if( ret == null )
ret = new FrameBlock();
ret._numRows = _numRows;
- ret._schema = new ArrayList<ValueType>(_schema);
- ret._colnames = new ArrayList<String>(_colnames);
+ ret._schema = _schema.clone();
+ ret._colnames = _colnames.clone();
//concatenate data (deep copy first, append second)
- for( Array tmp : _coldata )
- ret._coldata.add(tmp.clone());
+ ret._coldata = new Array[_coldata.length];
+ for( int j=0; j<_coldata.length; j++ )
+ ret._coldata[j] = _coldata[j].clone();
Iterator<Object[]> iter = that.getObjectRowIterator();
while( iter.hasNext() )
ret.appendRow(iter.next());
@@ -882,13 +902,13 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
//copy values
for( int j=cl; j<=cu; j++ ) {
//special case: column memcopy
- if( _schema.get(j).equals(src._schema.get(j-cl)) )
- _coldata.get(j).set(rl, ru, src._coldata.get(j-cl));
+ if( _schema[j].equals(src._schema[j-cl]) )
+ _coldata[j].set(rl, ru, src._coldata[j-cl]);
//general case w/ schema transformation
else
for( int i=rl; i<=ru; i++ ) {
String tmp = src.get(i-rl, j-cl)!=null ? src.get(i-rl, j-cl).toString() : null;
- set(i, j, UtilFunctions.stringToObject(_schema.get(j), tmp));
+ set(i, j, UtilFunctions.stringToObject(_schema[j], tmp));
}
}
}
@@ -912,7 +932,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
//construct recode map
HashMap<String,Long> map = new HashMap<String,Long>();
- Array ldata = _coldata.get(col);
+ Array ldata = _coldata[col];
for( int i=0; i<getNumRows(); i++ ) {
Object val = ldata.get(i);
if( val != null ) {
@@ -955,21 +975,21 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
//meta data copy if necessary
for( int j=0; j<getNumColumns(); j++ )
if( !that.isColumnMetadataDefault(j) ) {
- _colmeta.get(j).setNumDistinct(that._colmeta.get(j).getNumDistinct());
- _colmeta.get(j).setMvValue(that._colmeta.get(j).getMvValue());
+ _colmeta[j].setNumDistinct(that._colmeta[j].getNumDistinct());
+ _colmeta[j].setMvValue(that._colmeta[j].getMvValue());
}
//core frame block merge through cell copy
//with column-wide access pattern
for( int j=0; j<getNumColumns(); j++ ) {
//special case: copy non-zeros of column
- if( _schema.get(j).equals(that._schema.get(j)) )
- _coldata.get(j).setNz(0, _numRows-1, that._coldata.get(j));
+ if( _schema[j].equals(that._schema[j]) )
+ _coldata[j].setNz(0, _numRows-1, that._coldata[j]);
//general case w/ schema transformation
else {
for( int i=0; i<_numRows; i++ ) {
Object obj = UtilFunctions.objectToObject(
- getSchema().get(j), that.get(i,j), true);
+ _schema[j], that.get(i,j), true);
if (obj != null) //merge non-zeros
set(i, j,obj);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index f2c7ecb..bd457ca 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -21,8 +21,7 @@
package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.Arrays;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -569,7 +568,7 @@ public class OperationsOnMatrixValues
//allocate space for the output value
for(long r=resultBlockIndexTop; r<=resultBlockIndexBottom; r++)
{
- List<ValueType> schema = UtilFunctions.getSubSchema(block.getSchema(), tmpRange.colStart, tmpRange.colEnd);
+ ValueType[] schema = Arrays.copyOfRange(block.getSchema(), (int)tmpRange.colStart, (int)tmpRange.colEnd+1);
long iResultIndex = Math.max(((r-1)*brlen - ixrange.rowStart + 1), 0);
Pair<Long,FrameBlock> out=new Pair<Long,FrameBlock>(new Long(iResultIndex+1), new FrameBlock(schema));
outlist.add(out);
@@ -631,12 +630,11 @@ public class OperationsOnMatrixValues
int lbclen = clenLeft;
- List<ValueType> schemaPartialLeft = Collections.nCopies(lhs_lcl, ValueType.STRING);
- List<ValueType> schemaRHS = UtilFunctions.getSubSchema(fb.getSchema(), rhs_lcl, rhs_lcl-lhs_lcl+lhs_lcu);
- List<ValueType> schema = new ArrayList<ValueType>(schemaPartialLeft);
- schema.addAll(schemaRHS);
- List<ValueType> schemaPartialRight = Collections.nCopies(lbclen-schema.size(), ValueType.STRING);
- schema.addAll(schemaPartialRight);
+ ValueType[] schemaPartialLeft = UtilFunctions.nCopies(lhs_lcl, ValueType.STRING);
+ ValueType[] schemaRHS = Arrays.copyOfRange(fb.getSchema(), (int)(rhs_lcl), (int)(rhs_lcl-lhs_lcl+lhs_lcu+1));
+ ValueType[] schema = UtilFunctions.copyOf(schemaPartialLeft, schemaRHS);
+ ValueType[] schemaPartialRight = UtilFunctions.nCopies(lbclen-schema.length, ValueType.STRING);
+ schema = UtilFunctions.copyOf(schema, schemaPartialRight);
FrameBlock resultBlock = new FrameBlock(schema);
int iRHSRows = (int)(leftRowIndex<=rlen/brlenLeft?brlenLeft:rlen-(rlen/brlenLeft)*brlenLeft);
resultBlock.ensureAllocatedColumns(iRHSRows);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
index e844be1..41615c5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
-import java.util.List;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.OutputCollector;
@@ -54,7 +53,7 @@ public class FrameReblockBuffer
private int _brlen = -1;
private int _bclen = -1;
- private List<ValueType> _schema;
+ private ValueType[] _schema;
/**
@@ -64,7 +63,7 @@ public class FrameReblockBuffer
* @return
*
*/
- public FrameReblockBuffer( long rlen, long clen, List<ValueType> schema )
+ public FrameReblockBuffer( long rlen, long clen, ValueType[] schema )
{
this( DEFAULT_BUFFER_SIZE, rlen, clen, schema );
}
@@ -77,7 +76,7 @@ public class FrameReblockBuffer
* @return
*
*/
- public FrameReblockBuffer( int buffersize, long rlen, long clen, List<ValueType> schema )
+ public FrameReblockBuffer( int buffersize, long rlen, long clen, ValueType[] schema )
{
_bufflen = buffersize;
_count = 0;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
index ee0e56a..46090c0 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
@@ -68,7 +68,7 @@ public class BinAgent extends Encoder
super( null, clen );
}
- public BinAgent(JSONObject parsedSpec, List<String> colnames, int clen)
+ public BinAgent(JSONObject parsedSpec, String[] colnames, int clen)
throws JSONException, IOException
{
this(parsedSpec, colnames, clen, false);
@@ -81,7 +81,7 @@ public class BinAgent extends Encoder
* @throws JSONException
* @throws IOException
*/
- public BinAgent(JSONObject parsedSpec, List<String> colnames, int clen, boolean colsOnly)
+ public BinAgent(JSONObject parsedSpec, String[] colnames, int clen, boolean colsOnly)
throws JSONException, IOException
{
super( null, clen );
@@ -364,7 +364,7 @@ public class BinAgent extends Encoder
int colID = _colList[j];
for( int i=0; i<in.getNumRows(); i++ ) {
double inVal = UtilFunctions.objectToDouble(
- in.getSchema().get(colID-1), in.get(i, colID-1));
+ in.getSchema()[colID-1], in.get(i, colID-1));
int ix = Arrays.binarySearch(_binMaxs[j], inVal);
int binID = ((ix < 0) ? Math.abs(ix+1) : ix) + 1;
out.quickSetValue(i, colID-1, binID);
@@ -384,7 +384,7 @@ public class BinAgent extends Encoder
_binMaxs = new double[_colList.length][];
for( int j=0; j<_colList.length; j++ ) {
int colID = _colList[j]; //1-based
- int nbins = (int)meta.getColumnMetadata().get(colID-1).getNumDistinct();
+ int nbins = (int)meta.getColumnMetadata()[colID-1].getNumDistinct();
_binMins[j] = new double[nbins];
_binMaxs[j] = new double[nbins];
for( int i=0; i<nbins; i++ ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
index b51d639..e0b4826 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
@@ -64,7 +64,7 @@ public class DummycodeAgent extends Encoder
super(list, clen);
}
- public DummycodeAgent(JSONObject parsedSpec, List<String> colnames, int clen) throws JSONException {
+ public DummycodeAgent(JSONObject parsedSpec, String[] colnames, int clen) throws JSONException {
super(null, clen);
if ( parsedSpec.containsKey(TfUtils.TXMETHOD_DUMMYCODE) ) {
@@ -443,7 +443,7 @@ public class DummycodeAgent extends Encoder
idx++;
}
else {
- double ptval = UtilFunctions.objectToDouble(in.getSchema().get(colID-1), in.get(i, colID-1));
+ double ptval = UtilFunctions.objectToDouble(in.getSchema()[colID-1], in.get(i, colID-1));
ret.quickSetValue(i, ncolID-1, ptval);
ncolID++;
}
@@ -465,7 +465,7 @@ public class DummycodeAgent extends Encoder
_dummycodedLength = _clen;
for( int j=0; j<_colList.length; j++ ) {
int colID = _colList[j]; //1-based
- _domainSizes[j] = (int)meta.getColumnMetadata().get(colID-1).getNumDistinct();
+ _domainSizes[j] = (int)meta.getColumnMetadata()[colID-1].getNumDistinct();
_dummycodedLength += _domainSizes[j]-1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
index 1266ced..4ff93a5 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
@@ -98,7 +98,7 @@ public class MVImputeAgent extends Encoder
public KahanObject[] getMeans_scnomv() { return _scnomvMeanList; }
public CM_COV_Object[] getVars_scnomv() { return _scnomvVarList; }
- public MVImputeAgent(JSONObject parsedSpec, List<String> colnames, int clen)
+ public MVImputeAgent(JSONObject parsedSpec, String[] colnames, int clen)
throws JSONException
{
super(null, clen);
@@ -114,7 +114,7 @@ public class MVImputeAgent extends Encoder
_hist = new HashMap<Integer, HashMap<String,Long>>();
}
- public MVImputeAgent(JSONObject parsedSpec, String[] NAstrings, int clen)
+ public MVImputeAgent(JSONObject parsedSpec, String[] colnames, String[] NAstrings, int clen)
throws JSONException
{
super(null, clen);
@@ -947,7 +947,7 @@ public class MVImputeAgent extends Encoder
long off = _countList[j];
for( int i=0; i<in.getNumRows(); i++ )
_meanFn.execute2(_meanList[j], UtilFunctions.objectToDouble(
- in.getSchema().get(colID-1), in.get(i, colID-1)), off+i+1);
+ in.getSchema()[colID-1], in.get(i, colID-1)), off+i+1);
_replacementList[j] = String.valueOf(_meanList[j]._sum);
_countList[j] += in.getNumRows();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
index de6d59f..982f4b9 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.util.Iterator;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -50,7 +49,7 @@ public class OmitAgent extends Encoder
super(list, clen);
}
- public OmitAgent(JSONObject parsedSpec, List<String> colnames, int clen)
+ public OmitAgent(JSONObject parsedSpec, String[] colnames, int clen)
throws JSONException
{
super(null, clen);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
index edfdff4..8ec2db3 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
@@ -59,7 +59,7 @@ public class RecodeAgent extends Encoder
private HashMap<Integer, HashMap<String, Long>> _rcdMaps = new HashMap<Integer, HashMap<String, Long>>();
private HashMap<Integer, HashMap<String,String>> _finalMaps = null;
- public RecodeAgent(JSONObject parsedSpec, List<String> colnames, int clen)
+ public RecodeAgent(JSONObject parsedSpec, String[] colnames, int clen)
throws JSONException
{
super(null, clen);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
index dd18b43..7a5da65 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
@@ -23,7 +23,6 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
-import java.util.List;
import java.util.regex.Pattern;
import org.apache.hadoop.filecache.DistributedCache;
@@ -250,13 +249,11 @@ public class TfUtils implements Serializable{
private void createAgents(JSONObject spec, String[] naStrings)
throws IOException, JSONException
{
- List<String> colnames = Arrays.asList(_outputColumnNames);
-
- _oa = new OmitAgent(spec, colnames, _numInputCols);
- _mia = new MVImputeAgent(spec, naStrings, _numInputCols);
- _ra = new RecodeAgent(spec, colnames, _numInputCols);
- _ba = new BinAgent(spec, colnames, _numInputCols);
- _da = new DummycodeAgent(spec, colnames, _numInputCols);
+ _oa = new OmitAgent(spec, _outputColumnNames, _numInputCols);
+ _mia = new MVImputeAgent(spec, null, naStrings, _numInputCols);
+ _ra = new RecodeAgent(spec, _outputColumnNames, _numInputCols);
+ _ba = new BinAgent(spec, _outputColumnNames, _numInputCols);
+ _da = new DummycodeAgent(spec, _outputColumnNames, _numInputCols);
}
public void setupAgents(OmitAgent oa, MVImputeAgent mia, RecodeAgent ra, BinAgent ba, DummycodeAgent da) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderDummycode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderDummycode.java
index 2916742..3495cf3 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderDummycode.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderDummycode.java
@@ -52,7 +52,7 @@ public class DecoderDummycode extends Decoder
if( in.quickGetValue(i, k-1) != 0 ) {
int col = _colList[j] - 1;
out.set(i, col, UtilFunctions.doubleToObject(
- out.getSchema().get(col), k-_clPos[j]+1));
+ out.getSchema()[col], k-_clPos[j]+1));
}
return out;
}
@@ -63,8 +63,8 @@ public class DecoderDummycode extends Decoder
_cuPos = new int[_colList.length]; //col upper pos
for( int j=0, off=0; j<_colList.length; j++ ) {
int colID = _colList[j];
- int ndist = (int)meta.getColumnMetadata()
- .get(colID-1).getNumDistinct();
+ int ndist = (int)meta.getColumnMetadata()[colID-1]
+ .getNumDistinct();
_clPos[j] = off + colID;
_cuPos[j] = _clPos[j] + ndist;
off += ndist - 1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
index f276015..facfff8 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
@@ -46,7 +46,7 @@ public class DecoderFactory
* @throws DMLRuntimeException
*/
@SuppressWarnings("unchecked")
- public static Decoder createDecoder(String spec, List<String> colnames, List<ValueType> schema, FrameBlock meta)
+ public static Decoder createDecoder(String spec, String[] colnames, List<ValueType> schema, FrameBlock meta)
throws DMLRuntimeException
{
Decoder decoder = null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderPassThrough.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderPassThrough.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderPassThrough.java
index d2bf7fa..1ee0568 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderPassThrough.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderPassThrough.java
@@ -70,8 +70,8 @@ public class DecoderPassThrough extends Decoder
ix1 ++;
}
else { //_colList[ix1] > _dcCols[ix2]
- off += (int)meta.getColumnMetadata()
- .get(_dcCols[ix2]-1).getNumDistinct() - 1;
+ off += (int)meta.getColumnMetadata()[_dcCols[ix2]-1]
+ .getNumDistinct() - 1;
ix2 ++;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderRecode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderRecode.java
index 5484ded..42a0da9 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderRecode.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderRecode.java
@@ -54,7 +54,7 @@ public class DecoderRecode extends Decoder
for( int j=0; j<_colList.length; j++ ) {
int colID = _colList[j];
double val = UtilFunctions.objectToDouble(
- out.getSchema().get(colID-1), out.get(i, colID-1));
+ out.getSchema()[colID-1], out.get(i, colID-1));
long key = UtilFunctions.toLong(val);
out.set(i, colID-1, _rcMaps[j].get(key));
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
index 8adea7b..b71f563 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.transform.encode;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
@@ -48,8 +47,8 @@ public class EncoderFactory
* @return
* @throws DMLRuntimeException
*/
- public static Encoder createEncoder(String spec, List<String> colnames, int clen, FrameBlock meta) throws DMLRuntimeException {
- return createEncoder(spec, colnames, Collections.nCopies(clen, ValueType.STRING), meta);
+ public static Encoder createEncoder(String spec, String[] colnames, int clen, FrameBlock meta) throws DMLRuntimeException {
+ return createEncoder(spec, colnames, UtilFunctions.nCopies(clen, ValueType.STRING), meta);
}
/**
@@ -61,8 +60,8 @@ public class EncoderFactory
* @return
* @throws DMLRuntimeException
*/
- public static Encoder createEncoder(String spec, List<String> colnames, List<ValueType> schema, int clen, FrameBlock meta) throws DMLRuntimeException {
- List<ValueType> lschema = (schema==null) ? Collections.nCopies(clen, ValueType.STRING) : schema;
+ public static Encoder createEncoder(String spec, String[] colnames, ValueType[] schema, int clen, FrameBlock meta) throws DMLRuntimeException {
+ ValueType[] lschema = (schema==null) ? UtilFunctions.nCopies(clen, ValueType.STRING) : schema;
return createEncoder(spec, colnames, lschema, meta);
}
@@ -75,11 +74,11 @@ public class EncoderFactory
* @throws DMLRuntimeException
*/
@SuppressWarnings("unchecked")
- public static Encoder createEncoder(String spec, List<String> colnames, List<ValueType> schema, FrameBlock meta)
+ public static Encoder createEncoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta)
throws DMLRuntimeException
{
Encoder encoder = null;
- int clen = schema.size();
+ int clen = schema.length;
try {
//parse transform specification
@@ -111,13 +110,13 @@ public class EncoderFactory
lencoders.add(new EncoderPassThrough(
ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), clen));
if( !dcIDs.isEmpty() )
- lencoders.add(new DummycodeAgent(jSpec, colnames, schema.size()));
+ lencoders.add(new DummycodeAgent(jSpec, colnames, schema.length));
if( !binIDs.isEmpty() )
- lencoders.add(new BinAgent(jSpec, colnames, schema.size(), true));
+ lencoders.add(new BinAgent(jSpec, colnames, schema.length, true));
if( !oIDs.isEmpty() )
- lencoders.add(new OmitAgent(jSpec, colnames, schema.size()));
+ lencoders.add(new OmitAgent(jSpec, colnames, schema.length));
if( !mvIDs.isEmpty() ) {
- MVImputeAgent ma = new MVImputeAgent(jSpec, colnames, schema.size());
+ MVImputeAgent ma = new MVImputeAgent(jSpec, colnames, schema.length);
ma.initRecodeIDList(rcIDs);
lencoders.add(ma);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderPassThrough.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderPassThrough.java b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderPassThrough.java
index ab146ce..08722fd 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderPassThrough.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderPassThrough.java
@@ -67,7 +67,7 @@ public class EncoderPassThrough extends Encoder
public MatrixBlock apply(FrameBlock in, MatrixBlock out) {
for( int j=0; j<_colList.length; j++ ) {
int col = _colList[j]-1;
- ValueType vt = in.getSchema().get(col);
+ ValueType vt = in.getSchema()[col];
for( int i=0; i<in.getNumRows(); i++ ) {
Object val = in.get(i, col);
out.quickSetValue(i, col, (val==null||(vt==ValueType.STRING
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
index de883f3..d12ff1d 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
@@ -27,11 +27,12 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
+import jodd.util.ArraysUtil;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.sysml.api.jmlc.Connection;
@@ -74,7 +75,7 @@ public class TfMetaUtils
* @return
* @throws DMLRuntimeException
*/
- public static boolean containsOmitSpec(String spec, List<String> colnames) throws DMLRuntimeException {
+ public static boolean containsOmitSpec(String spec, String[] colnames) throws DMLRuntimeException {
return (TfMetaUtils.parseJsonIDList(spec, colnames, TfUtils.TXMETHOD_OMIT).length > 0);
}
@@ -86,7 +87,7 @@ public class TfMetaUtils
* @return
* @throws DMLRuntimeException
*/
- public static int[] parseJsonIDList(String spec, List<String> colnames, String group)
+ public static int[] parseJsonIDList(String spec, String[] colnames, String group)
throws DMLRuntimeException
{
try {
@@ -107,7 +108,7 @@ public class TfMetaUtils
* @return
* @throws JSONException
*/
- public static int[] parseJsonIDList(JSONObject spec, List<String> colnames, String group)
+ public static int[] parseJsonIDList(JSONObject spec, String[] colnames, String group)
throws JSONException
{
int[] colList = new int[0];
@@ -127,7 +128,7 @@ public class TfMetaUtils
colList = new int[attrs.size()];
for(int i=0; i < colList.length; i++) {
colList[i] = ids ? UtilFunctions.toInt(attrs.get(i)) :
- (colnames.indexOf(attrs.get(i)) + 1);
+ (ArraysUtil.indexOf(colnames, attrs.get(i)) + 1);
if( colList[i] <= 0 ) {
throw new RuntimeException("Specified column '" +
attrs.get(i)+"' does not exist.");
@@ -148,7 +149,7 @@ public class TfMetaUtils
* @return
* @throws JSONException
*/
- public static int[] parseJsonObjectIDList(JSONObject spec, List<String> colnames, String group)
+ public static int[] parseJsonObjectIDList(JSONObject spec, String[] colnames, String group)
throws JSONException
{
int[] colList = new int[0];
@@ -161,7 +162,7 @@ public class TfMetaUtils
for(int j=0; j<colspecs.size(); j++) {
JSONObject colspec = (JSONObject) colspecs.get(j);
colList[j] = ids ? colspec.getInt("id") :
- (colnames.indexOf(colspec.get("name")) + 1);
+ (ArrayUtils.indexOf(colnames, colspec.get("name")) + 1);
if( colList[j] <= 0 ) {
throw new RuntimeException("Specified column '" +
colspec.get(ids?"id":"name")+"' does not exist.");
@@ -190,15 +191,15 @@ public class TfMetaUtils
{
//read column names
String colnamesStr = MapReduceTool.readStringFromHDFSFile(metapath+File.separator+TfUtils.TXMTD_COLNAMES);
- List<String> colnames = Arrays.asList(IOUtilFunctions.split(colnamesStr.trim(), colDelim));
+ String[] colnames = IOUtilFunctions.split(colnamesStr.trim(), colDelim);
//read meta data (currently supported: recode, dummycode, bin, omit, impute)
//note: recode/binning and impute might be applied on the same column
HashMap<String,String> meta = new HashMap<String,String>();
HashMap<String,String> mvmeta = new HashMap<String,String>();
int rows = 0;
- for( int j=0; j<colnames.size(); j++ ) {
- String colName = colnames.get(j);
+ for( int j=0; j<colnames.length; j++ ) {
+ String colName = colnames[j];
//read recode maps for recoded or dummycoded columns
String name = metapath+File.separator+"Recode"+File.separator+colName;
if( MapReduceTool.existsFileOnHDFS(name+TfUtils.TXMTD_RCD_MAP_SUFFIX) ) {
@@ -244,15 +245,15 @@ public class TfMetaUtils
{
//read column names
String colnamesStr = IOUtilFunctions.toString(Connection.class.getResourceAsStream(metapath+"/"+TfUtils.TXMTD_COLNAMES));
- List<String> colnames = Arrays.asList(IOUtilFunctions.split(colnamesStr.trim(), colDelim));
+ String[] colnames = IOUtilFunctions.split(colnamesStr.trim(), colDelim);
//read meta data (currently supported: recode, dummycode, bin, omit)
//note: recode/binning and impute might be applied on the same column
HashMap<String,String> meta = new HashMap<String,String>();
HashMap<String,String> mvmeta = new HashMap<String,String>();
int rows = 0;
- for( int j=0; j<colnames.size(); j++ ) {
- String colName = colnames.get(j);
+ for( int j=0; j<colnames.length; j++ ) {
+ String colName = colnames[j];
//read recode maps for recoded or dummycoded columns
String name = metapath+"/"+"Recode"+"/"+colName;
String map = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_MAP_SUFFIX));
@@ -294,18 +295,18 @@ public class TfMetaUtils
* @return
* @throws IOException
*/
- private static FrameBlock convertToTransformMetaDataFrame(int rows, List<String> colnames, List<Integer> rcIDs, List<Integer> binIDs,
+ private static FrameBlock convertToTransformMetaDataFrame(int rows, String[] colnames, List<Integer> rcIDs, List<Integer> binIDs,
HashMap<String,String> meta, HashMap<String,String> mvmeta)
throws IOException
{
//create frame block w/ pure string schema
- List<ValueType> schema = Collections.nCopies(colnames.size(), ValueType.STRING);
+ ValueType[] schema = UtilFunctions.nCopies(colnames.length, ValueType.STRING);
FrameBlock ret = new FrameBlock(schema, colnames);
ret.ensureAllocatedColumns(rows);
//encode recode maps (recoding/dummycoding) into frame
for( Integer colID : rcIDs ) {
- String name = colnames.get(colID-1);
+ String name = colnames[colID-1];
String map = meta.get(name);
if( map == null )
throw new IOException("Recode map for column '"+name+"' (id="+colID+") not existing.");
@@ -324,7 +325,7 @@ public class TfMetaUtils
//encode bin maps (binning) into frame
for( Integer colID : binIDs ) {
- String name = colnames.get(colID-1);
+ String name = colnames[colID-1];
String map = meta.get(name);
if( map == null )
throw new IOException("Binning map for column '"+name+"' (id="+colID+") not existing.");
@@ -343,7 +344,7 @@ public class TfMetaUtils
//encode impute meta data into frame
for( Entry<String, String> e : mvmeta.entrySet() ) {
- int colID = colnames.indexOf(e.getKey()) + 1;
+ int colID = ArrayUtils.indexOf(colnames, e.getKey()) + 1;
String mvVal = e.getValue().split(TfUtils.TXMTD_SEP)[1];
ret.getColumnMetadata(colID-1).setMvValue(mvVal);
}
@@ -361,7 +362,7 @@ public class TfMetaUtils
* @throws IOException
*/
@SuppressWarnings("unchecked")
- private static List<Integer> parseRecodeColIDs(String spec, List<String> colnames)
+ private static List<Integer> parseRecodeColIDs(String spec, String[] colnames)
throws IOException
{
if( spec == null )
@@ -391,7 +392,7 @@ public class TfMetaUtils
* @return
* @throws IOException
*/
- public static List<Integer> parseBinningColIDs(String spec, List<String> colnames)
+ public static List<Integer> parseBinningColIDs(String spec, String[] colnames)
throws IOException
{
try {
@@ -409,7 +410,7 @@ public class TfMetaUtils
* @return
* @throws IOException
*/
- public static List<Integer> parseBinningColIDs(JSONObject jSpec, List<String> colnames)
+ public static List<Integer> parseBinningColIDs(JSONObject jSpec, String[] colnames)
throws IOException
{
try {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index 9bb27d9..381ad87 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -629,10 +628,10 @@ public class DataConverter
MatrixBlock mb = new MatrixBlock(m, n, false);
mb.allocateDenseBlock();
- List<ValueType> schema = frame.getSchema();
- int dFreq = Collections.frequency(schema, ValueType.DOUBLE);
+ ValueType[] schema = frame.getSchema();
+ int dFreq = UtilFunctions.frequency(schema, ValueType.DOUBLE);
- if( dFreq == schema.size() ) {
+ if( dFreq == schema.length ) {
// special case double schema (without cell-object creation,
// cache-friendly row-column copy)
double[][] a = new double[n][];
@@ -654,7 +653,7 @@ public class DataConverter
for( int i=0; i<frame.getNumRows(); i++ )
for( int j=0; j<frame.getNumColumns(); j++ ) {
mb.appendValue(i, j, UtilFunctions.objectToDouble(
- schema.get(j), frame.get(i, j)));
+ schema[j], frame.get(i, j)));
}
}
@@ -699,7 +698,7 @@ public class DataConverter
return new FrameBlock();
//create schema and frame block
- List<ValueType> schema = Collections.nCopies(data[0].length, ValueType.STRING);
+ ValueType[] schema = UtilFunctions.nCopies(data[0].length, ValueType.STRING);
return convertToFrameBlock(data, schema);
}
@@ -709,7 +708,7 @@ public class DataConverter
* @param schema
* @return
*/
- public static FrameBlock convertToFrameBlock(String[][] data, List<ValueType> schema) {
+ public static FrameBlock convertToFrameBlock(String[][] data, ValueType[] schema) {
//check for empty frame block
if( data == null || data.length==0 )
return new FrameBlock();
@@ -725,7 +724,7 @@ public class DataConverter
* @param colnames
* @return
*/
- public static FrameBlock convertToFrameBlock(String[][] data, List<ValueType> schema, List<String> colnames) {
+ public static FrameBlock convertToFrameBlock(String[][] data, ValueType[] schema, String[] colnames) {
//check for empty frame block
if( data == null || data.length==0 )
return new FrameBlock();
@@ -753,7 +752,7 @@ public class DataConverter
*/
public static FrameBlock convertToFrameBlock(MatrixBlock mb, ValueType vt) {
//create schema and frame block
- List<ValueType> schema = Collections.nCopies(mb.getNumColumns(), vt);
+ ValueType[] schema = UtilFunctions.nCopies(mb.getNumColumns(), vt);
return convertToFrameBlock(mb, schema);
}
@@ -763,7 +762,7 @@ public class DataConverter
* @param schema
* @return
*/
- public static FrameBlock convertToFrameBlock(MatrixBlock mb, List<ValueType> schema)
+ public static FrameBlock convertToFrameBlock(MatrixBlock mb, ValueType[] schema)
{
FrameBlock frame = new FrameBlock(schema);
Object[] row = new Object[mb.getNumColumns()];
@@ -780,7 +779,7 @@ public class DataConverter
double[] aval = sblock.values(i);
for( int j=apos; j<apos+alen; j++ ) {
row[aix[j]] = UtilFunctions.doubleToObject(
- schema.get(aix[j]), aval[j]);
+ schema[aix[j]], aval[j]);
}
}
frame.appendRow(row);
@@ -788,9 +787,9 @@ public class DataConverter
}
else //DENSE
{
- int dFreq = Collections.frequency(schema, ValueType.DOUBLE);
+ int dFreq = UtilFunctions.frequency(schema, ValueType.DOUBLE);
- if( dFreq == schema.size() ) {
+ if( dFreq == schema.length ) {
// special case double schema (without cell-object creation,
// col pre-allocation, and cache-friendly row-column copy)
int m = mb.getNumRows();
@@ -816,7 +815,7 @@ public class DataConverter
for( int i=0; i<mb.getNumRows(); i++ ) {
for( int j=0; j<mb.getNumColumns(); j++ ) {
row[j] = UtilFunctions.doubleToObject(
- schema.get(j), mb.quickGetValue(i, j));
+ schema[j], mb.quickGetValue(i, j));
}
frame.appendRow(row);
}
@@ -1052,7 +1051,7 @@ public class DataConverter
//print column names
sb.append("#"); sb.append(separator);
for( int j=0; j<colLength; j++ ) {
- sb.append(fb.getColumnNames().get(j));
+ sb.append(fb.getColumnNames()[j]);
if( j != colLength-1 )
sb.append(separator);
}
@@ -1061,7 +1060,7 @@ public class DataConverter
//print schema
sb.append("#"); sb.append(separator);
for( int j=0; j<colLength; j++ ) {
- sb.append(fb.getSchema().get(j));
+ sb.append(fb.getSchema()[j]);
if( j != colLength-1 )
sb.append(separator);
}
@@ -1078,7 +1077,7 @@ public class DataConverter
Object[] row = iter.next();
for( int j=0; j<colLength; j++ ) {
if( row[j]!=null ) {
- if( fb.getSchema().get(j) == ValueType.DOUBLE )
+ if( fb.getSchema()[j] == ValueType.DOUBLE )
sb.append(df.format(row[j]));
else
sb.append(row[j]);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
index c99b79b..de6b8e8 100644
--- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
+++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
@@ -25,7 +25,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
-import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -432,7 +431,7 @@ public class MapReduceTool
writeMetaDataFile(mtdfile, vt, null, DataType.MATRIX, mc, outinfo);
}
- public static void writeMetaDataFile(String mtdfile, ValueType vt, List<ValueType> schema, DataType dt, MatrixCharacteristics mc, OutputInfo outinfo)
+ public static void writeMetaDataFile(String mtdfile, ValueType vt, ValueType[] schema, DataType dt, MatrixCharacteristics mc, OutputInfo outinfo)
throws IOException {
writeMetaDataFile(mtdfile, vt, schema, dt, mc, outinfo, null);
}
@@ -442,7 +441,7 @@ public class MapReduceTool
writeMetaDataFile(mtdfile, vt, null, DataType.MATRIX, mc, outinfo, formatProperties);
}
- public static void writeMetaDataFile(String mtdfile, ValueType vt, List<ValueType> schema, DataType dt, MatrixCharacteristics mc,
+ public static void writeMetaDataFile(String mtdfile, ValueType vt, ValueType[] schema, DataType dt, MatrixCharacteristics mc,
OutputInfo outinfo, FileFormatProperties formatProperties)
throws IOException
{
@@ -490,7 +489,7 @@ public class MapReduceTool
* @throws JSONException
* @throws DMLRuntimeException
*/
- public static String metaDataToString(String mtdfile, ValueType vt, List<ValueType> schema, DataType dt, MatrixCharacteristics mc,
+ public static String metaDataToString(String mtdfile, ValueType vt, ValueType[] schema, DataType dt, MatrixCharacteristics mc,
OutputInfo outinfo, FileFormatProperties formatProperties) throws JSONException, DMLRuntimeException
{
OrderedJSONObject mtd = new OrderedJSONObject(); // maintain order in output file
@@ -502,11 +501,11 @@ public class MapReduceTool
}
else {
StringBuffer schemaSB = new StringBuffer();
- for(int i=0; i < schema.size(); i++) {
- if( schema.get(i) == ValueType.UNKNOWN )
+ for(int i=0; i < schema.length; i++) {
+ if( schema[i] == ValueType.UNKNOWN )
schemaSB.append("*");
else
- schemaSB.append(schema.get(i).toString());
+ schemaSB.append(schema[i].toString());
schemaSB.append(DataExpression.DEFAULT_DELIM_DELIMITER);
}
mtd.put(DataExpression.SCHEMAPARAM, schemaSB.toString());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index e5a792a..89472e1 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.instructions.InstructionUtils;
@@ -604,25 +605,6 @@ public class UtilFunctions
}
/**
- * Returns the subset of the schema
- *
- * @param srcSchema
- * @param lStart
- * @param lEnd
- *
- * @return
- */
- public static List<ValueType> getSubSchema(List<ValueType> srcSchema, long lStart, long lEnd)
- {
- ValueType [] schema = new ValueType[(int) (lEnd-lStart+1)];
- for(int i = 0; i < schema.length; i++)
- schema[i] = srcSchema.get((int) (lStart+i));
-
- return Arrays.asList(schema);
- }
-
-
- /**
* This function will return datatype, if its Matrix or Frame
*
* @param str
@@ -660,4 +642,39 @@ public class UtilFunctions
return (!sobj.equals("0") && !sobj.equals("0.0"));
}
}
+
+ /**
+ *
+ * @param n
+ * @param vt
+ * @return
+ */
+ public static ValueType[] nCopies(int n, ValueType vt) {
+ ValueType[] ret = new ValueType[n];
+ Arrays.fill(ret, vt);
+ return ret;
+ }
+
+ /**
+ *
+ * @param schema
+ * @param vt
+ * @return
+ */
+ public static int frequency(ValueType[] schema, ValueType vt) {
+ int count = 0;
+ for( ValueType tmp : schema )
+ count += tmp.equals(vt) ? 1 : 0;
+ return count;
+ }
+
+ /**
+ *
+ * @param schema1
+ * @param schema2
+ * @return
+ */
+ public static ValueType[] copyOf(ValueType[] schema1, ValueType[] schema2) {
+ return (ValueType[]) ArrayUtils.addAll(schema1, schema2);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 4273113..e7bb720 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -31,7 +31,6 @@ import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.List;
import org.apache.sysml.lops.Lop;
import org.apache.commons.io.FileUtils;
@@ -1700,7 +1699,7 @@ public abstract class AutomatedTestBase
* @throws IOException
* @throws DMLRuntimeException
*/
- protected double[][] writeInputFrame(String name, double[][] data, boolean bIncludeR, List<ValueType> schema, OutputInfo oi)
+ protected double[][] writeInputFrame(String name, double[][] data, boolean bIncludeR, ValueType[] schema, OutputInfo oi)
throws DMLRuntimeException, IOException
{
String completePath = baseDirectory + INPUT_DIR + name;
@@ -1725,14 +1724,14 @@ public abstract class AutomatedTestBase
return data;
}
- protected double[][] writeInputFrameWithMTD(String name, double[][] data, boolean bIncludeR, List<ValueType> schema, OutputInfo oi)
+ protected double[][] writeInputFrameWithMTD(String name, double[][] data, boolean bIncludeR, ValueType[] schema, OutputInfo oi)
throws DMLRuntimeException, IOException
{
MatrixCharacteristics mc = new MatrixCharacteristics(data.length, data[0].length, OptimizerUtils.DEFAULT_BLOCKSIZE, data[0].length, -1);
return writeInputFrameWithMTD(name, data, bIncludeR, mc, schema, oi);
}
- protected double[][] writeInputFrameWithMTD(String name, double[][] data, boolean bIncludeR, MatrixCharacteristics mc, List<ValueType> schema, OutputInfo oi)
+ protected double[][] writeInputFrameWithMTD(String name, double[][] data, boolean bIncludeR, MatrixCharacteristics mc, ValueType[] schema, OutputInfo oi)
throws DMLRuntimeException, IOException
{
writeInputFrame(name, data, bIncludeR, schema, oi);
@@ -1766,7 +1765,7 @@ public abstract class AutomatedTestBase
* @throws IOException
* @throws DMLRuntimeException
*/
- protected double[][] writeInputFrame(String name, double[][] data, List<ValueType> schema, OutputInfo oi)
+ protected double[][] writeInputFrame(String name, double[][] data, ValueType[] schema, OutputInfo oi)
throws DMLRuntimeException, IOException
{
return writeInputFrame(name, data, false, schema, oi);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendDistTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendDistTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendDistTest.java
index 0d3b932..ccb91f4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendDistTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendDistTest.java
@@ -20,7 +20,6 @@
package org.apache.sysml.test.integration.functions.frame;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -158,11 +157,11 @@ public class FrameAppendDistTest extends AutomatedTestBase
inputDir() + " " + expectedDir() + " " + (rbind? "rbind": "cbind");
//initialize the frame data.
- List<ValueType> lschemaA = Arrays.asList(genMixSchema(cols1));
+ ValueType[] lschemaA = genMixSchema(cols1);
double[][] A = getRandomMatrix(rows1, cols1, min, max, sparsity, 1111 /*\\System.currentTimeMillis()*/);
writeInputFrameWithMTD("A", A, true, lschemaA, OutputInfo.BinaryBlockOutputInfo);
- List<ValueType> lschemaB = Arrays.asList(genMixSchema(cols2));
+ ValueType[] lschemaB = genMixSchema(cols2);
double[][] B = getRandomMatrix(rows2, cols2, min, max, sparsity, 2345 /*\\System.currentTimeMillis()*/);
writeInputFrameWithMTD("B", B, true, lschemaB, OutputInfo.BinaryBlockOutputInfo);
@@ -171,15 +170,14 @@ public class FrameAppendDistTest extends AutomatedTestBase
runTest(true, exceptionExpected, null, expectedNumberOfJobs);
runRScript(true);
- List<ValueType> lschemaAB = new ArrayList<ValueType>(lschemaA);
- lschemaAB.addAll(lschemaB);
+ ValueType[] lschemaAB = UtilFunctions.copyOf(lschemaA, lschemaB);
for(String file: config.getOutputFiles())
{
FrameBlock frameBlock = readDMLFrameFromHDFS(file, InputInfo.BinaryBlockInputInfo);
MatrixCharacteristics md = new MatrixCharacteristics(frameBlock.getNumRows(), frameBlock.getNumColumns(), -1, -1);
FrameBlock frameRBlock = readRFrameFromHDFS(file+".csv", InputInfo.CSVInputInfo, md);
- verifyFrameData(frameBlock, frameRBlock, (ValueType[]) lschemaAB.toArray(new ValueType[0]));
+ verifyFrameData(frameBlock, frameRBlock, (ValueType[]) lschemaAB);
System.out.println("File processed is " + file);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java
index 555cf55..46d07b5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.instructions.cp.AppendCPInstruction.AppendType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -90,24 +87,22 @@ public class FrameAppendTest extends AutomatedTestBase
double[][] B = getRandomMatrix(rows, schema2.length, -10, 10, 0.9, 129);
//init data frame 1
- List<ValueType> lschema1 = Arrays.asList(schema1);
- FrameBlock frame1 = new FrameBlock(lschema1);
- Object[] row1 = new Object[lschema1.size()];
+ FrameBlock frame1 = new FrameBlock(schema1);
+ Object[] row1 = new Object[schema1.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema1.size(); j++ )
- A[i][j] = UtilFunctions.objectToDouble(lschema1.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema1.get(j), A[i][j]));
+ for( int j=0; j<schema1.length; j++ )
+ A[i][j] = UtilFunctions.objectToDouble(schema1[j],
+ row1[j] = UtilFunctions.doubleToObject(schema1[j], A[i][j]));
frame1.appendRow(row1);
}
//init data frame 2
- List<ValueType> lschema2 = Arrays.asList(schema2);
- FrameBlock frame2 = new FrameBlock(lschema2);
- Object[] row2 = new Object[lschema2.size()];
+ FrameBlock frame2 = new FrameBlock(schema2);
+ Object[] row2 = new Object[schema2.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema2.size(); j++ )
- B[i][j] = UtilFunctions.objectToDouble(lschema2.get(j),
- row2[j] = UtilFunctions.doubleToObject(lschema2.get(j), B[i][j]));
+ for( int j=0; j<schema2.length; j++ )
+ B[i][j] = UtilFunctions.objectToDouble(schema2[j],
+ row2[j] = UtilFunctions.doubleToObject(schema2[j], B[i][j]));
frame2.appendRow(row2);
}
@@ -125,10 +120,10 @@ public class FrameAppendTest extends AutomatedTestBase
Assert.fail("Wrong number of rows: "+frame3.getNumRows()+", expected: "+mbC.getNumRows());
//check correct values
- List<ValueType> lschema = frame3.getSchema();
+ ValueType[] lschema = frame3.getSchema();
for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame3.get(i, j));
+ for( int j=0; j<lschema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(lschema[j], frame3.get(i, j));
if( tmp != mbC.quickGetValue(i, j) )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+mbC.quickGetValue(i, j));
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCastingTest.java
index 5fe14dd..c7a5557 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCastingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCastingTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -102,12 +99,11 @@ public class FrameCastingTest extends AutomatedTestBase
if( ctype == CastType.F2M )
{
//construct input schema
- List<ValueType> lschema1 = Arrays.asList(schema);
- FrameBlock frame1 = new FrameBlock(lschema1);
- Object[] row1 = new Object[lschema1.size()];
+ FrameBlock frame1 = new FrameBlock(schema);
+ Object[] row1 = new Object[schema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema1.size(); j++ )
- row1[j] = UtilFunctions.doubleToObject(lschema1.get(j), A[i][j]);
+ for( int j=0; j<schema.length; j++ )
+ row1[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]);
frame1.appendRow(row1);
}
@@ -122,7 +118,7 @@ public class FrameCastingTest extends AutomatedTestBase
else if( ctype == CastType.M2F_S )
{
MatrixBlock mb = DataConverter.convertToMatrixBlock(A);
- frame = DataConverter.convertToFrameBlock(mb, Arrays.asList(schema));
+ frame = DataConverter.convertToFrameBlock(mb, schema);
}
//check basic meta data
@@ -130,10 +126,10 @@ public class FrameCastingTest extends AutomatedTestBase
Assert.fail("Wrong number of rows: "+frame.getNumRows()+", expected: "+rows);
//check correct values
- List<ValueType> lschema = frame.getSchema();
+ ValueType[] lschema = frame.getSchema();
for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j));
+ for( int j=0; j<lschema.length; j++ ) {
+ double tmp = UtilFunctions.objectToDouble(lschema[j], frame.get(i, j));
if( tmp != A[i][j] )
Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
index e8c3c51..511f11c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
@@ -285,9 +285,8 @@ public class FrameConverterTest extends AutomatedTestBase
try
{
//initialize the frame data.
- List<ValueType> lschema = Arrays.asList(schema);
- FrameBlock frame1 = new FrameBlock(lschema);
- initFrameData(frame1, A, lschema);
+ FrameBlock frame1 = new FrameBlock(schema);
+ initFrameData(frame1, A, schema);
//write frame data to hdfs
FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
@@ -346,9 +345,8 @@ public class FrameConverterTest extends AutomatedTestBase
}
else {
//initialize the frame data.
- List<ValueType> lschema = Arrays.asList(schema);
- frame1 = new FrameBlock(lschema);
- initFrameData(frame1, A, lschema);
+ frame1 = new FrameBlock(schema);
+ initFrameData(frame1, A, schema);
//write frame data to hdfs
FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
@@ -393,12 +391,12 @@ public class FrameConverterTest extends AutomatedTestBase
* @param data
* @param lschema
*/
- private void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema) {
- Object[] row1 = new Object[lschema.size()];
+ private void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema) {
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ for( int j=0; j<lschema.length; j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
frame.appendRow(row1);
}
}
@@ -428,10 +426,10 @@ public class FrameConverterTest extends AutomatedTestBase
private void verifyFrameMatrixData(FrameBlock frame, MatrixBlock matrix) {
for ( int i=0; i<frame.getNumRows(); i++ )
for( int j=0; j<frame.getNumColumns(); j++ ) {
- Object val1 = UtilFunctions.doubleToObject(frame.getSchema().get(j),
- UtilFunctions.objectToDouble(frame.getSchema().get(j), frame.get(i, j)));
- Object val2 = UtilFunctions.doubleToObject(frame.getSchema().get(j), matrix.getValue(i, j));
- if(( UtilFunctions.compareTo(frame.getSchema().get(j), val1, val2)) != 0)
+ Object val1 = UtilFunctions.doubleToObject(frame.getSchema()[j],
+ UtilFunctions.objectToDouble(frame.getSchema()[j], frame.get(i, j)));
+ Object val2 = UtilFunctions.doubleToObject(frame.getSchema()[j], matrix.getValue(i, j));
+ if(( UtilFunctions.compareTo(frame.getSchema()[j], val1, val2)) != 0)
Assert.fail("Frame value for cell ("+ i + "," + j + ") is " + val1 +
", is not same as matrix value " + val2);
}
@@ -455,6 +453,7 @@ public class FrameConverterTest extends AutomatedTestBase
{
SparkExecutionContext sec = (SparkExecutionContext) ExecutionContextFactory.createContext();
JavaSparkContext sc = sec.getSparkContext();
+ ValueType[] lschema = schema.toArray(new ValueType[0]);
MapReduceTool.deleteFileIfExistOnHDFS(fnameOut);
@@ -483,7 +482,7 @@ public class FrameConverterTest extends AutomatedTestBase
OutputInfo oinfo = OutputInfo.BinaryBlockOutputInfo;
JavaPairRDD<LongWritable,Text> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, iinfo.inputKeyClass, iinfo.inputValueClass);
JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils
- .textCellToBinaryBlock(sc, rddIn, mc, schema)
+ .textCellToBinaryBlock(sc, rddIn, mc, lschema)
.mapToPair(new LongFrameToLongWritableFrameFunction());
rddOut.saveAsHadoopFile(fnameOut, LongWritable.class, FrameBlock.class, oinfo.outputFormatClass);
break;
@@ -519,8 +518,8 @@ public class FrameConverterTest extends AutomatedTestBase
//Create DataFrame
SQLContext sqlContext = new SQLContext(sc);
- StructType dfSchema = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(schema, false);
- JavaRDD<Row> rowRDD = FrameRDDConverterUtils.csvToRowRDD(sc, fnameIn, separator, schema);
+ StructType dfSchema = FrameRDDConverterUtils.convertFrameSchemaToDFSchema(lschema, false);
+ JavaRDD<Row> rowRDD = FrameRDDConverterUtils.csvToRowRDD(sc, fnameIn, separator, lschema);
DataFrame df = sqlContext.createDataFrame(rowRDD, dfSchema);
JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils
@@ -535,7 +534,7 @@ public class FrameConverterTest extends AutomatedTestBase
JavaPairRDD<Long, FrameBlock> rddIn = sc
.hadoopFile(fnameIn, iinfo.inputFormatClass, LongWritable.class, FrameBlock.class)
.mapToPair(new LongWritableFrameToLongFrameFunction());
- DataFrame df = FrameRDDConverterUtils.binaryBlockToDataFrame(new SQLContext(sc), rddIn, mc, schema);
+ DataFrame df = FrameRDDConverterUtils.binaryBlockToDataFrame(new SQLContext(sc), rddIn, mc, lschema);
//Convert back DataFrame to binary block for comparison using original binary to converted DF and back to binary
JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCopyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCopyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCopyTest.java
index e713a86..84bd36d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCopyTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameCopyTest.java
@@ -19,9 +19,6 @@
package org.apache.sysml.test.integration.functions.frame;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.instructions.cp.AppendCPInstruction.AppendType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -89,14 +86,12 @@ public class FrameCopyTest extends AutomatedTestBase
//Initialize the frame data.
//init data frame 1
- List<ValueType> lschema1 = Arrays.asList(schema1);
- FrameBlock frame1 = new FrameBlock(lschema1);
- initFrameData(frame1, A, lschema1);
+ FrameBlock frame1 = new FrameBlock(schema1);
+ initFrameData(frame1, A, schema1);
//init data frame 2
- List<ValueType> lschema2 = Arrays.asList(schema2);
- FrameBlock frame2 = new FrameBlock(lschema2);
- initFrameData(frame2, B, lschema2);
+ FrameBlock frame2 = new FrameBlock(schema2);
+ initFrameData(frame2, B, schema2);
//copy from one frame to another.
FrameBlock frame1Backup = new FrameBlock(frame1.getSchema(), frame1.getColumnNames());
@@ -125,44 +120,44 @@ public class FrameCopyTest extends AutomatedTestBase
}
}
- void initFrameData(FrameBlock frame, double[][] data, List<ValueType> lschema)
+ void initFrameData(FrameBlock frame, double[][] data, ValueType[] lschema)
{
- Object[] row1 = new Object[lschema.size()];
+ Object[] row1 = new Object[lschema.length];
for( int i=0; i<rows; i++ ) {
- for( int j=0; j<lschema.size(); j++ )
- data[i][j] = UtilFunctions.objectToDouble(lschema.get(j),
- row1[j] = UtilFunctions.doubleToObject(lschema.get(j), data[i][j]));
+ for( int j=0; j<lschema.length; j++ )
+ data[i][j] = UtilFunctions.objectToDouble(lschema[j],
+ row1[j] = UtilFunctions.doubleToObject(lschema[j], data[i][j]));
frame.appendRow(row1);
}
}
void updateFrameWithDummyData(FrameBlock frame, int updateRow)
{
- List<ValueType>lschema = frame.getSchema();
- for( int j=0; j<lschema.size(); j++ ) {
- switch( lschema.get(j) ) {
+ ValueType[] lschema = frame.getSchema();
+ for( int j=0; j<lschema.length; j++ ) {
+ switch( lschema[j] ) {
case STRING: frame.set(updateRow, j, "String:"+ frame.get(updateRow, j)); break;
case BOOLEAN: frame.set(updateRow, j, ((Boolean)frame.get(updateRow, j))?(new Boolean(false)):(new Boolean(true))); break;
case INT: frame.set(updateRow, j, (Long)frame.get(updateRow, j) * 2 + 5); break;
case DOUBLE: frame.set(updateRow, j, (Double)frame.get(updateRow, j) * 2 + 7); break;
- default: throw new RuntimeException("Unsupported value type: "+lschema.get(j));
+ default: throw new RuntimeException("Unsupported value type: "+lschema[j]);
}
}
}
void verifyFrameData(FrameBlock frame1, FrameBlock frame2, int updateRow, boolean bEqual)
{
- List<ValueType>lschema = frame1.getSchema();
- for( int j=0; j<lschema.size(); j++ ) {
+ ValueType[ ]lschema = frame1.getSchema();
+ for( int j=0; j<lschema.length; j++ ) {
if(!bEqual)
{
- if( UtilFunctions.compareTo(lschema.get(j), frame1.get(updateRow, j), frame2.get(updateRow, j)) == 0)
+ if( UtilFunctions.compareTo(lschema[j], frame1.get(updateRow, j), frame2.get(updateRow, j)) == 0)
Assert.fail("Updated value for cell ("+ updateRow + "," + j + ") is " + frame1.get(updateRow, j) +
", same as original value "+frame2.get(updateRow, j));
}
else
{
- if( UtilFunctions.compareTo(lschema.get(j), frame1.get(updateRow, j), frame2.get(updateRow, j)) != 0)
+ if( UtilFunctions.compareTo(lschema[j], frame1.get(updateRow, j), frame2.get(updateRow, j)) != 0)
Assert.fail("Updated value for cell ("+ updateRow + "," + j + ") is " + frame1.get(updateRow, j) +
", not same as original value "+frame2.get(updateRow, j));
}
@@ -171,10 +166,10 @@ public class FrameCopyTest extends AutomatedTestBase
void verifyFrameData(FrameBlock frame1, FrameBlock frame2)
{
- List<ValueType> lschema = frame1.getSchema();
+ ValueType[] lschema = frame1.getSchema();
for ( int i=0; i<frame1.getNumRows(); i++ )
- for( int j=0; j<lschema.size(); j++ ) {
- if( UtilFunctions.compareTo(lschema.get(j), frame1.get(i, j), frame2.get(i, j)) != 0)
+ for( int j=0; j<lschema.length; j++ ) {
+ if( UtilFunctions.compareTo(lschema[j], frame1.get(i, j), frame2.get(i, j)) != 0)
Assert.fail("Target value for cell ("+ i + "," + j + ") is " + frame1.get(i, j) +
", is not same as original value " + frame2.get(i, j));
}
[4/4] incubator-systemml git commit: [SYSTEMML-557] Memory efficiency
frame block (array-based schema/meta)
Posted by mb...@apache.org.
[SYSTEMML-557] Memory efficiency frame block (array-based schema/meta)
This patch reworks the core frame block schema/meta data storage from
lists to arrays. Although the actual size difference is small, it has
large impact for distributed frame representations as RDDs on Spark
because Spark apparently largely overestimates the list-based storage
(for a subset of partitions), which led to unnecessary spilling. We now
consistently use array-based schemas and column names throughout the
entire system, except external APIs such as MLContext and JMLC in order
to stay backwards compatible.
Experiments with the following script for transformencode and input data
of size 1G x 5 with double schema, improved the RDD storage size from
117GB to 37.5GB and performance from 388s to 124s.
Xf = read("mboehm/frame/Xf");
jspec = "{\"ids\": true,\"recode\": [1,2,3,4,5]}";
[X,M] = transformencode(target=Xf, spec=jspec);
print(max(X));
print(toString(M))
Additional merges
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/eb988781
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/eb988781
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/eb988781
Branch: refs/heads/master
Commit: eb988781d8f2f2af20103f4aecbe634b1a19a47e
Parents: 3c3d1a4
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Sep 22 21:43:08 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Sep 22 22:21:10 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/api/jmlc/PreparedScript.java | 5 +-
.../sysml/api/mlcontext/BinaryBlockFrame.java | 3 +-
.../api/mlcontext/MLContextConversionUtil.java | 41 +--
.../org/apache/sysml/hops/OptimizerUtils.java | 5 +-
.../controlprogram/caching/CacheableData.java | 3 +-
.../controlprogram/caching/FrameObject.java | 47 ++-
.../context/ExecutionContext.java | 2 +-
.../context/SparkExecutionContext.java | 7 +-
...ReturnParameterizedBuiltinCPInstruction.java | 3 +-
.../cp/ParameterizedBuiltinCPInstruction.java | 5 +-
.../instructions/cp/VariableCPInstruction.java | 2 +-
.../spark/CSVReblockSPInstruction.java | 4 +-
.../instructions/spark/CastSPInstruction.java | 5 +-
...ReturnParameterizedBuiltinSPInstruction.java | 3 +-
.../ParameterizedBuiltinSPInstruction.java | 9 +-
.../instructions/spark/WriteSPInstruction.java | 5 +-
.../spark/utils/FrameRDDConverterUtils.java | 81 +++---
.../apache/sysml/runtime/io/FrameReader.java | 34 +--
.../runtime/io/FrameReaderBinaryBlock.java | 7 +-
.../sysml/runtime/io/FrameReaderTextCSV.java | 19 +-
.../runtime/io/FrameReaderTextCSVParallel.java | 2 +-
.../sysml/runtime/io/FrameReaderTextCell.java | 25 +-
.../runtime/io/FrameReaderTextCellParallel.java | 2 +-
.../apache/sysml/runtime/io/FrameWriter.java | 3 +-
.../sysml/runtime/io/FrameWriterTextCSV.java | 2 +-
.../sysml/runtime/matrix/data/FrameBlock.java | 284 ++++++++++---------
.../matrix/data/OperationsOnMatrixValues.java | 16 +-
.../matrix/mapred/FrameReblockBuffer.java | 7 +-
.../sysml/runtime/transform/BinAgent.java | 8 +-
.../sysml/runtime/transform/DummycodeAgent.java | 6 +-
.../sysml/runtime/transform/MVImputeAgent.java | 6 +-
.../sysml/runtime/transform/OmitAgent.java | 3 +-
.../sysml/runtime/transform/RecodeAgent.java | 2 +-
.../apache/sysml/runtime/transform/TfUtils.java | 13 +-
.../transform/decode/DecoderDummycode.java | 6 +-
.../transform/decode/DecoderFactory.java | 2 +-
.../transform/decode/DecoderPassThrough.java | 4 +-
.../runtime/transform/decode/DecoderRecode.java | 2 +-
.../transform/encode/EncoderFactory.java | 21 +-
.../transform/encode/EncoderPassThrough.java | 2 +-
.../runtime/transform/meta/TfMetaUtils.java | 43 +--
.../sysml/runtime/util/DataConverter.java | 33 ++-
.../sysml/runtime/util/MapReduceTool.java | 13 +-
.../sysml/runtime/util/UtilFunctions.java | 55 ++--
.../test/integration/AutomatedTestBase.java | 9 +-
.../functions/frame/FrameAppendDistTest.java | 10 +-
.../functions/frame/FrameAppendTest.java | 31 +-
.../functions/frame/FrameCastingTest.java | 20 +-
.../functions/frame/FrameConverterTest.java | 37 ++-
.../functions/frame/FrameCopyTest.java | 45 ++-
.../functions/frame/FrameGetSetTest.java | 32 +--
.../functions/frame/FrameIndexingDistTest.java | 12 +-
.../functions/frame/FrameIndexingTest.java | 33 +--
.../functions/frame/FrameReadWriteTest.java | 28 +-
.../functions/frame/FrameScalarCastingTest.java | 5 +-
.../functions/frame/FrameSchemaReadTest.java | 33 +--
.../functions/frame/FrameSerializationTest.java | 17 +-
.../mlcontext/DataFrameFrameConversionTest.java | 6 +-
.../functions/mlcontext/FrameTest.java | 12 +-
.../mlcontext/MLContextFrameTest.java | 14 +-
.../org/apache/sysml/test/utils/TestUtils.java | 19 +-
61 files changed, 588 insertions(+), 625 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index e06810b..4fcef44 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -310,7 +310,7 @@ public class PreparedScript
* @throws DMLException
*/
public void setFrame(String varname, String[][] frame, List<ValueType> schema, boolean reuse) throws DMLException {
- setFrame(varname, DataConverter.convertToFrameBlock(frame, schema), reuse);
+ setFrame(varname, DataConverter.convertToFrameBlock(frame, schema.toArray(new ValueType[0])), reuse);
}
/**
@@ -324,7 +324,8 @@ public class PreparedScript
* @throws DMLException
*/
public void setFrame(String varname, String[][] frame, List<ValueType> schema, List<String> colnames, boolean reuse) throws DMLException {
- setFrame(varname, DataConverter.convertToFrameBlock(frame, schema, colnames), reuse);
+ setFrame(varname, DataConverter.convertToFrameBlock( frame,
+ schema.toArray(new ValueType[0]), colnames.toArray(new String[0])), reuse);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
index 9d7ab9d..5812cf6 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
@@ -22,6 +22,7 @@ package org.apache.sysml.api.mlcontext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.sql.DataFrame;
import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -122,7 +123,7 @@ public class BinaryBlockFrame {
try {
MatrixCharacteristics mc = getMatrixCharacteristics();
FrameSchema frameSchema = frameMetadata.getFrameSchema();
- return SparkExecutionContext.toFrameBlock(binaryBlocks, frameSchema.getSchema(),
+ return SparkExecutionContext.toFrameBlock(binaryBlocks, frameSchema.getSchema().toArray(new ValueType[0]),
(int) mc.getRows(), (int) mc.getCols());
} catch (DMLRuntimeException e) {
throw new MLContextException("Exception while getting FrameBlock from binary-block frame", e);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
index aa0366d..ca9cc2a 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
@@ -22,7 +22,7 @@ package org.apache.sysml.api.mlcontext;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@@ -63,6 +63,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.UtilFunctions;
import scala.collection.JavaConversions;
import scala.reflect.ClassTag;
@@ -193,7 +194,8 @@ public class MLContextConversionUtil {
frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
MatrixFormatMetaData mtd = new MatrixFormatMetaData(mc,
OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd, frameMetadata.getFrameSchema().getSchema());
+ FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
frameObject.acquireModify(frameBlock);
frameObject.release();
return frameObject;
@@ -282,7 +284,8 @@ public class MLContextConversionUtil {
frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema());
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
frameObject.setRDDHandle(new RDDObject(binaryBlocks, variableName));
return frameObject;
}
@@ -357,22 +360,22 @@ public class MLContextConversionUtil {
JavaSparkContext javaSparkContext = MLContextUtil
.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
boolean containsID = isDataFrameWithIDColumn(frameMetadata);
- MatrixCharacteristics matrixCharacteristics = frameMetadata.asMatrixCharacteristics();
- if (matrixCharacteristics == null) {
- matrixCharacteristics = new MatrixCharacteristics();
+ MatrixCharacteristics mc = frameMetadata.asMatrixCharacteristics();
+ if (mc == null) {
+ mc = new MatrixCharacteristics();
long rows = dataFrame.count();
- int cols = dataFrame.columns().length;
- matrixCharacteristics.setDimension(rows, cols);
- frameMetadata.setMatrixCharacteristics(matrixCharacteristics);
+ int cols = dataFrame.columns().length - ((containsID)?1:0);
+ mc.setDimension(rows, cols);
+ frameMetadata.setMatrixCharacteristics(mc);
}
- List<String> colnames = new ArrayList<String>();
- List<ValueType> fschema = new ArrayList<ValueType>();
+ String[] colnames = new String[(int)mc.getCols()];
+ ValueType[] fschema = new ValueType[(int)mc.getCols()];
FrameRDDConverterUtils.convertDFSchemaToFrameSchema(dataFrame.schema(), colnames, fschema, containsID);
- frameMetadata.setFrameSchema(new FrameSchema(fschema));
+ frameMetadata.setFrameSchema(new FrameSchema(Arrays.asList(fschema)));
JavaPairRDD<Long, FrameBlock> binaryBlock = FrameRDDConverterUtils.dataFrameToBinaryBlock(javaSparkContext,
- dataFrame, matrixCharacteristics, containsID);
+ dataFrame, mc, containsID);
return MLContextConversionUtil.binaryBlocksToFrameObject(variableName, binaryBlock, frameMetadata);
} catch (DMLRuntimeException e) {
@@ -604,7 +607,8 @@ public class MLContextConversionUtil {
JavaSparkContext jsc = MLContextUtil.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema());
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
rdd = FrameRDDConverterUtils.csvToBinaryBlock(jsc, javaPairRDDText, mc,
@@ -665,12 +669,13 @@ public class MLContextConversionUtil {
JavaSparkContext jsc = MLContextUtil.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema());
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
- List<ValueType> lschema = null;
+ ValueType[] lschema = null;
if (lschema == null)
- lschema = Collections.nCopies((int) mc.getCols(), ValueType.STRING);
+ lschema = UtilFunctions.nCopies((int) mc.getCols(), ValueType.STRING);
rdd = FrameRDDConverterUtils.textCellToBinaryBlock(jsc, javaPairRDDText, mc, lschema);
} catch (DMLRuntimeException e) {
e.printStackTrace();
@@ -1272,7 +1277,7 @@ public class MLContextConversionUtil {
JavaPairRDD<Long, FrameBlock> binaryBlock = (JavaPairRDD<Long, FrameBlock>) sparkExecutionContext
.getRDDHandleForFrameObject(frameObject, InputInfo.BinaryBlockInputInfo);
MatrixCharacteristics matrixCharacteristics = frameObject.getMatrixCharacteristics();
- FrameSchema fs = new FrameSchema(frameObject.getSchema());
+ FrameSchema fs = new FrameSchema(Arrays.asList(frameObject.getSchema()));
FrameMetadata fm = new FrameMetadata(fs, matrixCharacteristics);
return new BinaryBlockFrame(binaryBlock, fm);
} catch (DMLRuntimeException e) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index b0a938d..8b07f0a 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -20,7 +20,6 @@
package org.apache.sysml.hops;
import java.util.HashMap;
-import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -899,10 +898,10 @@ public class OptimizerUtils
* @param names
* @return
*/
- public static boolean isValidCPDimensions( List<ValueType> schema, List<String> names )
+ public static boolean isValidCPDimensions( ValueType[] schema, String[] names )
{
// Length of schema and names to be same, and > 0.
- return (schema != null && names != null && schema.size() > 0 && schema.size() == names.size());
+ return (schema != null && names != null && schema.length > 0 && schema.length == names.length);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 3e2f6c1..d3addd6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -22,7 +22,6 @@ package org.apache.sysml.runtime.controlprogram.caching;
import java.io.File;
import java.io.IOException;
import java.lang.ref.SoftReference;
-import java.util.List;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.logging.Log;
@@ -973,7 +972,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
return (_data.getInMemorySize() <= CACHING_THRESHOLD);
}
- protected List<ValueType> getSchema() {
+ protected ValueType[] getSchema() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 8c5fe8b..effb604 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -21,10 +21,9 @@ package org.apache.sysml.runtime.controlprogram.caching;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.Arrays;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
@@ -44,12 +43,13 @@ import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class FrameObject extends CacheableData<FrameBlock>
{
private static final long serialVersionUID = 1755082174281927785L;
- private List<ValueType> _schema = null;
+ private ValueType[] _schema = null;
/**
*
@@ -85,7 +85,7 @@ public class FrameObject extends CacheableData<FrameBlock>
* @param schema
*
*/
- public FrameObject(String fname, MetaData meta, List<ValueType> schema) {
+ public FrameObject(String fname, MetaData meta, ValueType[] schema) {
this();
setFileName(fname);
setMetaData(meta);
@@ -102,7 +102,7 @@ public class FrameObject extends CacheableData<FrameBlock>
}
@Override
- public List<ValueType> getSchema() {
+ public ValueType[] getSchema() {
return _schema;
}
@@ -112,9 +112,9 @@ public class FrameObject extends CacheableData<FrameBlock>
* @param cu column upper bound, inclusive
* @return
*/
- public List<ValueType> getSchema(int cl, int cu) {
- return (_schema!=null && _schema.size()>cu) ? _schema.subList(cl, cu+1) :
- Collections.nCopies(cu-cl+1, ValueType.STRING);
+ public ValueType[] getSchema(int cl, int cu) {
+ return (_schema!=null && _schema.length>cu) ? Arrays.copyOfRange(_schema, cl, cu+1) :
+ UtilFunctions.nCopies(cu-cl+1, ValueType.STRING);
}
/**
@@ -124,13 +124,10 @@ public class FrameObject extends CacheableData<FrameBlock>
* @param fo
* @return
*/
- public List<ValueType> mergeSchemas(FrameObject fo) {
- ArrayList<ValueType> ret = new ArrayList<ValueType>();
- ret.addAll((_schema!=null) ? _schema :
- Collections.nCopies((int)getNumColumns(), ValueType.STRING));
- ret.addAll((fo.getSchema()!=null) ? fo.getSchema() :
- Collections.nCopies((int)fo.getNumColumns(), ValueType.STRING));
- return ret;
+ public ValueType[] mergeSchemas(FrameObject fo) {
+ return (ValueType[]) ArrayUtils.addAll(
+ (_schema!=null) ? _schema : UtilFunctions.nCopies((int)getNumColumns(), ValueType.STRING),
+ (fo._schema!=null) ? fo._schema : UtilFunctions.nCopies((int)fo.getNumColumns(), ValueType.STRING));
}
public void setSchema(String schema) {
@@ -138,18 +135,18 @@ public class FrameObject extends CacheableData<FrameBlock>
//populate default schema
int clen = (int) getNumColumns();
if( clen > 0 ) //known number of cols
- _schema = Collections.nCopies(clen, ValueType.STRING);
+ _schema = UtilFunctions.nCopies(clen, ValueType.STRING);
}
else {
//parse given schema
- _schema = new ArrayList<ValueType>();
String[] parts = schema.split(DataExpression.DEFAULT_DELIM_DELIMITER);
- for( String svt : parts )
- _schema.add(ValueType.valueOf(svt.toUpperCase()));
+ _schema = new ValueType[parts.length];
+ for( int i=0; i<parts.length; i++ )
+ _schema[i] = ValueType.valueOf(parts[i].toUpperCase());
}
}
- public void setSchema(List<ValueType> schema) {
+ public void setSchema(ValueType[] schema) {
_schema = schema;
}
@@ -200,8 +197,8 @@ public class FrameObject extends CacheableData<FrameBlock>
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
//handle missing schema if necessary
- List<ValueType> lschema = (_schema!=null) ? _schema :
- Collections.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
+ ValueType[] lschema = (_schema!=null) ? _schema :
+ UtilFunctions.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
//read the frame block
FrameBlock data = null;
@@ -237,8 +234,8 @@ public class FrameObject extends CacheableData<FrameBlock>
int clen = (int)mc.getCols();
//handle missing schema if necessary
- List<ValueType> lschema = (_schema!=null) ? _schema :
- Collections.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
+ ValueType[] lschema = (_schema!=null) ? _schema :
+ UtilFunctions.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
FrameBlock fb = null;
try {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 14b9c44..b6011ae 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -458,7 +458,7 @@ public class ExecutionContext
{
FrameObject fo = getFrameObject(varName);
if( outputData.getNumColumns()>0 && outputData.getSchema()!=null )
- fo.setValueType(outputData.getSchema().get(0));
+ fo.setValueType(outputData.getSchema()[0]);
fo.acquireModify(outputData);
fo.release();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index a6f99e8..964d2d6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.controlprogram.context;
import java.io.IOException;
-import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -958,7 +957,7 @@ public class SparkExecutionContext extends ExecutionContext
* @throws DMLRuntimeException
*/
@SuppressWarnings("unchecked")
- public static FrameBlock toFrameBlock(RDDObject rdd, List<ValueType> schema, int rlen, int clen)
+ public static FrameBlock toFrameBlock(RDDObject rdd, ValueType[] schema, int rlen, int clen)
throws DMLRuntimeException
{
JavaPairRDD<Long,FrameBlock> lrdd = (JavaPairRDD<Long,FrameBlock>) rdd.getRDD();
@@ -974,13 +973,13 @@ public class SparkExecutionContext extends ExecutionContext
* @return
* @throws DMLRuntimeException
*/
- public static FrameBlock toFrameBlock(JavaPairRDD<Long,FrameBlock> rdd, List<ValueType> schema, int rlen, int clen)
+ public static FrameBlock toFrameBlock(JavaPairRDD<Long,FrameBlock> rdd, ValueType[] schema, int rlen, int clen)
throws DMLRuntimeException
{
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
if(schema == null)
- schema = Collections.nCopies(clen, ValueType.STRING);
+ schema = UtilFunctions.nCopies(clen, ValueType.STRING);
//create output frame block (w/ lazy allocation)
FrameBlock out = new FrameBlock(schema);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
index 05cf19e..5af403d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.instructions.cp;
import java.util.ArrayList;
-import java.util.List;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -82,7 +81,7 @@ public class MultiReturnParameterizedBuiltinCPInstruction extends ComputationCPI
//obtain and pin input frame
FrameBlock fin = ec.getFrameInput(input1.getName());
String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
- List<String> colnames = fin.getColumnNames();
+ String[] colnames = fin.getColumnNames();
//execute block transform encode
Encoder encoder = EncoderFactory.createEncoder(spec, colnames, fin.getNumColumns(), null);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index a245d10..299fb3c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.instructions.cp;
import java.util.HashMap;
-import java.util.List;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
@@ -270,7 +269,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
//acquire locks
FrameBlock data = ec.getFrameInput(params.get("target"));
FrameBlock meta = ec.getFrameInput(params.get("meta"));
- List<String> colNames = data.getColumnNames();
+ String[] colNames = data.getColumnNames();
//compute transformapply
Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, data.getNumColumns(), meta);
@@ -285,7 +284,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
//acquire locks
MatrixBlock data = ec.getMatrixInput(params.get("target"));
FrameBlock meta = ec.getFrameInput(params.get("meta"));
- List<String> colnames = meta.getColumnNames();
+ String[] colnames = meta.getColumnNames();
//compute transformdecode
Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index ee86b59..9c45366 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -536,7 +536,7 @@ public class VariableCPInstruction extends CPInstruction
Object value = fBlock.get(0,0);
ec.releaseFrameInput(input1.getName());
ec.setScalarOutput(output.getName(),
- ScalarObjectFactory.createScalarObject(fBlock.getSchema().get(0), value));
+ ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value));
}
else { //assume DataType.MATRIX otherwise
MatrixBlock mBlock = ec.getMatrixInput(input1.getName());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index e11f505..0e87be5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -19,8 +19,6 @@
package org.apache.sysml.runtime.instructions.spark;
-import java.util.List;
-
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
@@ -163,7 +161,7 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
* @throws DMLRuntimeException
*/
@SuppressWarnings("unchecked")
- protected JavaPairRDD<Long,FrameBlock> processFrameCSVReblockInstruction(SparkExecutionContext sec, MatrixCharacteristics mcOut, List<ValueType> schema)
+ protected JavaPairRDD<Long,FrameBlock> processFrameCSVReblockInstruction(SparkExecutionContext sec, MatrixCharacteristics mcOut, ValueType[] schema)
throws DMLRuntimeException
{
//get input rdd (needs to be longwritable/text for consistency with meta data, in case of
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
index 4487b20..4fdd617 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
@@ -19,8 +19,6 @@
package org.apache.sysml.runtime.instructions.spark;
-import java.util.Collections;
-
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.UnaryCP;
@@ -37,6 +35,7 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class CastSPInstruction extends UnarySPInstruction
@@ -95,7 +94,7 @@ public class CastSPInstruction extends UnarySPInstruction
//update schema information for output frame
if( opcode.equals(UnaryCP.CAST_AS_FRAME_OPCODE) ) {
sec.getFrameObject(output.getName()).setSchema(
- Collections.nCopies((int)mcIn.getCols(), ValueType.DOUBLE));
+ UtilFunctions.nCopies((int)mcIn.getCols(), ValueType.DOUBLE));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
index 9b35daa..b36b1f3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
@@ -24,7 +24,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map.Entry;
import org.apache.spark.Accumulator;
@@ -127,7 +126,7 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI
String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
- List<String> colnames = !TfMetaUtils.isIDSpecification(spec) ?
+ String[] colnames = !TfMetaUtils.isIDSpecification(spec) ?
in.lookup(1L).get(0).getColumnNames() : null;
//step 1: build transform meta data
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
index f880f57..adbc06a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
@@ -429,7 +428,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
FrameBlock meta = sec.getFrameInput(params.get("meta"));
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(params.get("target"));
MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
- List<String> colnames = !TfMetaUtils.isIDSpecification(params.get("spec")) ?
+ String[] colnames = !TfMetaUtils.isIDSpecification(params.get("spec")) ?
in.lookup(1L).get(0).getColumnNames() : null;
//compute omit offset map for block shifts
@@ -463,7 +462,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(params.get("target"));
MatrixCharacteristics mc = sec.getMatrixCharacteristics(params.get("target"));
FrameBlock meta = sec.getFrameInput(params.get("meta"));
- List<String> colnames = meta.getColumnNames();
+ String[] colnames = meta.getColumnNames();
//reblock if necessary (clen > bclen)
if( mc.getCols() > mc.getNumColBlocks() ) {
@@ -773,7 +772,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
private int[] _omitColList = null;
- public RDDTransformApplyOffsetFunction(String spec, List<String> colnames) {
+ public RDDTransformApplyOffsetFunction(String spec, String[] colnames) {
try {
_omitColList = TfMetaUtils.parseJsonIDList(spec, colnames, TfUtils.TXMETHOD_OMIT);
}
@@ -796,7 +795,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
for( int j=0; j<_omitColList.length; j++ ) {
int colID = _omitColList[j];
Object val = blk.get(i, colID-1);
- valid &= !(val==null || (blk.getSchema().get(colID-1)==
+ valid &= !(val==null || (blk.getSchema()[colID-1]==
ValueType.STRING && val.toString().isEmpty()));
}
rmRows += valid ? 0 : 1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
index 1b974f9..584a4e9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.instructions.spark;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
import org.apache.hadoop.io.LongWritable;
@@ -137,7 +136,7 @@ public class WriteSPInstruction extends SPInstruction
//get filename (literal or variable expression)
String fname = ec.getScalarInput(input2.getName(), ValueType.STRING, input2.isLiteral()).getStringValue();
- List<ValueType> schema = (input1.getDataType()==DataType.FRAME) ?
+ ValueType[] schema = (input1.getDataType()==DataType.FRAME) ?
sec.getFrameObject(input1.getName()).getSchema() : null;
try
@@ -282,7 +281,7 @@ public class WriteSPInstruction extends SPInstruction
* @throws IOException
*/
@SuppressWarnings("unchecked")
- protected void processFrameWriteInstruction(SparkExecutionContext sec, String fname, OutputInfo oi, List<ValueType> schema)
+ protected void processFrameWriteInstruction(SparkExecutionContext sec, String fname, OutputInfo oi, ValueType[] schema)
throws DMLRuntimeException, IOException
{
//get input rdd
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index 3f6cc4c..f8fb05c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -22,7 +22,6 @@ package org.apache.sysml.runtime.instructions.spark.utils;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Arrays;
@@ -92,7 +91,7 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> csvToBinaryBlock(JavaSparkContext sc,
- JavaPairRDD<LongWritable, Text> input, MatrixCharacteristics mc, List<ValueType> schema,
+ JavaPairRDD<LongWritable, Text> input, MatrixCharacteristics mc, ValueType[] schema,
boolean hasHeader, String delim, boolean fill, double fillValue)
throws DMLRuntimeException
{
@@ -114,11 +113,9 @@ public class FrameRDDConverterUtils
.zipWithIndex(); //zip row index
//prepare default schema if needed
- if( schema == null || schema.size()==1 ) {
- schema = Collections.nCopies((int)mc.getCols(),
- (schema!=null) ? schema.get(0) : ValueType.STRING);
- }
-
+ if( schema == null || schema.length==1 )
+ schema = UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING);
+
//convert csv rdd to binary block rdd (w/ partial blocks)
JavaPairRDD<Long, FrameBlock> out = prepinput.mapPartitionsToPair(
new CSVToBinaryBlockFunction(mc, schema, hasHeader, delim));
@@ -140,7 +137,7 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> csvToBinaryBlock(JavaSparkContext sc,
- JavaRDD<String> input, MatrixCharacteristics mcOut, List<ValueType> schema,
+ JavaRDD<String> input, MatrixCharacteristics mcOut, ValueType[] schema,
boolean hasHeader, String delim, boolean fill, double fillValue)
throws DMLRuntimeException
{
@@ -189,7 +186,7 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> textCellToBinaryBlock(JavaSparkContext sc,
- JavaPairRDD<LongWritable, Text> in, MatrixCharacteristics mcOut, List<ValueType> schema )
+ JavaPairRDD<LongWritable, Text> in, MatrixCharacteristics mcOut, ValueType[] schema )
throws DMLRuntimeException
{
//convert input rdd to serializable long/frame block
@@ -210,13 +207,13 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> textCellToBinaryBlockLongIndex(JavaSparkContext sc,
- JavaPairRDD<Long, Text> input, MatrixCharacteristics mc, List<ValueType> schema )
+ JavaPairRDD<Long, Text> input, MatrixCharacteristics mc, ValueType[] schema )
throws DMLRuntimeException
{
//prepare default schema if needed
- if( schema == null || schema.size()==1 ) {
- schema = Collections.nCopies((int)mc.getCols(),
- (schema!=null) ? schema.get(0) : ValueType.STRING);
+ if( schema == null || schema.length==1 ) {
+ schema = UtilFunctions.nCopies((int)mc.getCols(),
+ (schema!=null) ? schema[0] : ValueType.STRING);
}
//convert textcell rdd to binary block rdd (w/ partial blocks)
@@ -342,8 +339,8 @@ public class FrameRDDConverterUtils
df.javaRDD().zipWithIndex(); //zip row index
//convert data frame to frame schema (prepare once)
- List<String> colnames = new ArrayList<String>();
- List<ValueType> fschema = new ArrayList<ValueType>();
+ String[] colnames = new String[(int)mc.getCols()];
+ ValueType[] fschema = new ValueType[(int)mc.getCols()];
convertDFSchemaToFrameSchema(df.schema(), colnames, fschema, containsID);
//convert rdd to binary block rdd
@@ -360,7 +357,7 @@ public class FrameRDDConverterUtils
* @return
*/
public static DataFrame binaryBlockToDataFrame(SQLContext sqlctx, JavaPairRDD<Long,FrameBlock> in,
- MatrixCharacteristics mc, List<ValueType> schema)
+ MatrixCharacteristics mc, ValueType[] schema)
{
if( !mc.colsKnown() )
throw new RuntimeException("Number of columns needed to convert binary block to data frame.");
@@ -371,7 +368,7 @@ public class FrameRDDConverterUtils
//create data frame schema
if( schema == null )
- schema = Collections.nCopies((int)mc.getCols(), ValueType.STRING);
+ schema = UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING);
StructType dfSchema = convertFrameSchemaToDFSchema(schema, true);
//rdd to data frame conversion
@@ -387,7 +384,7 @@ public class FrameRDDConverterUtils
* @return
* Returns the DataFrame schema (StructType)
*/
- public static StructType convertFrameSchemaToDFSchema(List<ValueType> fschema, boolean containsID)
+ public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID)
{
// generate the schema based on the string of schema
List<StructField> fields = new ArrayList<StructField>();
@@ -421,30 +418,30 @@ public class FrameRDDConverterUtils
* @param containsID
* @return
*/
- public static void convertDFSchemaToFrameSchema(StructType dfschema, List<String> colnames,
- List<ValueType> fschema, boolean containsID)
+ public static void convertDFSchemaToFrameSchema(StructType dfschema, String[] colnames,
+ ValueType[] fschema, boolean containsID)
{
int off = containsID ? 1 : 0;
for( int i=off; i<dfschema.fields().length; i++ ) {
StructField structType = dfschema.apply(i);
- colnames.add(structType.name());
+ colnames[i-off] = structType.name();
if(structType.dataType() == DataTypes.DoubleType
|| structType.dataType() == DataTypes.FloatType)
- fschema.add(ValueType.DOUBLE);
+ fschema[i-off] = ValueType.DOUBLE;
else if(structType.dataType() == DataTypes.LongType
|| structType.dataType() == DataTypes.IntegerType)
- fschema.add(ValueType.INT);
+ fschema[i-off] = ValueType.INT;
else if(structType.dataType() == DataTypes.BooleanType)
- fschema.add(ValueType.BOOLEAN);
+ fschema[i-off] = ValueType.BOOLEAN;
else
- fschema.add(ValueType.STRING);
+ fschema[i-off] = ValueType.STRING;
}
}
/*
* It will return JavaRDD<Row> based on csv data input file.
*/
- public static JavaRDD<Row> csvToRowRDD(JavaSparkContext sc, String fnameIn, String delim, List<ValueType> schema)
+ public static JavaRDD<Row> csvToRowRDD(JavaSparkContext sc, String fnameIn, String delim, ValueType[] schema)
{
// Load a text file and convert each line to a java rdd.
JavaRDD<String> dataRdd = sc.textFile(fnameIn);
@@ -458,10 +455,10 @@ public class FrameRDDConverterUtils
{
private static final long serialVersionUID = -6736256507697511070L;
- private List<ValueType> _schema = null;
+ private ValueType[] _schema = null;
private String _delim = null;
- public RowGenerator(List<ValueType> schema, String delim) {
+ public RowGenerator(ValueType[] schema, String delim) {
_schema = schema;
_delim = delim;
}
@@ -471,7 +468,7 @@ public class FrameRDDConverterUtils
String[] fields = IOUtilFunctions.splitCSV(record, _delim);
Object[] objects = new Object[fields.length];
for (int i=0; i<fields.length; i++) {
- objects[i] = UtilFunctions.stringToObject(_schema.get(i), fields[i]);
+ objects[i] = UtilFunctions.stringToObject(_schema[i], fields[i]);
}
return RowFactory.create(objects);
}
@@ -617,12 +614,12 @@ public class FrameRDDConverterUtils
private boolean _hasHeader = false;
private String _delim = null;
private int _maxRowsPerBlock = -1;
- private List<ValueType> _schema = null;
- private List<String> _colnames = null;
+ private ValueType[] _schema = null;
+ private String[] _colnames = null;
private List<String> _mvMeta = null; //missing value meta data
private List<String> _ndMeta = null; //num distinct meta data
- public CSVToBinaryBlockFunction(MatrixCharacteristics mc, List<ValueType> schema, boolean hasHeader, String delim) {
+ public CSVToBinaryBlockFunction(MatrixCharacteristics mc, ValueType[] schema, boolean hasHeader, String delim) {
_clen = mc.getCols();
_schema = schema;
_hasHeader = hasHeader;
@@ -646,7 +643,7 @@ public class FrameRDDConverterUtils
String row = tmp._1().toString().trim();
long rowix = tmp._2();
if(_hasHeader && rowix == 0) { //Skip header
- _colnames = Arrays.asList(IOUtilFunctions.splitCSV(row, _delim));
+ _colnames = IOUtilFunctions.splitCSV(row, _delim);
continue;
}
if( row.startsWith(TfUtils.TXMTD_MVPREFIX) ) {
@@ -744,7 +741,7 @@ public class FrameRDDConverterUtils
if( ix==1 ) {
if( _props.hasHeader() ) {
for(int j = 1; j <= blk.getNumColumns(); j++) {
- sb.append(blk.getColumnNames().get(j)
+ sb.append(blk.getColumnNames()[j]
+ ((j<blk.getNumColumns()-1)?_props.getDelim():""));
}
ret.add(sb.toString());
@@ -790,13 +787,13 @@ public class FrameRDDConverterUtils
private static final long serialVersionUID = 2269315691094111843L;
private long _clen = -1;
- private List<String> _colnames = null;
- private List<ValueType> _schema = null;
+ private String[] _colnames = null;
+ private ValueType[] _schema = null;
private boolean _containsID = false;
private int _maxRowsPerBlock = -1;
- public DataFrameToBinaryBlockFunction(MatrixCharacteristics mc, List<String> colnames,
- List<ValueType> schema, boolean containsID) {
+ public DataFrameToBinaryBlockFunction(MatrixCharacteristics mc, String[] colnames,
+ ValueType[] schema, boolean containsID) {
_clen = mc.getCols();
_colnames = colnames;
_schema = schema;
@@ -831,7 +828,7 @@ public class FrameRDDConverterUtils
int off = _containsID ? 1 : 0;
for(int i=off; i<row.size(); i++) {
tmprow[i-off] = UtilFunctions.objectToObject(
- _schema.get(i-off), row.get(i));
+ _schema[i-off], row.get(i));
}
fb.appendRow(tmprow);
}
@@ -932,9 +929,9 @@ public class FrameRDDConverterUtils
private static class TextToBinaryBlockFunction extends CellToBinaryBlockFunction implements PairFlatMapFunction<Iterator<Text>,Long,FrameBlock>
{
private static final long serialVersionUID = -2042208027876880588L;
- List<ValueType> _schema = null;
+ ValueType[] _schema = null;
- protected TextToBinaryBlockFunction(MatrixCharacteristics mc, List<ValueType> schema ) {
+ protected TextToBinaryBlockFunction(MatrixCharacteristics mc, ValueType[] schema ) {
super(mc);
_schema = schema;
}
@@ -958,7 +955,7 @@ public class FrameRDDConverterUtils
st.reset( strVal );
long row = st.nextLong();
long col = st.nextLong();
- Object val = UtilFunctions.stringToObject(_schema.get((int)col-1), st.nextToken());
+ Object val = UtilFunctions.stringToObject(_schema[(int)col-1], st.nextToken());
//flush buffer if necessary
if( rbuff.getSize() >= rbuff.getCapacity() )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
index e318fff..1dc39e5 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
@@ -21,10 +21,7 @@ package org.apache.sysml.runtime.io;
import java.io.EOFException;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.LinkedList;
-import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -34,6 +31,7 @@ import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* Base class for all format-specific frame readers. Every reader is required to implement the basic
@@ -53,7 +51,7 @@ public abstract class FrameReader
* @param clen
* @return
*/
- public abstract FrameBlock readFrameFromHDFS( String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public abstract FrameBlock readFrameFromHDFS( String fname, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException;
/**
@@ -64,10 +62,10 @@ public abstract class FrameReader
* @param clen
* @return
*/
- public FrameBlock readFrameFromHDFS( String fname, List<ValueType> schema, long rlen, long clen )
+ public FrameBlock readFrameFromHDFS( String fname, ValueType[] schema, long rlen, long clen )
throws IOException, DMLRuntimeException
{
- return readFrameFromHDFS(fname, schema, getDefColNames(schema.size()), rlen, clen);
+ return readFrameFromHDFS(fname, schema, getDefColNames(schema.length), rlen, clen);
}
/**
@@ -88,11 +86,11 @@ public abstract class FrameReader
* @param iNumColumns
* @return
*/
- public List<ValueType> getDefSchema( long clen )
+ public ValueType[] getDefSchema( long clen )
throws IOException, DMLRuntimeException
{
int lclen = Math.max((int)clen, 1);
- return Collections.nCopies(lclen, ValueType.STRING);
+ return UtilFunctions.nCopies(lclen, ValueType.STRING);
}
/**
@@ -100,13 +98,11 @@ public abstract class FrameReader
* @param iNumColumns
* @return
*/
- public List<String> getDefColNames( long clen )
+ public String[] getDefColNames( long clen )
throws IOException, DMLRuntimeException
{
- List<String> colNames = new ArrayList<String>();
- for (int i=0; i < clen; i++)
- colNames.add("C"+i);
- return colNames;
+ return (clen < 0) ? new String[0] :
+ FrameBlock.createColNames((int)clen);
}
/**
@@ -148,7 +144,7 @@ public abstract class FrameReader
* @throws DMLRuntimeException
* @throws IOException
*/
- protected static FrameBlock createOutputFrameBlock(List<ValueType> schema, List<String> names, long nrow)
+ protected static FrameBlock createOutputFrameBlock(ValueType[] schema, String[] names, long nrow)
throws IOException, DMLRuntimeException
{
//check schema and column names
@@ -167,9 +163,9 @@ public abstract class FrameReader
* @param ncol
* @return
*/
- protected static List<ValueType> createOutputSchema(List<ValueType> schema, long ncol) {
- if( schema.size()==1 && ncol > 1 )
- return Collections.nCopies((int)ncol, schema.get(0));
+ protected static ValueType[] createOutputSchema(ValueType[] schema, long ncol) {
+ if( schema.length==1 && ncol > 1 )
+ return UtilFunctions.nCopies((int)ncol, schema[0]);
return schema;
}
@@ -179,8 +175,8 @@ public abstract class FrameReader
* @param ncol
* @return
*/
- protected static List<String> createOutputNames(List<String> names, long ncol) {
- if( names.size() != ncol )
+ protected static String[] createOutputNames(String[] names, long ncol) {
+ if( names.length != ncol )
return FrameBlock.createColNames((int)ncol);
return names;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
index 88e3d96..c720fa0 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -51,12 +50,12 @@ public class FrameReaderBinaryBlock extends FrameReader
* @throws IOException
*/
@Override
- public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//prepare file access
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
index b608991..cfd8b7e 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
@@ -20,8 +20,6 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -67,7 +65,7 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
@Override
- public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+ public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names,
long rlen, long clen)
throws IOException, DMLRuntimeException
{
@@ -88,8 +86,8 @@ public class FrameReaderTextCSV extends FrameReader
}
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//core read (sequential/parallel)
@@ -112,7 +110,7 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
- FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
TextInputFormat informat = new TextInputFormat();
@@ -140,7 +138,7 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
protected final void readCSVFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job,
- FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen, int rl, boolean first)
+ FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen, int rl, boolean first)
throws IOException
{
boolean hasHeader = _props.hasHeader();
@@ -159,8 +157,7 @@ public class FrameReaderTextCSV extends FrameReader
//handle header if existing
if(first && hasHeader ) {
reader.next(key, value); //read header
- List<String> colnames = Arrays.asList(value.toString().split(delim));
- dest.setColumnNames(colnames);
+ dest.setColumnNames(value.toString().split(delim));
}
// Read the data
@@ -189,11 +186,11 @@ public class FrameReaderTextCSV extends FrameReader
part = part.trim();
if ( part.isEmpty() ) {
if( isFill && dfillValue!=0 )
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), sfillValue));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], sfillValue));
emptyValuesFound = true;
}
else {
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), part));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], part));
}
col++;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
index 2713231..84c0721 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
@@ -68,7 +68,7 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV
*/
@Override
protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
- FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
int numThreads = OptimizerUtils.getParallelTextReadParallelism();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
index e7efc2b..48a87fa 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
@@ -23,7 +23,6 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -60,12 +59,12 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
@Override
- public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//prepare file access
@@ -107,12 +106,12 @@ public class FrameReaderTextCell extends FrameReader
* @throws DMLRuntimeException
* @throws IOException
*/
- public final FrameBlock readFrameFromInputStream(InputStream is, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromInputStream(InputStream is, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//core read
@@ -134,7 +133,7 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen)
+ ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
if( fs.isDirectory(path) ) {
@@ -163,7 +162,7 @@ public class FrameReaderTextCell extends FrameReader
protected final void readTextCellFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest)
throws IOException
{
- List<ValueType> schema = dest.getSchema();
+ ValueType[] schema = dest.getSchema();
int rlen = dest.getNumRows();
int clen = dest.getNumColumns();
@@ -187,7 +186,7 @@ public class FrameReaderTextCell extends FrameReader
else if( row == -2 )
dest.getColumnMetadata(col).setNumDistinct(st.nextLong());
else
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], st.nextToken()));
}
}
catch(Exception ex)
@@ -221,7 +220,7 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
protected final void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen)
+ ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
//create input stream for path
@@ -242,7 +241,7 @@ public class FrameReaderTextCell extends FrameReader
* @return
* @throws IOException
*/
- protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
//create buffered reader
@@ -264,7 +263,7 @@ public class FrameReaderTextCell extends FrameReader
else if (row == -2)
dest.getColumnMetadata(col).setNumDistinct(st.nextLong());
else
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], st.nextToken()));
}
}
catch(Exception ex)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
index ce3993f..39b635f 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
@@ -57,7 +57,7 @@ public class FrameReaderTextCellParallel extends FrameReaderTextCell
*/
@Override
protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen)
+ ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
int numThreads = OptimizerUtils.getParallelTextReadParallelism();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
index b5058f0..841aaea 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
-import java.util.List;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -56,7 +55,7 @@ public abstract class FrameWriter
* @return
* @throws DMLRuntimeException
*/
- public static FrameBlock[] createFrameBlocksForReuse( List<ValueType> schema, List<String> names, long rlen )
+ public static FrameBlock[] createFrameBlocksForReuse( ValueType[] schema, String[] names, long rlen )
throws DMLRuntimeException
{
FrameBlock frameBlock[] = new FrameBlock[1];
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
index 274319f..55b55ce 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -130,7 +130,7 @@ public class FrameWriterTextCSV extends FrameWriter
//append column names if header requested
if( props.hasHeader() ) {
for( int j=0; j<cols; j++ ) {
- sb.append(src.getColumnNames().get(j));
+ sb.append(src.getColumnNames()[j]);
if ( j < cols-1 )
sb.append(delim);
}