You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/09/23 05:42:48 UTC
[4/4] incubator-systemml git commit: [SYSTEMML-557] Memory efficiency
frame block (array-based schema/meta)
[SYSTEMML-557] Memory efficiency frame block (array-based schema/meta)
This patch reworks the core frame block schema/meta data storage from
lists to arrays. Although the actual size difference is small, it has
large impact for distributed frame representations as RDDs on Spark
because Spark apparently largely overestimates the list-based storage
(for a subset of partitions), which led to unnecessary spilling. We now
consistently use array-based schemas and column names throughout the
entire system, except external APIs such as MLContext and JMLC in order
to stay backwards compatible.
Experiments with the following script for transformencode and input data
of size 1G x 5 with double schema, improved the RDD storage size from
117GB to 37.5GB and performance from 388s to 124s.
Xf = read("mboehm/frame/Xf");
jspec = "{\"ids\": true,\"recode\": [1,2,3,4,5]}";
[X,M] = transformencode(target=Xf, spec=jspec);
print(max(X));
print(toString(M))
Additional merges
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/eb988781
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/eb988781
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/eb988781
Branch: refs/heads/master
Commit: eb988781d8f2f2af20103f4aecbe634b1a19a47e
Parents: 3c3d1a4
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Sep 22 21:43:08 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Sep 22 22:21:10 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/api/jmlc/PreparedScript.java | 5 +-
.../sysml/api/mlcontext/BinaryBlockFrame.java | 3 +-
.../api/mlcontext/MLContextConversionUtil.java | 41 +--
.../org/apache/sysml/hops/OptimizerUtils.java | 5 +-
.../controlprogram/caching/CacheableData.java | 3 +-
.../controlprogram/caching/FrameObject.java | 47 ++-
.../context/ExecutionContext.java | 2 +-
.../context/SparkExecutionContext.java | 7 +-
...ReturnParameterizedBuiltinCPInstruction.java | 3 +-
.../cp/ParameterizedBuiltinCPInstruction.java | 5 +-
.../instructions/cp/VariableCPInstruction.java | 2 +-
.../spark/CSVReblockSPInstruction.java | 4 +-
.../instructions/spark/CastSPInstruction.java | 5 +-
...ReturnParameterizedBuiltinSPInstruction.java | 3 +-
.../ParameterizedBuiltinSPInstruction.java | 9 +-
.../instructions/spark/WriteSPInstruction.java | 5 +-
.../spark/utils/FrameRDDConverterUtils.java | 81 +++---
.../apache/sysml/runtime/io/FrameReader.java | 34 +--
.../runtime/io/FrameReaderBinaryBlock.java | 7 +-
.../sysml/runtime/io/FrameReaderTextCSV.java | 19 +-
.../runtime/io/FrameReaderTextCSVParallel.java | 2 +-
.../sysml/runtime/io/FrameReaderTextCell.java | 25 +-
.../runtime/io/FrameReaderTextCellParallel.java | 2 +-
.../apache/sysml/runtime/io/FrameWriter.java | 3 +-
.../sysml/runtime/io/FrameWriterTextCSV.java | 2 +-
.../sysml/runtime/matrix/data/FrameBlock.java | 284 ++++++++++---------
.../matrix/data/OperationsOnMatrixValues.java | 16 +-
.../matrix/mapred/FrameReblockBuffer.java | 7 +-
.../sysml/runtime/transform/BinAgent.java | 8 +-
.../sysml/runtime/transform/DummycodeAgent.java | 6 +-
.../sysml/runtime/transform/MVImputeAgent.java | 6 +-
.../sysml/runtime/transform/OmitAgent.java | 3 +-
.../sysml/runtime/transform/RecodeAgent.java | 2 +-
.../apache/sysml/runtime/transform/TfUtils.java | 13 +-
.../transform/decode/DecoderDummycode.java | 6 +-
.../transform/decode/DecoderFactory.java | 2 +-
.../transform/decode/DecoderPassThrough.java | 4 +-
.../runtime/transform/decode/DecoderRecode.java | 2 +-
.../transform/encode/EncoderFactory.java | 21 +-
.../transform/encode/EncoderPassThrough.java | 2 +-
.../runtime/transform/meta/TfMetaUtils.java | 43 +--
.../sysml/runtime/util/DataConverter.java | 33 ++-
.../sysml/runtime/util/MapReduceTool.java | 13 +-
.../sysml/runtime/util/UtilFunctions.java | 55 ++--
.../test/integration/AutomatedTestBase.java | 9 +-
.../functions/frame/FrameAppendDistTest.java | 10 +-
.../functions/frame/FrameAppendTest.java | 31 +-
.../functions/frame/FrameCastingTest.java | 20 +-
.../functions/frame/FrameConverterTest.java | 37 ++-
.../functions/frame/FrameCopyTest.java | 45 ++-
.../functions/frame/FrameGetSetTest.java | 32 +--
.../functions/frame/FrameIndexingDistTest.java | 12 +-
.../functions/frame/FrameIndexingTest.java | 33 +--
.../functions/frame/FrameReadWriteTest.java | 28 +-
.../functions/frame/FrameScalarCastingTest.java | 5 +-
.../functions/frame/FrameSchemaReadTest.java | 33 +--
.../functions/frame/FrameSerializationTest.java | 17 +-
.../mlcontext/DataFrameFrameConversionTest.java | 6 +-
.../functions/mlcontext/FrameTest.java | 12 +-
.../mlcontext/MLContextFrameTest.java | 14 +-
.../org/apache/sysml/test/utils/TestUtils.java | 19 +-
61 files changed, 588 insertions(+), 625 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index e06810b..4fcef44 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -310,7 +310,7 @@ public class PreparedScript
* @throws DMLException
*/
public void setFrame(String varname, String[][] frame, List<ValueType> schema, boolean reuse) throws DMLException {
- setFrame(varname, DataConverter.convertToFrameBlock(frame, schema), reuse);
+ setFrame(varname, DataConverter.convertToFrameBlock(frame, schema.toArray(new ValueType[0])), reuse);
}
/**
@@ -324,7 +324,8 @@ public class PreparedScript
* @throws DMLException
*/
public void setFrame(String varname, String[][] frame, List<ValueType> schema, List<String> colnames, boolean reuse) throws DMLException {
- setFrame(varname, DataConverter.convertToFrameBlock(frame, schema, colnames), reuse);
+ setFrame(varname, DataConverter.convertToFrameBlock( frame,
+ schema.toArray(new ValueType[0]), colnames.toArray(new String[0])), reuse);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
index 9d7ab9d..5812cf6 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
@@ -22,6 +22,7 @@ package org.apache.sysml.api.mlcontext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.sql.DataFrame;
import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -122,7 +123,7 @@ public class BinaryBlockFrame {
try {
MatrixCharacteristics mc = getMatrixCharacteristics();
FrameSchema frameSchema = frameMetadata.getFrameSchema();
- return SparkExecutionContext.toFrameBlock(binaryBlocks, frameSchema.getSchema(),
+ return SparkExecutionContext.toFrameBlock(binaryBlocks, frameSchema.getSchema().toArray(new ValueType[0]),
(int) mc.getRows(), (int) mc.getCols());
} catch (DMLRuntimeException e) {
throw new MLContextException("Exception while getting FrameBlock from binary-block frame", e);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
index aa0366d..ca9cc2a 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
@@ -22,7 +22,7 @@ package org.apache.sysml.api.mlcontext;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@@ -63,6 +63,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.UtilFunctions;
import scala.collection.JavaConversions;
import scala.reflect.ClassTag;
@@ -193,7 +194,8 @@ public class MLContextConversionUtil {
frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
MatrixFormatMetaData mtd = new MatrixFormatMetaData(mc,
OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd, frameMetadata.getFrameSchema().getSchema());
+ FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
frameObject.acquireModify(frameBlock);
frameObject.release();
return frameObject;
@@ -282,7 +284,8 @@ public class MLContextConversionUtil {
frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema());
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
frameObject.setRDDHandle(new RDDObject(binaryBlocks, variableName));
return frameObject;
}
@@ -357,22 +360,22 @@ public class MLContextConversionUtil {
JavaSparkContext javaSparkContext = MLContextUtil
.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
boolean containsID = isDataFrameWithIDColumn(frameMetadata);
- MatrixCharacteristics matrixCharacteristics = frameMetadata.asMatrixCharacteristics();
- if (matrixCharacteristics == null) {
- matrixCharacteristics = new MatrixCharacteristics();
+ MatrixCharacteristics mc = frameMetadata.asMatrixCharacteristics();
+ if (mc == null) {
+ mc = new MatrixCharacteristics();
long rows = dataFrame.count();
- int cols = dataFrame.columns().length;
- matrixCharacteristics.setDimension(rows, cols);
- frameMetadata.setMatrixCharacteristics(matrixCharacteristics);
+ int cols = dataFrame.columns().length - ((containsID)?1:0);
+ mc.setDimension(rows, cols);
+ frameMetadata.setMatrixCharacteristics(mc);
}
- List<String> colnames = new ArrayList<String>();
- List<ValueType> fschema = new ArrayList<ValueType>();
+ String[] colnames = new String[(int)mc.getCols()];
+ ValueType[] fschema = new ValueType[(int)mc.getCols()];
FrameRDDConverterUtils.convertDFSchemaToFrameSchema(dataFrame.schema(), colnames, fschema, containsID);
- frameMetadata.setFrameSchema(new FrameSchema(fschema));
+ frameMetadata.setFrameSchema(new FrameSchema(Arrays.asList(fschema)));
JavaPairRDD<Long, FrameBlock> binaryBlock = FrameRDDConverterUtils.dataFrameToBinaryBlock(javaSparkContext,
- dataFrame, matrixCharacteristics, containsID);
+ dataFrame, mc, containsID);
return MLContextConversionUtil.binaryBlocksToFrameObject(variableName, binaryBlock, frameMetadata);
} catch (DMLRuntimeException e) {
@@ -604,7 +607,8 @@ public class MLContextConversionUtil {
JavaSparkContext jsc = MLContextUtil.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema());
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
rdd = FrameRDDConverterUtils.csvToBinaryBlock(jsc, javaPairRDDText, mc,
@@ -665,12 +669,13 @@ public class MLContextConversionUtil {
JavaSparkContext jsc = MLContextUtil.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema());
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
- List<ValueType> lschema = null;
+ ValueType[] lschema = null;
if (lschema == null)
- lschema = Collections.nCopies((int) mc.getCols(), ValueType.STRING);
+ lschema = UtilFunctions.nCopies((int) mc.getCols(), ValueType.STRING);
rdd = FrameRDDConverterUtils.textCellToBinaryBlock(jsc, javaPairRDDText, mc, lschema);
} catch (DMLRuntimeException e) {
e.printStackTrace();
@@ -1272,7 +1277,7 @@ public class MLContextConversionUtil {
JavaPairRDD<Long, FrameBlock> binaryBlock = (JavaPairRDD<Long, FrameBlock>) sparkExecutionContext
.getRDDHandleForFrameObject(frameObject, InputInfo.BinaryBlockInputInfo);
MatrixCharacteristics matrixCharacteristics = frameObject.getMatrixCharacteristics();
- FrameSchema fs = new FrameSchema(frameObject.getSchema());
+ FrameSchema fs = new FrameSchema(Arrays.asList(frameObject.getSchema()));
FrameMetadata fm = new FrameMetadata(fs, matrixCharacteristics);
return new BinaryBlockFrame(binaryBlock, fm);
} catch (DMLRuntimeException e) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index b0a938d..8b07f0a 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -20,7 +20,6 @@
package org.apache.sysml.hops;
import java.util.HashMap;
-import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -899,10 +898,10 @@ public class OptimizerUtils
* @param names
* @return
*/
- public static boolean isValidCPDimensions( List<ValueType> schema, List<String> names )
+ public static boolean isValidCPDimensions( ValueType[] schema, String[] names )
{
// Length of schema and names to be same, and > 0.
- return (schema != null && names != null && schema.size() > 0 && schema.size() == names.size());
+ return (schema != null && names != null && schema.length > 0 && schema.length == names.length);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 3e2f6c1..d3addd6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -22,7 +22,6 @@ package org.apache.sysml.runtime.controlprogram.caching;
import java.io.File;
import java.io.IOException;
import java.lang.ref.SoftReference;
-import java.util.List;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.logging.Log;
@@ -973,7 +972,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
return (_data.getInMemorySize() <= CACHING_THRESHOLD);
}
- protected List<ValueType> getSchema() {
+ protected ValueType[] getSchema() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 8c5fe8b..effb604 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -21,10 +21,9 @@ package org.apache.sysml.runtime.controlprogram.caching;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.Arrays;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
@@ -44,12 +43,13 @@ import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class FrameObject extends CacheableData<FrameBlock>
{
private static final long serialVersionUID = 1755082174281927785L;
- private List<ValueType> _schema = null;
+ private ValueType[] _schema = null;
/**
*
@@ -85,7 +85,7 @@ public class FrameObject extends CacheableData<FrameBlock>
* @param schema
*
*/
- public FrameObject(String fname, MetaData meta, List<ValueType> schema) {
+ public FrameObject(String fname, MetaData meta, ValueType[] schema) {
this();
setFileName(fname);
setMetaData(meta);
@@ -102,7 +102,7 @@ public class FrameObject extends CacheableData<FrameBlock>
}
@Override
- public List<ValueType> getSchema() {
+ public ValueType[] getSchema() {
return _schema;
}
@@ -112,9 +112,9 @@ public class FrameObject extends CacheableData<FrameBlock>
* @param cu column upper bound, inclusive
* @return
*/
- public List<ValueType> getSchema(int cl, int cu) {
- return (_schema!=null && _schema.size()>cu) ? _schema.subList(cl, cu+1) :
- Collections.nCopies(cu-cl+1, ValueType.STRING);
+ public ValueType[] getSchema(int cl, int cu) {
+ return (_schema!=null && _schema.length>cu) ? Arrays.copyOfRange(_schema, cl, cu+1) :
+ UtilFunctions.nCopies(cu-cl+1, ValueType.STRING);
}
/**
@@ -124,13 +124,10 @@ public class FrameObject extends CacheableData<FrameBlock>
* @param fo
* @return
*/
- public List<ValueType> mergeSchemas(FrameObject fo) {
- ArrayList<ValueType> ret = new ArrayList<ValueType>();
- ret.addAll((_schema!=null) ? _schema :
- Collections.nCopies((int)getNumColumns(), ValueType.STRING));
- ret.addAll((fo.getSchema()!=null) ? fo.getSchema() :
- Collections.nCopies((int)fo.getNumColumns(), ValueType.STRING));
- return ret;
+ public ValueType[] mergeSchemas(FrameObject fo) {
+ return (ValueType[]) ArrayUtils.addAll(
+ (_schema!=null) ? _schema : UtilFunctions.nCopies((int)getNumColumns(), ValueType.STRING),
+ (fo._schema!=null) ? fo._schema : UtilFunctions.nCopies((int)fo.getNumColumns(), ValueType.STRING));
}
public void setSchema(String schema) {
@@ -138,18 +135,18 @@ public class FrameObject extends CacheableData<FrameBlock>
//populate default schema
int clen = (int) getNumColumns();
if( clen > 0 ) //known number of cols
- _schema = Collections.nCopies(clen, ValueType.STRING);
+ _schema = UtilFunctions.nCopies(clen, ValueType.STRING);
}
else {
//parse given schema
- _schema = new ArrayList<ValueType>();
String[] parts = schema.split(DataExpression.DEFAULT_DELIM_DELIMITER);
- for( String svt : parts )
- _schema.add(ValueType.valueOf(svt.toUpperCase()));
+ _schema = new ValueType[parts.length];
+ for( int i=0; i<parts.length; i++ )
+ _schema[i] = ValueType.valueOf(parts[i].toUpperCase());
}
}
- public void setSchema(List<ValueType> schema) {
+ public void setSchema(ValueType[] schema) {
_schema = schema;
}
@@ -200,8 +197,8 @@ public class FrameObject extends CacheableData<FrameBlock>
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
//handle missing schema if necessary
- List<ValueType> lschema = (_schema!=null) ? _schema :
- Collections.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
+ ValueType[] lschema = (_schema!=null) ? _schema :
+ UtilFunctions.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
//read the frame block
FrameBlock data = null;
@@ -237,8 +234,8 @@ public class FrameObject extends CacheableData<FrameBlock>
int clen = (int)mc.getCols();
//handle missing schema if necessary
- List<ValueType> lschema = (_schema!=null) ? _schema :
- Collections.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
+ ValueType[] lschema = (_schema!=null) ? _schema :
+ UtilFunctions.nCopies(clen>=1 ? (int)clen : 1, ValueType.STRING);
FrameBlock fb = null;
try {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 14b9c44..b6011ae 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -458,7 +458,7 @@ public class ExecutionContext
{
FrameObject fo = getFrameObject(varName);
if( outputData.getNumColumns()>0 && outputData.getSchema()!=null )
- fo.setValueType(outputData.getSchema().get(0));
+ fo.setValueType(outputData.getSchema()[0]);
fo.acquireModify(outputData);
fo.release();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index a6f99e8..964d2d6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.controlprogram.context;
import java.io.IOException;
-import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -958,7 +957,7 @@ public class SparkExecutionContext extends ExecutionContext
* @throws DMLRuntimeException
*/
@SuppressWarnings("unchecked")
- public static FrameBlock toFrameBlock(RDDObject rdd, List<ValueType> schema, int rlen, int clen)
+ public static FrameBlock toFrameBlock(RDDObject rdd, ValueType[] schema, int rlen, int clen)
throws DMLRuntimeException
{
JavaPairRDD<Long,FrameBlock> lrdd = (JavaPairRDD<Long,FrameBlock>) rdd.getRDD();
@@ -974,13 +973,13 @@ public class SparkExecutionContext extends ExecutionContext
* @return
* @throws DMLRuntimeException
*/
- public static FrameBlock toFrameBlock(JavaPairRDD<Long,FrameBlock> rdd, List<ValueType> schema, int rlen, int clen)
+ public static FrameBlock toFrameBlock(JavaPairRDD<Long,FrameBlock> rdd, ValueType[] schema, int rlen, int clen)
throws DMLRuntimeException
{
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
if(schema == null)
- schema = Collections.nCopies(clen, ValueType.STRING);
+ schema = UtilFunctions.nCopies(clen, ValueType.STRING);
//create output frame block (w/ lazy allocation)
FrameBlock out = new FrameBlock(schema);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
index 05cf19e..5af403d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.instructions.cp;
import java.util.ArrayList;
-import java.util.List;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -82,7 +81,7 @@ public class MultiReturnParameterizedBuiltinCPInstruction extends ComputationCPI
//obtain and pin input frame
FrameBlock fin = ec.getFrameInput(input1.getName());
String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
- List<String> colnames = fin.getColumnNames();
+ String[] colnames = fin.getColumnNames();
//execute block transform encode
Encoder encoder = EncoderFactory.createEncoder(spec, colnames, fin.getNumColumns(), null);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index a245d10..299fb3c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.instructions.cp;
import java.util.HashMap;
-import java.util.List;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
@@ -270,7 +269,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
//acquire locks
FrameBlock data = ec.getFrameInput(params.get("target"));
FrameBlock meta = ec.getFrameInput(params.get("meta"));
- List<String> colNames = data.getColumnNames();
+ String[] colNames = data.getColumnNames();
//compute transformapply
Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, data.getNumColumns(), meta);
@@ -285,7 +284,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
//acquire locks
MatrixBlock data = ec.getMatrixInput(params.get("target"));
FrameBlock meta = ec.getFrameInput(params.get("meta"));
- List<String> colnames = meta.getColumnNames();
+ String[] colnames = meta.getColumnNames();
//compute transformdecode
Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index ee86b59..9c45366 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -536,7 +536,7 @@ public class VariableCPInstruction extends CPInstruction
Object value = fBlock.get(0,0);
ec.releaseFrameInput(input1.getName());
ec.setScalarOutput(output.getName(),
- ScalarObjectFactory.createScalarObject(fBlock.getSchema().get(0), value));
+ ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value));
}
else { //assume DataType.MATRIX otherwise
MatrixBlock mBlock = ec.getMatrixInput(input1.getName());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index e11f505..0e87be5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -19,8 +19,6 @@
package org.apache.sysml.runtime.instructions.spark;
-import java.util.List;
-
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
@@ -163,7 +161,7 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
* @throws DMLRuntimeException
*/
@SuppressWarnings("unchecked")
- protected JavaPairRDD<Long,FrameBlock> processFrameCSVReblockInstruction(SparkExecutionContext sec, MatrixCharacteristics mcOut, List<ValueType> schema)
+ protected JavaPairRDD<Long,FrameBlock> processFrameCSVReblockInstruction(SparkExecutionContext sec, MatrixCharacteristics mcOut, ValueType[] schema)
throws DMLRuntimeException
{
//get input rdd (needs to be longwritable/text for consistency with meta data, in case of
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
index 4487b20..4fdd617 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
@@ -19,8 +19,6 @@
package org.apache.sysml.runtime.instructions.spark;
-import java.util.Collections;
-
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.UnaryCP;
@@ -37,6 +35,7 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class CastSPInstruction extends UnarySPInstruction
@@ -95,7 +94,7 @@ public class CastSPInstruction extends UnarySPInstruction
//update schema information for output frame
if( opcode.equals(UnaryCP.CAST_AS_FRAME_OPCODE) ) {
sec.getFrameObject(output.getName()).setSchema(
- Collections.nCopies((int)mcIn.getCols(), ValueType.DOUBLE));
+ UtilFunctions.nCopies((int)mcIn.getCols(), ValueType.DOUBLE));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
index 9b35daa..b36b1f3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
@@ -24,7 +24,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map.Entry;
import org.apache.spark.Accumulator;
@@ -127,7 +126,7 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI
String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
- List<String> colnames = !TfMetaUtils.isIDSpecification(spec) ?
+ String[] colnames = !TfMetaUtils.isIDSpecification(spec) ?
in.lookup(1L).get(0).getColumnNames() : null;
//step 1: build transform meta data
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
index f880f57..adbc06a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
@@ -429,7 +428,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
FrameBlock meta = sec.getFrameInput(params.get("meta"));
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(params.get("target"));
MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
- List<String> colnames = !TfMetaUtils.isIDSpecification(params.get("spec")) ?
+ String[] colnames = !TfMetaUtils.isIDSpecification(params.get("spec")) ?
in.lookup(1L).get(0).getColumnNames() : null;
//compute omit offset map for block shifts
@@ -463,7 +462,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(params.get("target"));
MatrixCharacteristics mc = sec.getMatrixCharacteristics(params.get("target"));
FrameBlock meta = sec.getFrameInput(params.get("meta"));
- List<String> colnames = meta.getColumnNames();
+ String[] colnames = meta.getColumnNames();
//reblock if necessary (clen > bclen)
if( mc.getCols() > mc.getNumColBlocks() ) {
@@ -773,7 +772,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
private int[] _omitColList = null;
- public RDDTransformApplyOffsetFunction(String spec, List<String> colnames) {
+ public RDDTransformApplyOffsetFunction(String spec, String[] colnames) {
try {
_omitColList = TfMetaUtils.parseJsonIDList(spec, colnames, TfUtils.TXMETHOD_OMIT);
}
@@ -796,7 +795,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
for( int j=0; j<_omitColList.length; j++ ) {
int colID = _omitColList[j];
Object val = blk.get(i, colID-1);
- valid &= !(val==null || (blk.getSchema().get(colID-1)==
+ valid &= !(val==null || (blk.getSchema()[colID-1]==
ValueType.STRING && val.toString().isEmpty()));
}
rmRows += valid ? 0 : 1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
index 1b974f9..584a4e9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.instructions.spark;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
import org.apache.hadoop.io.LongWritable;
@@ -137,7 +136,7 @@ public class WriteSPInstruction extends SPInstruction
//get filename (literal or variable expression)
String fname = ec.getScalarInput(input2.getName(), ValueType.STRING, input2.isLiteral()).getStringValue();
- List<ValueType> schema = (input1.getDataType()==DataType.FRAME) ?
+ ValueType[] schema = (input1.getDataType()==DataType.FRAME) ?
sec.getFrameObject(input1.getName()).getSchema() : null;
try
@@ -282,7 +281,7 @@ public class WriteSPInstruction extends SPInstruction
* @throws IOException
*/
@SuppressWarnings("unchecked")
- protected void processFrameWriteInstruction(SparkExecutionContext sec, String fname, OutputInfo oi, List<ValueType> schema)
+ protected void processFrameWriteInstruction(SparkExecutionContext sec, String fname, OutputInfo oi, ValueType[] schema)
throws DMLRuntimeException, IOException
{
//get input rdd
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index 3f6cc4c..f8fb05c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -22,7 +22,6 @@ package org.apache.sysml.runtime.instructions.spark.utils;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Arrays;
@@ -92,7 +91,7 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> csvToBinaryBlock(JavaSparkContext sc,
- JavaPairRDD<LongWritable, Text> input, MatrixCharacteristics mc, List<ValueType> schema,
+ JavaPairRDD<LongWritable, Text> input, MatrixCharacteristics mc, ValueType[] schema,
boolean hasHeader, String delim, boolean fill, double fillValue)
throws DMLRuntimeException
{
@@ -114,11 +113,9 @@ public class FrameRDDConverterUtils
.zipWithIndex(); //zip row index
//prepare default schema if needed
- if( schema == null || schema.size()==1 ) {
- schema = Collections.nCopies((int)mc.getCols(),
- (schema!=null) ? schema.get(0) : ValueType.STRING);
- }
-
+ if( schema == null || schema.length==1 )
+ schema = UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING);
+
//convert csv rdd to binary block rdd (w/ partial blocks)
JavaPairRDD<Long, FrameBlock> out = prepinput.mapPartitionsToPair(
new CSVToBinaryBlockFunction(mc, schema, hasHeader, delim));
@@ -140,7 +137,7 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> csvToBinaryBlock(JavaSparkContext sc,
- JavaRDD<String> input, MatrixCharacteristics mcOut, List<ValueType> schema,
+ JavaRDD<String> input, MatrixCharacteristics mcOut, ValueType[] schema,
boolean hasHeader, String delim, boolean fill, double fillValue)
throws DMLRuntimeException
{
@@ -189,7 +186,7 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> textCellToBinaryBlock(JavaSparkContext sc,
- JavaPairRDD<LongWritable, Text> in, MatrixCharacteristics mcOut, List<ValueType> schema )
+ JavaPairRDD<LongWritable, Text> in, MatrixCharacteristics mcOut, ValueType[] schema )
throws DMLRuntimeException
{
//convert input rdd to serializable long/frame block
@@ -210,13 +207,13 @@ public class FrameRDDConverterUtils
* @throws DMLRuntimeException
*/
public static JavaPairRDD<Long, FrameBlock> textCellToBinaryBlockLongIndex(JavaSparkContext sc,
- JavaPairRDD<Long, Text> input, MatrixCharacteristics mc, List<ValueType> schema )
+ JavaPairRDD<Long, Text> input, MatrixCharacteristics mc, ValueType[] schema )
throws DMLRuntimeException
{
//prepare default schema if needed
- if( schema == null || schema.size()==1 ) {
- schema = Collections.nCopies((int)mc.getCols(),
- (schema!=null) ? schema.get(0) : ValueType.STRING);
+ if( schema == null || schema.length==1 ) {
+ schema = UtilFunctions.nCopies((int)mc.getCols(),
+ (schema!=null) ? schema[0] : ValueType.STRING);
}
//convert textcell rdd to binary block rdd (w/ partial blocks)
@@ -342,8 +339,8 @@ public class FrameRDDConverterUtils
df.javaRDD().zipWithIndex(); //zip row index
//convert data frame to frame schema (prepare once)
- List<String> colnames = new ArrayList<String>();
- List<ValueType> fschema = new ArrayList<ValueType>();
+ String[] colnames = new String[(int)mc.getCols()];
+ ValueType[] fschema = new ValueType[(int)mc.getCols()];
convertDFSchemaToFrameSchema(df.schema(), colnames, fschema, containsID);
//convert rdd to binary block rdd
@@ -360,7 +357,7 @@ public class FrameRDDConverterUtils
* @return
*/
public static DataFrame binaryBlockToDataFrame(SQLContext sqlctx, JavaPairRDD<Long,FrameBlock> in,
- MatrixCharacteristics mc, List<ValueType> schema)
+ MatrixCharacteristics mc, ValueType[] schema)
{
if( !mc.colsKnown() )
throw new RuntimeException("Number of columns needed to convert binary block to data frame.");
@@ -371,7 +368,7 @@ public class FrameRDDConverterUtils
//create data frame schema
if( schema == null )
- schema = Collections.nCopies((int)mc.getCols(), ValueType.STRING);
+ schema = UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING);
StructType dfSchema = convertFrameSchemaToDFSchema(schema, true);
//rdd to data frame conversion
@@ -387,7 +384,7 @@ public class FrameRDDConverterUtils
* @return
* Returns the DataFrame schema (StructType)
*/
- public static StructType convertFrameSchemaToDFSchema(List<ValueType> fschema, boolean containsID)
+ public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID)
{
// generate the schema based on the string of schema
List<StructField> fields = new ArrayList<StructField>();
@@ -421,30 +418,30 @@ public class FrameRDDConverterUtils
* @param containsID
* @return
*/
- public static void convertDFSchemaToFrameSchema(StructType dfschema, List<String> colnames,
- List<ValueType> fschema, boolean containsID)
+ public static void convertDFSchemaToFrameSchema(StructType dfschema, String[] colnames,
+ ValueType[] fschema, boolean containsID)
{
int off = containsID ? 1 : 0;
for( int i=off; i<dfschema.fields().length; i++ ) {
StructField structType = dfschema.apply(i);
- colnames.add(structType.name());
+ colnames[i-off] = structType.name();
if(structType.dataType() == DataTypes.DoubleType
|| structType.dataType() == DataTypes.FloatType)
- fschema.add(ValueType.DOUBLE);
+ fschema[i-off] = ValueType.DOUBLE;
else if(structType.dataType() == DataTypes.LongType
|| structType.dataType() == DataTypes.IntegerType)
- fschema.add(ValueType.INT);
+ fschema[i-off] = ValueType.INT;
else if(structType.dataType() == DataTypes.BooleanType)
- fschema.add(ValueType.BOOLEAN);
+ fschema[i-off] = ValueType.BOOLEAN;
else
- fschema.add(ValueType.STRING);
+ fschema[i-off] = ValueType.STRING;
}
}
/*
* It will return JavaRDD<Row> based on csv data input file.
*/
- public static JavaRDD<Row> csvToRowRDD(JavaSparkContext sc, String fnameIn, String delim, List<ValueType> schema)
+ public static JavaRDD<Row> csvToRowRDD(JavaSparkContext sc, String fnameIn, String delim, ValueType[] schema)
{
// Load a text file and convert each line to a java rdd.
JavaRDD<String> dataRdd = sc.textFile(fnameIn);
@@ -458,10 +455,10 @@ public class FrameRDDConverterUtils
{
private static final long serialVersionUID = -6736256507697511070L;
- private List<ValueType> _schema = null;
+ private ValueType[] _schema = null;
private String _delim = null;
- public RowGenerator(List<ValueType> schema, String delim) {
+ public RowGenerator(ValueType[] schema, String delim) {
_schema = schema;
_delim = delim;
}
@@ -471,7 +468,7 @@ public class FrameRDDConverterUtils
String[] fields = IOUtilFunctions.splitCSV(record, _delim);
Object[] objects = new Object[fields.length];
for (int i=0; i<fields.length; i++) {
- objects[i] = UtilFunctions.stringToObject(_schema.get(i), fields[i]);
+ objects[i] = UtilFunctions.stringToObject(_schema[i], fields[i]);
}
return RowFactory.create(objects);
}
@@ -617,12 +614,12 @@ public class FrameRDDConverterUtils
private boolean _hasHeader = false;
private String _delim = null;
private int _maxRowsPerBlock = -1;
- private List<ValueType> _schema = null;
- private List<String> _colnames = null;
+ private ValueType[] _schema = null;
+ private String[] _colnames = null;
private List<String> _mvMeta = null; //missing value meta data
private List<String> _ndMeta = null; //num distinct meta data
- public CSVToBinaryBlockFunction(MatrixCharacteristics mc, List<ValueType> schema, boolean hasHeader, String delim) {
+ public CSVToBinaryBlockFunction(MatrixCharacteristics mc, ValueType[] schema, boolean hasHeader, String delim) {
_clen = mc.getCols();
_schema = schema;
_hasHeader = hasHeader;
@@ -646,7 +643,7 @@ public class FrameRDDConverterUtils
String row = tmp._1().toString().trim();
long rowix = tmp._2();
if(_hasHeader && rowix == 0) { //Skip header
- _colnames = Arrays.asList(IOUtilFunctions.splitCSV(row, _delim));
+ _colnames = IOUtilFunctions.splitCSV(row, _delim);
continue;
}
if( row.startsWith(TfUtils.TXMTD_MVPREFIX) ) {
@@ -744,7 +741,7 @@ public class FrameRDDConverterUtils
if( ix==1 ) {
if( _props.hasHeader() ) {
for(int j = 1; j <= blk.getNumColumns(); j++) {
- sb.append(blk.getColumnNames().get(j)
+ sb.append(blk.getColumnNames()[j]
+ ((j<blk.getNumColumns()-1)?_props.getDelim():""));
}
ret.add(sb.toString());
@@ -790,13 +787,13 @@ public class FrameRDDConverterUtils
private static final long serialVersionUID = 2269315691094111843L;
private long _clen = -1;
- private List<String> _colnames = null;
- private List<ValueType> _schema = null;
+ private String[] _colnames = null;
+ private ValueType[] _schema = null;
private boolean _containsID = false;
private int _maxRowsPerBlock = -1;
- public DataFrameToBinaryBlockFunction(MatrixCharacteristics mc, List<String> colnames,
- List<ValueType> schema, boolean containsID) {
+ public DataFrameToBinaryBlockFunction(MatrixCharacteristics mc, String[] colnames,
+ ValueType[] schema, boolean containsID) {
_clen = mc.getCols();
_colnames = colnames;
_schema = schema;
@@ -831,7 +828,7 @@ public class FrameRDDConverterUtils
int off = _containsID ? 1 : 0;
for(int i=off; i<row.size(); i++) {
tmprow[i-off] = UtilFunctions.objectToObject(
- _schema.get(i-off), row.get(i));
+ _schema[i-off], row.get(i));
}
fb.appendRow(tmprow);
}
@@ -932,9 +929,9 @@ public class FrameRDDConverterUtils
private static class TextToBinaryBlockFunction extends CellToBinaryBlockFunction implements PairFlatMapFunction<Iterator<Text>,Long,FrameBlock>
{
private static final long serialVersionUID = -2042208027876880588L;
- List<ValueType> _schema = null;
+ ValueType[] _schema = null;
- protected TextToBinaryBlockFunction(MatrixCharacteristics mc, List<ValueType> schema ) {
+ protected TextToBinaryBlockFunction(MatrixCharacteristics mc, ValueType[] schema ) {
super(mc);
_schema = schema;
}
@@ -958,7 +955,7 @@ public class FrameRDDConverterUtils
st.reset( strVal );
long row = st.nextLong();
long col = st.nextLong();
- Object val = UtilFunctions.stringToObject(_schema.get((int)col-1), st.nextToken());
+ Object val = UtilFunctions.stringToObject(_schema[(int)col-1], st.nextToken());
//flush buffer if necessary
if( rbuff.getSize() >= rbuff.getCapacity() )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
index e318fff..1dc39e5 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
@@ -21,10 +21,7 @@ package org.apache.sysml.runtime.io;
import java.io.EOFException;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.LinkedList;
-import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -34,6 +31,7 @@ import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* Base class for all format-specific frame readers. Every reader is required to implement the basic
@@ -53,7 +51,7 @@ public abstract class FrameReader
* @param clen
* @return
*/
- public abstract FrameBlock readFrameFromHDFS( String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public abstract FrameBlock readFrameFromHDFS( String fname, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException;
/**
@@ -64,10 +62,10 @@ public abstract class FrameReader
* @param clen
* @return
*/
- public FrameBlock readFrameFromHDFS( String fname, List<ValueType> schema, long rlen, long clen )
+ public FrameBlock readFrameFromHDFS( String fname, ValueType[] schema, long rlen, long clen )
throws IOException, DMLRuntimeException
{
- return readFrameFromHDFS(fname, schema, getDefColNames(schema.size()), rlen, clen);
+ return readFrameFromHDFS(fname, schema, getDefColNames(schema.length), rlen, clen);
}
/**
@@ -88,11 +86,11 @@ public abstract class FrameReader
* @param iNumColumns
* @return
*/
- public List<ValueType> getDefSchema( long clen )
+ public ValueType[] getDefSchema( long clen )
throws IOException, DMLRuntimeException
{
int lclen = Math.max((int)clen, 1);
- return Collections.nCopies(lclen, ValueType.STRING);
+ return UtilFunctions.nCopies(lclen, ValueType.STRING);
}
/**
@@ -100,13 +98,11 @@ public abstract class FrameReader
* @param iNumColumns
* @return
*/
- public List<String> getDefColNames( long clen )
+ public String[] getDefColNames( long clen )
throws IOException, DMLRuntimeException
{
- List<String> colNames = new ArrayList<String>();
- for (int i=0; i < clen; i++)
- colNames.add("C"+i);
- return colNames;
+ return (clen < 0) ? new String[0] :
+ FrameBlock.createColNames((int)clen);
}
/**
@@ -148,7 +144,7 @@ public abstract class FrameReader
* @throws DMLRuntimeException
* @throws IOException
*/
- protected static FrameBlock createOutputFrameBlock(List<ValueType> schema, List<String> names, long nrow)
+ protected static FrameBlock createOutputFrameBlock(ValueType[] schema, String[] names, long nrow)
throws IOException, DMLRuntimeException
{
//check schema and column names
@@ -167,9 +163,9 @@ public abstract class FrameReader
* @param ncol
* @return
*/
- protected static List<ValueType> createOutputSchema(List<ValueType> schema, long ncol) {
- if( schema.size()==1 && ncol > 1 )
- return Collections.nCopies((int)ncol, schema.get(0));
+ protected static ValueType[] createOutputSchema(ValueType[] schema, long ncol) {
+ if( schema.length==1 && ncol > 1 )
+ return UtilFunctions.nCopies((int)ncol, schema[0]);
return schema;
}
@@ -179,8 +175,8 @@ public abstract class FrameReader
* @param ncol
* @return
*/
- protected static List<String> createOutputNames(List<String> names, long ncol) {
- if( names.size() != ncol )
+ protected static String[] createOutputNames(String[] names, long ncol) {
+ if( names.length != ncol )
return FrameBlock.createColNames((int)ncol);
return names;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
index 88e3d96..c720fa0 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -51,12 +50,12 @@ public class FrameReaderBinaryBlock extends FrameReader
* @throws IOException
*/
@Override
- public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//prepare file access
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
index b608991..cfd8b7e 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
@@ -20,8 +20,6 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -67,7 +65,7 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
@Override
- public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+ public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names,
long rlen, long clen)
throws IOException, DMLRuntimeException
{
@@ -88,8 +86,8 @@ public class FrameReaderTextCSV extends FrameReader
}
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//core read (sequential/parallel)
@@ -112,7 +110,7 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
- FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
TextInputFormat informat = new TextInputFormat();
@@ -140,7 +138,7 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
protected final void readCSVFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job,
- FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen, int rl, boolean first)
+ FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen, int rl, boolean first)
throws IOException
{
boolean hasHeader = _props.hasHeader();
@@ -159,8 +157,7 @@ public class FrameReaderTextCSV extends FrameReader
//handle header if existing
if(first && hasHeader ) {
reader.next(key, value); //read header
- List<String> colnames = Arrays.asList(value.toString().split(delim));
- dest.setColumnNames(colnames);
+ dest.setColumnNames(value.toString().split(delim));
}
// Read the data
@@ -189,11 +186,11 @@ public class FrameReaderTextCSV extends FrameReader
part = part.trim();
if ( part.isEmpty() ) {
if( isFill && dfillValue!=0 )
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), sfillValue));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], sfillValue));
emptyValuesFound = true;
}
else {
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), part));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], part));
}
col++;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
index 2713231..84c0721 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
@@ -68,7 +68,7 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV
*/
@Override
protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
- FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
int numThreads = OptimizerUtils.getParallelTextReadParallelism();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
index e7efc2b..48a87fa 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
@@ -23,7 +23,6 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -60,12 +59,12 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
@Override
- public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//prepare file access
@@ -107,12 +106,12 @@ public class FrameReaderTextCell extends FrameReader
* @throws DMLRuntimeException
* @throws IOException
*/
- public final FrameBlock readFrameFromInputStream(InputStream is, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromInputStream(InputStream is, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- List<ValueType> lschema = createOutputSchema(schema, clen);
- List<String> lnames = createOutputNames(names, clen);
+ ValueType[] lschema = createOutputSchema(schema, clen);
+ String[] lnames = createOutputNames(names, clen);
FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//core read
@@ -134,7 +133,7 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen)
+ ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
if( fs.isDirectory(path) ) {
@@ -163,7 +162,7 @@ public class FrameReaderTextCell extends FrameReader
protected final void readTextCellFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest)
throws IOException
{
- List<ValueType> schema = dest.getSchema();
+ ValueType[] schema = dest.getSchema();
int rlen = dest.getNumRows();
int clen = dest.getNumColumns();
@@ -187,7 +186,7 @@ public class FrameReaderTextCell extends FrameReader
else if( row == -2 )
dest.getColumnMetadata(col).setNumDistinct(st.nextLong());
else
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], st.nextToken()));
}
}
catch(Exception ex)
@@ -221,7 +220,7 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
protected final void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen)
+ ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
//create input stream for path
@@ -242,7 +241,7 @@ public class FrameReaderTextCell extends FrameReader
* @return
* @throws IOException
*/
- protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
//create buffered reader
@@ -264,7 +263,7 @@ public class FrameReaderTextCell extends FrameReader
else if (row == -2)
dest.getColumnMetadata(col).setNumDistinct(st.nextLong());
else
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
+ dest.set(row, col, UtilFunctions.stringToObject(schema[col], st.nextToken()));
}
}
catch(Exception ex)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
index ce3993f..39b635f 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
@@ -57,7 +57,7 @@ public class FrameReaderTextCellParallel extends FrameReaderTextCell
*/
@Override
protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen)
+ ValueType[] schema, String[] names, long rlen, long clen)
throws IOException
{
int numThreads = OptimizerUtils.getParallelTextReadParallelism();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
index b5058f0..841aaea 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
-import java.util.List;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -56,7 +55,7 @@ public abstract class FrameWriter
* @return
* @throws DMLRuntimeException
*/
- public static FrameBlock[] createFrameBlocksForReuse( List<ValueType> schema, List<String> names, long rlen )
+ public static FrameBlock[] createFrameBlocksForReuse( ValueType[] schema, String[] names, long rlen )
throws DMLRuntimeException
{
FrameBlock frameBlock[] = new FrameBlock[1];
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/eb988781/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
index 274319f..55b55ce 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -130,7 +130,7 @@ public class FrameWriterTextCSV extends FrameWriter
//append column names if header requested
if( props.hasHeader() ) {
for( int j=0; j<cols; j++ ) {
- sb.append(src.getColumnNames().get(j));
+ sb.append(src.getColumnNames()[j]);
if ( j < cols-1 )
sb.append(delim);
}