You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/03/18 23:37:00 UTC
incubator-systemml git commit: [SYSTEMML-573] Frame IO interface
Repository: incubator-systemml
Updated Branches:
refs/heads/master 25429fcb8 -> a939658ee
[SYSTEMML-573] Frame IO interface
Closes #92.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/a939658e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/a939658e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/a939658e
Branch: refs/heads/master
Commit: a939658ee0d904bbe6816478b39e740ccc8907a4
Parents: 25429fc
Author: Arvind Surve <ac...@yahoo.com>
Authored: Fri Mar 18 15:32:19 2016 -0700
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Fri Mar 18 15:32:19 2016 -0700
----------------------------------------------------------------------
.../org/apache/sysml/hops/OptimizerUtils.java | 16 ++
.../apache/sysml/runtime/io/FrameReader.java | 184 +++++++++++++++++++
.../runtime/io/FrameReaderBinaryBlock.java | 53 ++++++
.../sysml/runtime/io/FrameReaderFactory.java | 94 ++++++++++
.../sysml/runtime/io/FrameReaderTextCSV.java | 63 +++++++
.../sysml/runtime/io/FrameReaderTextCell.java | 54 ++++++
.../apache/sysml/runtime/io/FrameWriter.java | 77 ++++++++
.../runtime/io/FrameWriterBinaryBlock.java | 44 +++++
.../sysml/runtime/io/FrameWriterFactory.java | 78 ++++++++
.../sysml/runtime/io/FrameWriterTextCSV.java | 58 ++++++
.../sysml/runtime/io/FrameWriterTextCell.java | 44 +++++
11 files changed, 765 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 73de4c4..b0a0790 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -20,6 +20,7 @@
package org.apache.sysml.hops;
import java.util.HashMap;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,6 +34,7 @@ import org.apache.sysml.hops.Hop.OpOp2;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
@@ -862,6 +864,20 @@ public class OptimizerUtils
}
/**
+ * Returns false if schema and names are not properly specified; other true
+ * Length to be > 0, and length of both to be equal.
+ *
+ * @param schema
+ * @param names
+ * @return
+ */
+ public static boolean isValidCPDimensions( List<ValueType> schema, List<String> names )
+ {
+ // Length of schema and names to be same, and > 0.
+ return (schema != null && names != null && schema.size() > 0 && schema.size() == names.size());
+ }
+
+ /**
* Determines if valid matrix size to be represented in CP data structures. Note that
* sparsity needs to be specified as rows*cols if unknown.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
new file mode 100644
index 0000000..023c0ab
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.util.MapReduceTool;
+
+/**
+ * Base class for all format-specific frame readers. Every reader is required to implement the basic
+ * read functionality but might provide additional custom functionality. Any non-default parameters
+ * (e.g., CSV read properties) should be passed into custom constructors. There is also a factory
+ * for creating format-specific readers.
+ *
+ */
+public abstract class FrameReader
+{
+ /**
+ *
+ * @param fname
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @return
+ */
+ public abstract FrameBlock readFrameFromHDFS( String fname, List<ValueType> schema, List<String> names,
+ long rlen, long clen)
+ throws IOException, DMLRuntimeException;
+
+ /**
+ *
+ * @param fname
+ * @param schema
+ * @param rlen
+ * @param clen
+ * @return
+ */
+ public FrameBlock readFrameFromHDFS( String fname, List<ValueType> schema, long rlen, long clen )
+ throws IOException, DMLRuntimeException
+ {
+ return readFrameFromHDFS(fname, schema, getDefColNames(schema.size()), rlen, clen);
+ }
+
+ /**
+ *
+ * @param fname
+ * @param rlen
+ * @param clen
+ * @return
+ */
+ public FrameBlock readFrameFromHDFS( String fname, long rlen, long clen )
+ throws IOException, DMLRuntimeException
+ {
+ return readFrameFromHDFS(fname, getDefSchema(clen), getDefColNames(clen), rlen, clen);
+ }
+
+ /**
+ *
+ * @param iNumColumns
+ * @return
+ */
+ public List<ValueType> getDefSchema( long lNumColumns )
+ throws IOException, DMLRuntimeException
+ {
+ List<ValueType> schema = new ArrayList<ValueType>();
+ for (int i=0; i < lNumColumns; ++i)
+ schema.add(ValueType.STRING);
+ return schema;
+ }
+
+ /**
+ *
+ * @param iNumColumns
+ * @return
+ */
+ public List<String> getDefColNames( long lNumColumns )
+ throws IOException, DMLRuntimeException
+ {
+ List<String> colNames = new ArrayList<String>();
+ for (int i=0; i < lNumColumns; ++i)
+ colNames.add("C"+i);
+ return colNames;
+ }
+
+ /**
+ *
+ * @param fs
+ * @param file
+ * @return
+ * @throws IOException
+ */
+ public static Path[] getSequenceFilePaths( FileSystem fs, Path file )
+ throws IOException
+ {
+ Path[] ret = null;
+
+ if( fs.isDirectory(file) )
+ {
+ LinkedList<Path> tmp = new LinkedList<Path>();
+ FileStatus[] dStatus = fs.listStatus(file);
+ for( FileStatus fdStatus : dStatus )
+ if( !fdStatus.getPath().getName().startsWith("_") ) //skip internal files
+ tmp.add(fdStatus.getPath());
+ ret = tmp.toArray(new Path[0]);
+ }
+ else
+ {
+ ret = new Path[]{ file };
+ }
+
+ return ret;
+ }
+
+ /**
+ * NOTE: mallocDense controls if the output matrix blocks is fully allocated, this can be redundant
+ * if binary block read and single block.
+ *
+ * @param schema
+ * @param names
+ * @return
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ protected static FrameBlock createOutputFrameBlock(List<ValueType> schema, List<String> names)
+ throws IOException, DMLRuntimeException
+ {
+ //check schema and column names
+ if( !OptimizerUtils.isValidCPDimensions(schema, names) )
+ throw new DMLRuntimeException("Schema and names to be define with equal size.");
+
+ //prepare result frame block
+ FrameBlock ret = new FrameBlock(schema, names);
+
+ return ret;
+ }
+
+ /**
+ *
+ * @param fs
+ * @param path
+ * @throws IOException
+ */
+ protected static void checkValidInputFile(FileSystem fs, Path path)
+ throws IOException
+ {
+ //check non-existing file
+ if( !fs.exists(path) )
+ throw new IOException("File "+path.toString()+" does not exist on HDFS/LFS.");
+
+ //check for empty file
+ if( MapReduceTool.isFileEmpty( fs, path.toString() ) )
+ throw new EOFException("Empty input file "+ path.toString() +".");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
new file mode 100644
index 0000000..6769d21
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+public class FrameReaderBinaryBlock extends FrameReader
+{
+ /**
+ *
+ * @param fname
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @return
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ @Override
+ public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+ long rlen, long clen)
+ throws IOException, DMLRuntimeException
+ {
+ //allocate output frame block
+ FrameBlock ret = null;
+
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
new file mode 100644
index 0000000..535e7a7
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+
+/**
+ *
+ *
+ */
+public class FrameReaderFactory
+{
+ /**
+ *
+ * @param iinfo
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static FrameReader createFrameReader( InputInfo iinfo )
+ throws DMLRuntimeException
+ {
+ FrameReader reader = null;
+
+ if( iinfo == InputInfo.TextCellInputInfo )
+ {
+ reader = new FrameReaderTextCell();
+ }
+ else if( iinfo == InputInfo.CSVInputInfo )
+ {
+ reader = new FrameReaderTextCSV(new CSVFileFormatProperties());
+ }
+ else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
+ reader = new FrameReaderBinaryBlock();
+ }
+ else {
+ throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
+ + InputInfo.inputInfoToString(iinfo));
+ }
+
+ return reader;
+ }
+
+ /**
+ *
+ * @param props
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static FrameReader createFrameReader( ReadProperties props )
+ throws DMLRuntimeException
+ {
+ //check valid read properties
+ if( props == null )
+ throw new DMLRuntimeException("Failed to create frame reader with empty properties.");
+
+ FrameReader reader = null;
+ InputInfo iinfo = props.inputInfo;
+
+ if( iinfo == InputInfo.TextCellInputInfo ) {
+ reader = new FrameReaderTextCell();
+ }
+ else if( iinfo == InputInfo.CSVInputInfo ) {
+ reader = new FrameReaderTextCSV( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
+ }
+ else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
+ reader = new FrameReaderBinaryBlock();
+ }
+ else {
+ throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
+ + InputInfo.inputInfoToString(iinfo));
+ }
+
+ return reader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
new file mode 100644
index 0000000..b364916
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+public class FrameReaderTextCSV extends FrameReader
+{
+
+ private CSVFileFormatProperties _props = null;
+
+ public FrameReaderTextCSV(CSVFileFormatProperties props)
+ {
+ _props = props;
+ }
+
+
+ /**
+ *
+ * @param fname
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @return
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ @Override
+ public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+ long rlen, long clen)
+ throws IOException, DMLRuntimeException
+ {
+ //allocate output frame block
+ FrameBlock ret = null;
+
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
new file mode 100644
index 0000000..5488f94
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+public class FrameReaderTextCell extends FrameReader
+{
+
+ /**
+ *
+ * @param fname
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @return
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ @Override
+ public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+ long rlen, long clen)
+ throws IOException, DMLRuntimeException
+ {
+ //allocate output frame block
+ FrameBlock ret = null;
+
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
new file mode 100644
index 0000000..0dc528a
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+/**
+ * Base class for all format-specific matrix writers. Every writer is required to implement the basic
+ * write functionality but might provide additional custom functionality. Any non-default parameters
+ * (e.g., CSV read properties) should be passed into custom constructors. There is also a factory
+ * for creating format-specific writers.
+ *
+ */
+public abstract class FrameWriter
+{
+
+ /**
+ *
+ * @param src
+ * @param fname
+ * @return
+ * @throws IOException
+ * @throws DMLUnsupportedOperationException
+ * @throws DMLRuntimeException
+ */
+ public abstract void writeFrameToHDFS( FrameBlock src, String fname )
+ throws IOException, DMLRuntimeException, DMLUnsupportedOperationException;
+
+ /**
+ *
+ * @param schema
+ * @param names
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static FrameBlock[] createFrameBlocksForReuse( List<ValueType> schema, List<String> names )
+ throws DMLRuntimeException
+ {
+ FrameBlock frameBlock[] = new FrameBlock[1];
+ frameBlock[0] = new FrameBlock(schema, names);
+
+ return frameBlock;
+ }
+
+ /**
+ *
+ * @param blocks
+ * @return
+ */
+ public static FrameBlock getMatrixBlockForReuse( FrameBlock[] blocks) //TODO do we need this function?
+ {
+ return blocks[ 0 ];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java
new file mode 100644
index 0000000..46e12fa
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+public class FrameWriterBinaryBlock extends FrameWriter
+{
+ /**
+ * @param src
+ * @param fname
+ * @return
+ * @throws IOException
+ * @throws DMLRuntimeException
+ * @throws DMLUnsupportedOperationException
+ */
+ @Override
+ public void writeFrameToHDFS( FrameBlock src, String fname )
+ throws IOException, DMLRuntimeException, DMLUnsupportedOperationException
+ {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
new file mode 100644
index 0000000..578ba36
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+
+/**
+ *
+ *
+ */
+public class FrameWriterFactory
+{
+
+
+ /**
+ *
+ * @param oinfo
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static FrameWriter createFrameWriter( OutputInfo oinfo )
+ throws DMLRuntimeException
+ {
+ return createFrameWriter(oinfo, null);
+ }
+
+ /**
+ *
+ * @param oinfo
+ * @param props
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static FrameWriter createFrameWriter( OutputInfo oinfo, FileFormatProperties props )
+ throws DMLRuntimeException
+ {
+ FrameWriter writer = null;
+
+ if( oinfo == OutputInfo.TextCellOutputInfo ) {
+ writer = new FrameWriterTextCell();
+ }
+ else if( oinfo == OutputInfo.CSVOutputInfo ) {
+ if( props!=null && !(props instanceof CSVFileFormatProperties) )
+ throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
+ writer = new FrameWriterTextCSV((CSVFileFormatProperties)props);
+ }
+ else if( oinfo == OutputInfo.BinaryBlockOutputInfo ) {
+ writer = new FrameWriterBinaryBlock();
+ }
+ else {
+ throw new DMLRuntimeException("Failed to create frame writer for unknown output info: "
+ + OutputInfo.outputInfoToString(oinfo));
+ }
+
+ return writer;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
new file mode 100644
index 0000000..f118b43
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+/**
+ *
+ */
+public class FrameWriterTextCSV extends FrameWriter
+{
+ //blocksize for string concatenation in order to prevent write OOM
+ //(can be set to very large value to disable blocking)
+ public static final int BLOCKSIZE_J = 32; //32 cells (typically ~512B, should be less than write buffer of 1KB)
+
+ protected CSVFileFormatProperties _props = null;
+
+ public FrameWriterTextCSV( CSVFileFormatProperties props ) {
+ _props = props;
+ }
+
+ /**
+ * @param src
+ * @param fname
+ * @return
+ * @throws IOException
+ * @throws DMLRuntimeException
+ * @throws DMLUnsupportedOperationException
+ */
+ @Override
+ public void writeFrameToHDFS( FrameBlock src, String fname )
+ throws IOException, DMLRuntimeException, DMLUnsupportedOperationException
+ {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a939658e/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
new file mode 100644
index 0000000..065a11b
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+public class FrameWriterTextCell extends FrameWriter
+{
+ /**
+ * @param src
+ * @param fname
+ * @return
+ * @throws IOException
+ * @throws DMLRuntimeException
+ * @throws DMLUnsupportedOperationException
+ */
+ @Override
+ public void writeFrameToHDFS( FrameBlock src, String fname )
+ throws IOException, DMLRuntimeException, DMLUnsupportedOperationException
+ {
+ }
+
+}