You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/28 19:04:04 UTC
[1/4] incubator-systemml git commit: [SYSTEMML-570] Refactoring
matrix buffer pool integration (prep frames)
Repository: incubator-systemml
Updated Branches:
refs/heads/master 42e471079 -> de8d49d34
[SYSTEMML-570] Refactoring matrix buffer pool integration (prep frames)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b580a6de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b580a6de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b580a6de
Branch: refs/heads/master
Commit: b580a6dec1906eabd702dbb4a0859665dee1c081
Parents: 42e4710
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Mar 27 16:07:09 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Mar 27 16:07:09 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/lops/runtime/RunMRJobs.java | 2 +-
.../controlprogram/caching/CacheableData.java | 306 ++++++++++++++++---
.../controlprogram/caching/FrameObject.java | 21 +-
.../controlprogram/caching/MatrixObject.java | 276 +++--------------
.../context/ExecutionContext.java | 2 +-
.../context/SparkExecutionContext.java | 2 +-
6 files changed, 300 insertions(+), 309 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b580a6de/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
index a49c61b..f6ae1a7 100644
--- a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
+++ b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
@@ -360,7 +360,7 @@ public class RunMRJobs
mc.getRowsPerBlock(), mc.getColsPerBlock());
}
- outputMatrices[i].setFileExists(true);
+ outputMatrices[i].setHDFSFileExists(true);
if ( inst.getJobType() != JobType.CSV_WRITE && inst.getJobType() != JobType.TRANSFORM) {
// write out metadata file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b580a6de/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 9967f30..7c372d6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -21,6 +21,7 @@ package org.apache.sysml.runtime.controlprogram.caching;
import java.io.File;
import java.io.IOException;
+import java.lang.ref.SoftReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,44 +32,36 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.LazyWriteBuffer.RPolicy;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.util.LocalFileUtils;
/**
* Each object of this class is a cache envelope for some large piece of data
- * called "data blob". (I prefer "blob" to "block" to avoid ambiguity.) For
- * example, the body of a matrix can be the data blob. The term "data blob"
- * refers strictly to the cacheable portion of the data object, often excluding
- * metadata and auxiliary parameters, as defined in the subclasses.
+ * called "cache block". For example, the body of a matrix can be the cache block.
+ * The term cache block refers strictly to the cacheable portion of the data object,
+ * often excluding metadata and auxiliary parameters, as defined in the subclasses.
* Under the protection of the envelope, the data blob may be evicted to
* the file system; then the subclass must set its reference to <code>null</code>
- * to allow Java garbage collection. If other parts of the system continue
+ * to allow Java garbage collection. If other parts of the system continue
* keep references to the data blob, its eviction will not release any memory.
- * To make the eviction meaningful, the rest of the system
- * must dispose of all references prior to giving the permission for eviction.
- *
+ * To make the eviction meaningful, the rest of the system must dispose all references.
*/
-public abstract class CacheableData<T> extends Data
+public abstract class CacheableData<T extends CacheBlock> extends Data
{
private static final long serialVersionUID = -413810592207212835L;
+ /** Global logging instance for all subclasses of CacheableData */
protected static final Log LOG = LogFactory.getLog(CacheableData.class.getName());
+ // global constant configuration parameters
public static final long CACHING_THRESHOLD = 4*1024; //obj not s.t. caching if below threshold [in bytes]
public static final double CACHING_BUFFER_SIZE = 0.15;
public static final RPolicy CACHING_BUFFER_POLICY = RPolicy.FIFO;
public static final boolean CACHING_BUFFER_PAGECACHE = false;
- public static final boolean CACHING_WRITE_CACHE_ON_READ = false;
-
+ public static final boolean CACHING_WRITE_CACHE_ON_READ = false;
public static final String CACHING_COUNTER_GROUP_NAME = "SystemML Caching Counters";
-
-
- //flag indicating if caching is turned on (eviction writes only happen if activeFlag is true)
- private static boolean _activeFlag = false;
-
- public static String cacheEvictionLocalFilePath = null; //set during init
- public static String cacheEvictionLocalFilePrefix = "cache";
- public static final String cacheEvictionLocalFileExtension = ".dat";
+ public static final String CACHEING_EVICTION_FILEEXTENSION = ".dat";
/**
* Defines all possible cache status types for a data blob.
@@ -92,31 +85,65 @@ public abstract class CacheableData<T> extends Data
CACHED,
CACHED_NOWRITE,
};
-
- private static IDSequence _seq = null;
- static
- {
+ /** Global flag indicating if caching is enabled (controls eviction) */
+ private static boolean _activeFlag = false;
+
+ /** Global sequence for generating unique ids. */
+ private static IDSequence _seq = null;
+
+ // Global eviction path and prefix (prefix used for isolation purposes)
+ public static String cacheEvictionLocalFilePath = null; //set during init
+ public static String cacheEvictionLocalFilePrefix = "cache";
+
+ static {
_seq = new IDSequence();
}
-
+
/**
* The unique (JVM-wide) ID of a cacheable data object; to ensure unique IDs across JVMs, we
* concatenate filenames with a unique prefix (map task ID).
*/
private final int _uniqueID;
- /**
- * The cache status of the data blob (whether it can be or is evicted, etc.)
- */
+ /** The cache status of the data blob (whether it can be or is evicted, etc. */
private CacheStatus _cacheStatus = null;
- private int _numReadThreads = 0;
- //additional status flags
- private boolean _cleanupFlag = true; //flag if obj unpinned (cleanup enabled)
+ /** Cache for actual data, evicted by garbage collector. */
+ protected SoftReference<T> _cache = null;
+ /** Container object that holds the actual data. */
+ protected T _data = null;
+
+ /** The name of HDFS file in which the data is backed up. */
+ protected String _hdfsFileName = null; // file name and path
+
+ /**
+ * Flag that indicates whether or not hdfs file exists.It is used
+ * for improving the performance of "rmvar" instruction. When it has
+ * value <code>false</code>, one can skip file system existence
+ * checks which can be expensive.
+ */
+ private boolean _hdfsFileExists = false;
+
+ /** Information relevant to specific external file formats. */
+ private FileFormatProperties _formatProps = null;
/**
+ * <code>true</code> if the in-memory or evicted matrix may be different from
+ * the matrix located at {@link #_hdfsFileName}; <code>false</code> if the two
+ * matrices should be the same.
+ */
+ private boolean _dirtyFlag = false;
+
+
+ // additional private flags and meta data
+ private int _numReadThreads = 0; //number of threads for read from HDFS
+ private boolean _cleanupFlag = true; //flag if obj unpinned (cleanup enabled)
+ private String _varName = ""; //plan variable name
+ private String _cacheFileName = null; //local eviction file name
+
+ /**
* Basic constructor for any cacheable data.
*
* @param dt
@@ -137,8 +164,117 @@ public abstract class CacheableData<T> extends Data
protected CacheableData(CacheableData<T> that) {
this( that.getDataType(), that.getValueType() );
_cleanupFlag = that._cleanupFlag;
+ _hdfsFileName = that._hdfsFileName;
+ _hdfsFileExists = that._hdfsFileExists;
+ _varName = that._varName;
}
+
+ /**
+ * Enables or disables the cleanup of the associated
+ * data object on clearData().
+ *
+ * @param flag
+ */
+ public void enableCleanup(boolean flag) {
+ _cleanupFlag = flag;
+ }
+
+ /**
+ * Indicates if cleanup of the associated data object
+ * is enabled on clearData().
+ *
+ * @return
+ */
+ public boolean isCleanupEnabled() {
+ return _cleanupFlag;
+ }
+
+ /**
+ *
+ * @param s
+ */
+ public void setVarName(String s) {
+ _varName = s;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public String getVarName() {
+ return _varName;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public boolean isHDFSFileExists() {
+ return _hdfsFileExists;
+ }
+
+ /**
+ *
+ * @param flag
+ */
+ public void setHDFSFileExists( boolean flag ) {
+ _hdfsFileExists = flag;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public String getFileName() {
+ return _hdfsFileName;
+ }
+
+ /**
+ *
+ * @param file
+ */
+ public synchronized void setFileName( String file ) {
+ if (!_hdfsFileName.equals( file )) {
+ _hdfsFileName = file;
+ if( !isEmpty(true) )
+ _dirtyFlag = true;
+ }
+ }
+
+ /**
+ * <code>true</code> if the in-memory or evicted matrix may be different from
+ * the matrix located at {@link #_hdfsFileName}; <code>false</code> if the two
+ * matrices are supposed to be the same.
+ */
+ public boolean isDirty() {
+ return _dirtyFlag;
+ }
+
+ /**
+ *
+ * @param flag
+ */
+ public void setDirty(boolean flag) {
+ _dirtyFlag = flag;
+ }
+
+ /**
+ *
+ * @param props
+ */
+ public void setFileFormatProperties(FileFormatProperties props) {
+ _formatProps = props;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public FileFormatProperties getFileFormatProperties() {
+ return _formatProps;
+ }
+
// --------- ABSTRACT HIGH-LEVEL CACHE I/O OPERATIONS ----------
/**
@@ -179,34 +315,69 @@ public abstract class CacheableData<T> extends Data
*/
public abstract void clearData()
throws CacheException;
-
+
/**
*
* @throws CacheException
*/
- public abstract void exportData()
- throws CacheException;
+ public synchronized void exportData()
+ throws CacheException
+ {
+ exportData( -1 );
+ }
/**
- * Enables or disables the cleanup of the associated
- * data object on clearData().
+ * Writes, or flushes, the matrix data to HDFS.
*
- * @param flag
+ * In-Status: EMPTY, EVICTABLE, EVICTED, READ;
+ * Out-Status: EMPTY, EVICTABLE, EVICTED, READ.
+ *
+ * @throws CacheException
*/
- public void enableCleanup(boolean flag) {
- _cleanupFlag = flag;
+ public synchronized void exportData( int replication )
+ throws CacheException
+ {
+ exportData(_hdfsFileName, null, replication, null);
+ _hdfsFileExists = true;
}
/**
- * Indicates if cleanup of the associated data object
- * is enabled on clearData().
*
- * @return
+ * @param fName
+ * @param outputFormat
+ * @throws CacheException
*/
- public boolean isCleanupEnabled() {
- return _cleanupFlag;
+ public synchronized void exportData(String fName, String outputFormat)
+ throws CacheException
+ {
+ exportData(fName, outputFormat, -1, null);
+ }
+
+ /**
+ *
+ * @param fName
+ * @param outputFormat
+ * @param formatProperties
+ * @throws CacheException
+ */
+ public synchronized void exportData(String fName, String outputFormat, FileFormatProperties formatProperties)
+ throws CacheException
+ {
+ exportData(fName, outputFormat, -1, formatProperties);
}
+ /**
+ *
+ * @param fName
+ * @param outputFormat
+ * @param replication
+ * @param formatProperties
+ * @throws CacheException
+ */
+ public abstract void exportData (String fName, String outputFormat, int replication, FileFormatProperties formatProperties)
+ throws CacheException;
+
+
// --------- ABSTRACT LOW-LEVEL CACHE I/O OPERATIONS ----------
/**
@@ -259,12 +430,27 @@ public abstract class CacheableData<T> extends Data
// ------------- IMPLEMENTED CACHE LOGIC METHODS --------------
- protected int getUniqueCacheID()
- {
+ protected int getUniqueCacheID() {
return _uniqueID;
}
/**
+ *
+ */
+ protected String getCacheFilePathAndName () {
+ if( _cacheFileName==null ) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(CacheableData.cacheEvictionLocalFilePath);
+ sb.append(CacheableData.cacheEvictionLocalFilePrefix);
+ sb.append(String.format ("%09d", getUniqueCacheID()));
+ sb.append(CacheableData.CACHEING_EVICTION_FILEEXTENSION);
+ _cacheFileName = sb.toString();
+ }
+
+ return _cacheFileName;
+ }
+
+ /**
* This method "acquires the lock" to ensure that the data blob is in main memory
* (not evicted) while it is being accessed. When called, the method will try to
* restore the blob if it has been evicted. There are two kinds of locks it may
@@ -438,9 +624,36 @@ public abstract class CacheableData<T> extends Data
|| _cacheStatus == CacheStatus.CACHED_NOWRITE);
}
- // --------- STATIC CACHE INIT/CLEANUP OPERATIONS ----------
+ /**
+ * Creates a new cache soft reference to the currently
+ * referenced cache block.
+ */
+ protected void createCache( ) {
+ _cache = new SoftReference<T>( _data );
+ }
+
+ /**
+ * Tries to get the cache block from the cache soft reference
+ * and subsequently clears the cache soft reference if existing.
+ */
+ protected void getCache() {
+ if( _cache !=null ) {
+ _data = _cache.get();
+ clearCache();
+ }
+ }
+
+ /** Clears the cache soft reference if existing. */
+ protected void clearCache() {
+ if( _cache != null ) {
+ _cache.clear();
+ _cache = null;
+ }
+ }
+ // --------- STATIC CACHE INIT/CLEANUP OPERATIONS ----------
+
/**
*
*/
@@ -531,5 +744,4 @@ public abstract class CacheableData<T> extends Data
{
_activeFlag = true;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b580a6de/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index ed131aa..375198d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -19,20 +19,16 @@
package org.apache.sysml.runtime.controlprogram.caching;
+
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
public class FrameObject extends CacheableData<FrameBlock>
{
private static final long serialVersionUID = 1755082174281927785L;
- /** Container object that holds the actual data.*/
- private FrameBlock _data = null;
-
- /** The name of HDFS file in which the data is backed up. */
- private String _hdfsFileName = null; // file name and path
-
/**
*
*/
@@ -56,15 +52,6 @@ public class FrameObject extends CacheableData<FrameBlock>
*/
public FrameObject(FrameObject fo) {
super(fo);
- _hdfsFileName = fo._hdfsFileName;
- }
-
- public void setFileName(String fname) {
- _hdfsFileName = fname;
- }
-
- public String getFileName() {
- return _hdfsFileName;
}
////////////////////////////////////
@@ -113,10 +100,10 @@ public class FrameObject extends CacheableData<FrameBlock>
}
@Override
- public void exportData()
+ public void exportData(String fName, String outputFormat, int replication, FileFormatProperties formatProperties)
throws CacheException
{
- // TODO @Arvind: please integrate writers here for now
+ // TODO @Arvind: please integrate writers here for now
}
////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b580a6de/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index 2ca6031..bd70b0f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -70,39 +70,8 @@ public class MatrixObject extends CacheableData<MatrixBlock>
private static ThreadLocal<Long> sizePinned = new ThreadLocal<Long>() {
@Override protected Long initialValue() { return 0L; }
};
-
- /**
- * Cache for actual data, evicted by garbage collector.
- */
- private SoftReference<MatrixBlock> _cache = null;
/**
- * Container object that holds the actual data.
- */
- private MatrixBlock _data = null;
-
- /**
- * The name of HDFS file in which the data is backed up.
- */
- private String _hdfsFileName = null; // file name and path
-
- /**
- * Flag that indicates whether or not hdfs file exists.
- * It is used for improving the performance of "rmvar" instruction.
- * When it has value <code>false</code>, one can skip invocations to
- * utility functions such as MapReduceTool.deleteFileIfExistOnHDFS(),
- * which can be potentially expensive.
- */
- private boolean _hdfsFileExists = false;
-
- /**
- * <code>true</code> if the in-memory or evicted matrix may be different from
- * the matrix located at {@link #_hdfsFileName}; <code>false</code> if the two
- * matrices should be the same.
- */
- private boolean _dirtyFlag = false;
-
- /**
* Object that holds the metadata associated with the matrix, which
* includes: 1) Matrix dimensions, if available 2) Number of non-zeros, if
* available 3) Block dimensions, if applicable 4) InputInfo -- subsequent
@@ -114,8 +83,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
private MetaData _metaData = null;
//additional names and flags
- private String _varName = ""; //plan variable name
- private String _cacheFileName = null; //local eviction file name
private boolean _requiresLocalWrite = false; //flag if local write for read obj
private boolean _isAcquireFromEmpty = false; //flag if read from status empty
private boolean _updateInPlaceFlag = false; //flag if in-place update
@@ -136,11 +103,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
private String _partitionCacheName = null; //name of cache block
private MatrixBlock _partitionInMemory = null;
- /**
- * Information relevant to specific external file formats
- */
- FileFormatProperties _formatProperties = null;
-
public RDDProperties getRddProperties() {
return _rddProperties;
}
@@ -180,52 +142,34 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//base copy constructor
super(mo);
- _hdfsFileName = mo._hdfsFileName;
- _hdfsFileExists = mo._hdfsFileExists;
-
MatrixFormatMetaData metaOld = (MatrixFormatMetaData)mo.getMetaData();
_metaData = new MatrixFormatMetaData(new MatrixCharacteristics(metaOld.getMatrixCharacteristics()),
metaOld.getOutputInfo(), metaOld.getInputInfo());
- _varName = mo._varName;
_updateInPlaceFlag = mo._updateInPlaceFlag;
_partitioned = mo._partitioned;
_partitionFormat = mo._partitionFormat;
_partitionSize = mo._partitionSize;
_partitionCacheName = mo._partitionCacheName;
}
-
- public void setVarName(String s)
- {
- _varName = s;
- }
-
- public String getVarName()
- {
- return _varName;
- }
@Override
- public void setMetaData(MetaData md)
- {
+ public void setMetaData(MetaData md) {
_metaData = md;
}
@Override
- public MetaData getMetaData()
- {
+ public MetaData getMetaData() {
return _metaData;
}
@Override
- public void removeMetaData()
- {
+ public void removeMetaData() {
_metaData = null;
}
@Override
- public void updateMatrixCharacteristics (MatrixCharacteristics mc)
- {
+ public void updateMatrixCharacteristics (MatrixCharacteristics mc) {
((MatrixDimensionsMetaData)_metaData).setMatrixCharacteristics( mc );
}
@@ -246,39 +190,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
mc.setNonZeros( _data.getNonZeros() );
}
- public void setFileFormatProperties(FileFormatProperties formatProperties) {
- _formatProperties = formatProperties;
- }
-
- public FileFormatProperties getFileFormatProperties() {
- return _formatProperties;
- }
-
- public boolean isFileExists()
- {
- return _hdfsFileExists;
- }
-
- public void setFileExists( boolean flag )
- {
- _hdfsFileExists = flag;
- }
-
- public String getFileName()
- {
- return _hdfsFileName;
- }
-
- public synchronized void setFileName( String file )
- {
- if (!_hdfsFileName.equals (file))
- {
- _hdfsFileName = file;
- if( ! isEmpty(true) )
- _dirtyFlag = true;
- }
- }
-
/**
*
* @return
@@ -355,23 +266,13 @@ public class MatrixObject extends CacheableData<MatrixBlock>
MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
return meta.getMatrixCharacteristics();
}
-
- /**
- * <code>true</code> if the in-memory or evicted matrix may be different from
- * the matrix located at {@link #_hdfsFileName}; <code>false</code> if the two
- * matrices are supposed to be the same.
- */
- public boolean isDirty ()
- {
- return _dirtyFlag;
- }
public String toString()
{
StringBuilder str = new StringBuilder();
str.append("Matrix: ");
str.append(_hdfsFileName + ", ");
- //System.out.println(_hdfsFileName);
+
if ( _metaData instanceof NumItemsByEachReducerMetaData ) {
str.append("NumItemsByEachReducerMetaData");
}
@@ -468,7 +369,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Acquire read "+_varName);
+ LOG.trace("Acquire read "+getVarName());
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
if ( !isAvailableToRead() )
@@ -512,11 +413,11 @@ public class MatrixObject extends CacheableData<MatrixBlock>
_requiresLocalWrite = true;
}
- _dirtyFlag = false;
+ setDirty(false);
}
catch (IOException e)
{
- throw new CacheIOException("Reading of " + _hdfsFileName + " ("+_varName+") failed.", e);
+ throw new CacheIOException("Reading of " + _hdfsFileName + " ("+getVarName()+") failed.", e);
}
_isAcquireFromEmpty = true;
@@ -555,7 +456,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Acquire modify "+_varName);
+ LOG.trace("Acquire modify "+getVarName());
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
if ( !isAvailableToModify() )
@@ -579,14 +480,14 @@ public class MatrixObject extends CacheableData<MatrixBlock>
}
catch (IOException e)
{
- throw new CacheIOException("Reading of " + _hdfsFileName + " ("+_varName+") failed.", e);
+ throw new CacheIOException("Reading of " + _hdfsFileName + " ("+getVarName()+") failed.", e);
}
}
//cache status maintenance
super.acquire( true, _data==null );
updateStatusPinned(true);
- _dirtyFlag = true;
+ setDirty(true);
_isAcquireFromEmpty = false;
if( DMLScript.STATISTICS ){
@@ -614,7 +515,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Acquire modify newdata "+_varName);
+ LOG.trace("Acquire modify newdata "+getVarName());
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
if (! isAvailableToModify ())
@@ -625,7 +526,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//cache status maintenance
super.acquire (true, false); //no need to load evicted matrix
- _dirtyFlag = true;
+ setDirty(true);
_isAcquireFromEmpty = false;
//set references to new data
@@ -659,7 +560,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Release "+_varName);
+ LOG.trace("Release "+getVarName());
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
boolean write = false;
@@ -667,7 +568,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
{
//set flags for write
write = true;
- _dirtyFlag = true;
+ setDirty(true);
//update meta data
refreshMetaData();
@@ -691,11 +592,11 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//evict blob
String filePath = getCacheFilePathAndName();
try {
- writeMatrix (filePath);
+ LazyWriteBuffer.writeBlock(filePath, _data);
}
catch (Exception e)
{
- throw new CacheException("Eviction to local path " + filePath + " ("+_varName+") failed.", e);
+ throw new CacheException("Eviction to local path " + filePath + " ("+getVarName()+") failed.", e);
}
_requiresLocalWrite = false;
}
@@ -705,7 +606,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
_data = null;
}
else if( LOG.isTraceEnabled() ){
- LOG.trace("Var "+_varName+" not subject to caching: rows="+_data.getNumRows()+", cols="+_data.getNumColumns()+", state="+getStatusAsString());
+ LOG.trace("Var "+getVarName()+" not subject to caching: rows="+_data.getNumRows()+", cols="+_data.getNumColumns()+", state="+getStatusAsString());
}
if( DMLScript.STATISTICS ){
@@ -728,7 +629,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Clear data "+_varName);
+ LOG.trace("Clear data "+getVarName());
// check if cleanup enabled and possible
if( !isCleanupEnabled() )
@@ -752,57 +653,10 @@ public class MatrixObject extends CacheableData<MatrixBlock>
_bcHandle.setBackReference(null);
// change object state EMPTY
- _dirtyFlag = false;
+ setDirty(false);
setEmpty();
}
- @Override
- public synchronized void exportData()
- throws CacheException
- {
- exportData( -1 );
- }
-
- /**
- * Writes, or flushes, the matrix data to HDFS.
- *
- * In-Status: EMPTY, EVICTABLE, EVICTED, READ;
- * Out-Status: EMPTY, EVICTABLE, EVICTED, READ.
- *
- * @throws CacheException
- */
- public synchronized void exportData( int replication )
- throws CacheException
- {
- exportData(_hdfsFileName, null, replication, null);
- _hdfsFileExists = true;
- }
-
- /**
- *
- * @param fName
- * @param outputFormat
- * @param formatProperties
- * @throws CacheException
- */
- public synchronized void exportData (String fName, String outputFormat, FileFormatProperties formatProperties)
- throws CacheException
- {
- exportData(fName, outputFormat, -1, formatProperties);
- }
-
- /**
- *
- * @param fName
- * @param outputFormat
- * @throws CacheException
- */
- public synchronized void exportData (String fName, String outputFormat)
- throws CacheException
- {
- exportData(fName, outputFormat, -1, null);
- }
-
/**
* Synchronized because there might be parallel threads (parfor local) that
* access the same MatrixObjectNew object (in case it was created before the loop).
@@ -818,11 +672,12 @@ public class MatrixObject extends CacheableData<MatrixBlock>
* @param outputFormat
* @throws CacheException
*/
+ @Override
public synchronized void exportData (String fName, String outputFormat, int replication, FileFormatProperties formatProperties)
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Export data "+_varName+" "+fName);
+ LOG.trace("Export data "+getVarName()+" "+fName);
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
//prevent concurrent modifications
@@ -833,7 +688,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
boolean pWrite = false; // !fName.equals(_hdfsFileName); //persistent write flag
if ( fName.equals(_hdfsFileName) ) {
- _hdfsFileExists = true;
+ setHDFSFileExists(true);
pWrite = false;
}
else {
@@ -856,11 +711,11 @@ public class MatrixObject extends CacheableData<MatrixBlock>
_data = readMatrixFromHDFS( _hdfsFileName );
else
_data = readMatrixFromRDD( getRDDHandle(), new MutableBoolean() );
- _dirtyFlag = false;
+ setDirty(false);
}
catch (IOException e)
{
- throw new CacheIOException("Reading of " + _hdfsFileName + " ("+_varName+") failed.", e);
+ throw new CacheIOException("Reading of " + _hdfsFileName + " ("+getVarName()+") failed.", e);
}
}
//get object from cache
@@ -874,7 +729,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
writeMetaData( fName, outputFormat, formatProperties );
writeMatrixToHDFS( fName, outputFormat, replication, formatProperties );
if ( !pWrite )
- _dirtyFlag = false;
+ setDirty(false);
}
catch (Exception e)
{
@@ -1038,7 +893,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
throws CacheException
{
if( LOG.isTraceEnabled() )
- LOG.trace("Acquire partition "+_varName+" "+pred);
+ LOG.trace("Acquire partition "+getVarName()+" "+pred);
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
if ( !_partitioned )
@@ -1214,14 +1069,14 @@ public class MatrixObject extends CacheableData<MatrixBlock>
long begin = 0;
if( LOG.isTraceEnabled() ) {
- LOG.trace("RESTORE of Matrix "+_varName+", "+_hdfsFileName);
+ LOG.trace("RESTORE of Matrix "+getVarName()+", "+_hdfsFileName);
begin = System.currentTimeMillis();
}
String filePath = getCacheFilePathAndName();
if( LOG.isTraceEnabled() )
- LOG.trace ("CACHE: Restoring matrix... " + _varName + " HDFS path: " +
+ LOG.trace ("CACHE: Restoring matrix... " + getVarName() + " HDFS path: " +
(_hdfsFileName == null ? "null" : _hdfsFileName) + ", Restore from path: " + filePath);
if (_data != null)
@@ -1250,7 +1105,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
String cacheFilePathAndName = getCacheFilePathAndName();
long begin = 0;
if( LOG.isTraceEnabled() ){
- LOG.trace("CACHE: Freeing evicted matrix... " + _varName + " HDFS path: " +
+ LOG.trace("CACHE: Freeing evicted matrix... " + getVarName() + " HDFS path: " +
(_hdfsFileName == null ? "null" : _hdfsFileName) + " Eviction path: " + cacheFilePathAndName);
begin = System.currentTimeMillis();
}
@@ -1289,25 +1144,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
/**
*
- */
- private String getCacheFilePathAndName ()
- {
- if( _cacheFileName==null )
- {
- StringBuilder sb = new StringBuilder();
- sb.append(CacheableData.cacheEvictionLocalFilePath);
- sb.append(CacheableData.cacheEvictionLocalFilePrefix);
- sb.append(String.format ("%09d", getUniqueCacheID()));
- sb.append(CacheableData.cacheEvictionLocalFileExtension);
-
- _cacheFileName = sb.toString();
- }
-
- return _cacheFileName;
- }
-
- /**
- *
* @param filePathAndName
* @return
* @throws IOException
@@ -1438,14 +1274,14 @@ public class MatrixObject extends CacheableData<MatrixBlock>
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
if( LOG.isTraceEnabled() ){
- LOG.trace("Reading matrix from HDFS... " + _varName + " Path: " + filePathAndName
+ LOG.trace("Reading matrix from HDFS... " + getVarName() + " Path: " + filePathAndName
+ ", dimensions: [" + mc.getRows() + ", " + mc.getCols() + ", " + mc.getNonZeros() + "]");
begin = System.currentTimeMillis();
}
double sparsity = ( mc.getNonZeros() >= 0 ? ((double)mc.getNonZeros())/(mc.getRows()*mc.getCols()) : 1.0d) ; //expected sparsity
MatrixBlock newData = DataConverter.readMatrixFromHDFS(filePathAndName, iimd.getInputInfo(),
- rlen, clen, mc.getRowsPerBlock(), mc.getColsPerBlock(), sparsity, _formatProperties);
+ rlen, clen, mc.getRowsPerBlock(), mc.getColsPerBlock(), sparsity, getFileFormatProperties());
//sanity check correct output
if( newData == null ) {
@@ -1457,18 +1293,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
return newData;
}
-
- /**
- *
- * @param filePathAndName
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void writeMatrix (String filePathAndName)
- throws DMLRuntimeException, IOException
- {
- LazyWriteBuffer.writeBlock(filePathAndName, _data);
- }
/**
* Writes in-memory matrix to HDFS in a specified format.
@@ -1481,7 +1305,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
{
long begin = 0;
if( LOG.isTraceEnabled() ){
- LOG.trace (" Writing matrix to HDFS... " + _varName + " Path: " + filePathAndName + ", Format: " +
+ LOG.trace (" Writing matrix to HDFS... " + getVarName() + " Path: " + filePathAndName + ", Format: " +
(outputFormat != null ? outputFormat : "inferred from metadata"));
begin = System.currentTimeMillis();
}
@@ -1593,7 +1417,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
String debugNameEnding = (_hdfsFileName == null ? "null" :
(_hdfsFileName.length() < maxLength ? _hdfsFileName : "..." +
_hdfsFileName.substring (_hdfsFileName.length() - maxLength + 3)));
- return _varName + " " + debugNameEnding;
+ return getVarName() + " " + debugNameEnding;
}
@@ -1606,38 +1430,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
/**
*
- */
- private void createCache( )
- {
- _cache = new SoftReference<MatrixBlock>( _data );
- }
-
- /**
- *
- */
- private void getCache()
- {
- if( _cache !=null )
- {
- _data = _cache.get();
- clearCache();
- }
- }
-
- /**
- *
- */
- private void clearCache()
- {
- if( _cache != null )
- {
- _cache.clear();
- _cache = null;
- }
- }
-
- /**
- *
* @param add
*/
private void updateStatusPinned(boolean add) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b580a6de/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index af339db..05d23a7 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -414,7 +414,7 @@ public class ExecutionContext
if ( !getVariables().hasReferences(mo) ) {
//clean cached data
mo.clearData();
- if( mo.isFileExists() )
+ if( mo.isHDFSFileExists() )
{
//clean hdfs data
String fpath = mo.getFileName();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b580a6de/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index 4f9c918..7ee9bb5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -952,7 +952,7 @@ public class SparkExecutionContext extends ExecutionContext
mo.clearData();
//clean hdfs data if no pending rdd operations on it
- if( mo.isFileExists() && mo.getFileName()!=null ) {
+ if( mo.isHDFSFileExists() && mo.getFileName()!=null ) {
if( mo.getRDDHandle()==null ) {
MapReduceTool.deleteFileWithMTDIfExistOnHDFS(mo.getFileName());
}
[4/4] incubator-systemml git commit: [SYSTEMML-399] Performance
parallel textcell reader (nnz, sort sparse)
Posted by mb...@apache.org.
[SYSTEMML-399] Performance parallel textcell reader (nnz, sort sparse)
Similar to parallel binary block readers, this patch improves the
parallel textcell reader to do any post-processing (nnz computation and
sorting of sparse rows) in parallel in order to reduce the serial
fraction of the entire read. This improvement is always beneficial but
especially important on many core machines with a large number of
threads. Furthermore, this change also cleans up the brittle error
handling via proper handling of task futures as well as fixes an issue
with the consolidated buffer pool integration of frames.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/de8d49d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/de8d49d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/de8d49d3
Branch: refs/heads/master
Commit: de8d49d34156fce8f328f8561d30eb01a5e09e0f
Parents: b1e6f16
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Mar 27 22:58:25 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Mon Mar 28 10:03:00 2016 -0700
----------------------------------------------------------------------
.../controlprogram/caching/CacheableData.java | 5 +-
.../apache/sysml/runtime/io/MatrixReader.java | 26 +++
.../runtime/io/ReaderBinaryBlockParallel.java | 23 ---
.../apache/sysml/runtime/io/ReaderTextCell.java | 30 +--
.../runtime/io/ReaderTextCellParallel.java | 183 ++++++++-----------
5 files changed, 114 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/de8d49d3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 7c372d6..b3bd8ff 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -235,11 +235,10 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
* @param file
*/
public synchronized void setFileName( String file ) {
- if (!_hdfsFileName.equals( file )) {
- _hdfsFileName = file;
+ if( _hdfsFileName!=null && !_hdfsFileName.equals(file) )
if( !isEmpty(true) )
_dirtyFlag = true;
- }
+ _hdfsFileName = file;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/de8d49d3/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
index d8a5c2d..e49b0f9 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
import java.io.EOFException;
import java.io.IOException;
import java.util.LinkedList;
+import java.util.concurrent.Callable;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -145,4 +146,29 @@ public abstract class MatrixReader
throw new EOFException("Empty input file "+ path.toString() +".");
}
+
+
+ /**
+ * Utility task for sorting sparse rows as potentially required
+ * by different parallel readers.
+ */
+ protected static class SortRowsTask implements Callable<Object>
+ {
+ private MatrixBlock _dest = null;
+ private int _rl = -1;
+ private int _ru = -1;
+
+ public SortRowsTask(MatrixBlock dest, int rl, int ru) {
+ _dest = dest;
+ _rl = rl;
+ _ru = ru;
+ }
+
+ @Override
+ public Object call() throws Exception {
+ for( int i=_rl; i<_ru; i++ )
+ _dest.getSparseBlock().sort(i);
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/de8d49d3/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
index cdd7b33..5bbb146 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
@@ -239,27 +239,4 @@ public class ReaderBinaryBlockParallel extends ReaderBinaryBlock
return lnnz;
}
}
-
- /**
- *
- */
- private static class SortRowsTask implements Callable<Object>
- {
- private MatrixBlock _dest = null;
- private int _rl = -1;
- private int _ru = -1;
-
- public SortRowsTask(MatrixBlock dest, int rl, int ru) {
- _dest = dest;
- _rl = rl;
- _ru = ru;
- }
-
- @Override
- public Object call() throws Exception {
- for( int i=_rl; i<_ru; i++ )
- _dest.getSparseBlock().sort(i);
- return null;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/de8d49d3/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
index 15e7caf..300d374 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
@@ -43,7 +43,6 @@ import org.apache.sysml.runtime.util.FastStringTokenizer;
public class ReaderTextCell extends MatrixReader
{
-
private boolean _isMMFile = false;
public ReaderTextCell(InputInfo info)
@@ -137,8 +136,7 @@ public class ReaderTextCell extends MatrixReader
{
if( sparse ) //SPARSE<-value
{
- while( reader.next(key, value) )
- {
+ while( reader.next(key, value) ) {
st.reset( value.toString() ); //reinit tokenizer
row = st.nextInt() - 1;
col = st.nextInt() - 1;
@@ -150,8 +148,7 @@ public class ReaderTextCell extends MatrixReader
}
else //DENSE<-value
{
- while( reader.next(key, value) )
- {
+ while( reader.next(key, value) ) {
st.reset( value.toString() ); //reinit tokenizer
row = st.nextInt()-1;
col = st.nextInt()-1;
@@ -160,25 +157,18 @@ public class ReaderTextCell extends MatrixReader
}
}
}
- finally
- {
- if( reader != null )
- reader.close();
+ finally {
+ IOUtilFunctions.closeSilently(reader);
}
}
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
//post-mortem error handling and bounds checking
if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
- {
throw new IOException("Matrix cell ["+(row+1)+","+(col+1)+"] " +
"out of overall matrix range [1:"+rlen+",1:"+clen+"].");
- }
else
- {
throw new IOException( "Unable to read matrix in text cell format.", ex );
- }
}
}
@@ -280,21 +270,15 @@ public class ReaderTextCell extends MatrixReader
}
}
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
//post-mortem error handling and bounds checking
if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
- {
throw new IOException("Matrix cell ["+(row+1)+","+(col+1)+"] " +
"out of overall matrix range [1:"+rlen+",1:"+clen+"].", ex);
- }
else
- {
throw new IOException( "Unable to read matrix in raw text cell format.", ex );
- }
}
- finally
- {
+ finally {
IOUtilFunctions.closeSilently(br);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/de8d49d3/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
index d35ff7c..c071302 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
@@ -21,9 +21,11 @@ package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -35,7 +37,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -64,7 +65,6 @@ import org.apache.sysml.runtime.util.MapReduceTool;
*/
public class ReaderTextCellParallel extends MatrixReader
{
-
private static final long MIN_FILESIZE_MM = 8L * 1024; //8KB
private boolean _isMMFile = false;
@@ -93,14 +93,12 @@ public class ReaderTextCellParallel extends MatrixReader
//core read
readTextCellMatrixFromHDFS(path, job, ret, rlen, clen, brlen, bclen, _isMMFile);
-
- //post-processing (representation-specific, change of sparse/dense block representation)
- if( ret.isInSparseFormat() )
- ret.sortSparseRows();
- else
+
+ //finally check if change of sparse/dense block representation required
+ if( !AGGREGATE_BLOCK_NNZ )
ret.recomputeNonZeros();
ret.examSparsity();
-
+
return ret;
}
@@ -133,12 +131,11 @@ public class ReaderTextCellParallel extends MatrixReader
par = ( len < MIN_FILESIZE_MM ) ? 1: par;
}
- ExecutorService pool = Executors.newFixedThreadPool(par);
- InputSplit[] splits = informat.getSplits(job, par);
-
try
{
//create read tasks for all splits
+ ExecutorService pool = Executors.newFixedThreadPool(par);
+ InputSplit[] splits = informat.getSplits(job, par);
ArrayList<ReadTask> tasks = new ArrayList<ReadTask>();
for( InputSplit split : splits ){
ReadTask t = new ReadTask(split, informat, job, dest, rlen, clen, matrixMarket);
@@ -146,28 +143,35 @@ public class ReaderTextCellParallel extends MatrixReader
}
//wait until all tasks have been executed
- pool.invokeAll(tasks);
- pool.shutdown();
+ List<Future<Long>> rt = pool.invokeAll(tasks);
- //early error notify in case not all tasks successful
- for(ReadTask rt : tasks) {
- if( !rt.getReturnCode() ) {
- throw new IOException("Read task for text input failed: " + rt.getErrMsg());
- }
+ //check for exceptions and aggregate nnz
+ long lnnz = 0;
+ for( Future<Long> task : rt )
+ lnnz += task.get();
+
+ //post-processing
+ dest.setNonZeros( lnnz );
+ if( dest.isInSparseFormat() ) {
+ ArrayList<SortRowsTask> tasks2 = new ArrayList<SortRowsTask>();
+ int blklen = (int)(Math.ceil((double)rlen/_numThreads));
+ for( int i=0; i<_numThreads & i*blklen<rlen; i++ )
+ tasks2.add(new SortRowsTask(dest, i*blklen, Math.min((i+1)*blklen, (int)rlen)));
+ pool.invokeAll(tasks2);
}
-
+
+ pool.shutdown();
}
catch (Exception e) {
throw new IOException("Threadpool issue, while parallel read.", e);
}
-
}
/**
*
*
*/
- public static class ReadTask implements Callable<Object>
+ public static class ReadTask implements Callable<Long>
{
private InputSplit _split = null;
private boolean _sparse = false;
@@ -178,9 +182,6 @@ public class ReaderTextCellParallel extends MatrixReader
private long _clen = -1;
private boolean _matrixMarket = false;
- private boolean _rc = true;
- private String _errMsg = null;
-
public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, MatrixBlock dest, long rlen, long clen, boolean matrixMarket )
{
_split = split;
@@ -193,17 +194,11 @@ public class ReaderTextCellParallel extends MatrixReader
_matrixMarket = matrixMarket;
}
- public boolean getReturnCode() {
- return _rc;
- }
-
- public String getErrMsg() {
- return _errMsg;
- }
-
@Override
- public Object call() throws Exception
+ public Long call() throws Exception
{
+ long lnnz = 0; //aggregate block nnz
+
//writables for reuse during read
LongWritable key = new LongWritable();
Text value = new Text();
@@ -212,14 +207,14 @@ public class ReaderTextCellParallel extends MatrixReader
int row = -1;
int col = -1;
+ FastStringTokenizer st = new FastStringTokenizer(' ');
+ RecordReader<LongWritable,Text> reader = _informat.getRecordReader(_split, _job, Reporter.NULL);
+
try
{
- FastStringTokenizer st = new FastStringTokenizer(' ');
- RecordReader<LongWritable,Text> reader = _informat.getRecordReader(_split, _job, Reporter.NULL);
// Read the header lines, if reading from a matrixMarket file
- if ( _matrixMarket ) {
-
+ if ( _matrixMarket ) {
// skip until end-of-comments (%% or %)
boolean foundComment = false;
while( reader.next(key, value) && value.toString().charAt(0) == '%' ) {
@@ -235,76 +230,61 @@ public class ReaderTextCellParallel extends MatrixReader
double lvalue = st.nextDoubleForParallel();
synchronized( _dest ){ //sparse requires lock
_dest.appendValue(row, col, lvalue);
+ lnnz++;
}
}
}
-
- try
+ if( _sparse ) //SPARSE<-value
{
- if( _sparse ) //SPARSE<-value
- {
- CellBuffer buff = new CellBuffer();
-
- while( reader.next(key, value) )
- {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt() - 1;
- col = st.nextInt() - 1;
- double lvalue = st.nextDoubleForParallel();
-
- buff.addCell(row, col, lvalue);
- //capacity buffer flush on demand
- if( buff.size()>=CellBuffer.CAPACITY )
- synchronized( _dest ){ //sparse requires lock
- buff.flushCellBufferToMatrixBlock(_dest);
- }
- }
+ CellBuffer buff = new CellBuffer();
+
+ while( reader.next(key, value) ) {
+ st.reset( value.toString() ); //reinit tokenizer
+ row = st.nextInt() - 1;
+ col = st.nextInt() - 1;
+ double lvalue = st.nextDoubleForParallel();
- //final buffer flush
- synchronized( _dest ){ //sparse requires lock
- buff.flushCellBufferToMatrixBlock(_dest);
- }
- }
- else //DENSE<-value
- {
- while( reader.next(key, value) )
- {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- double lvalue = st.nextDoubleForParallel();
- _dest.setValueDenseUnsafe( row, col, lvalue );
- }
+ buff.addCell(row, col, lvalue);
+ //capacity buffer flush on demand
+ if( buff.size()>=CellBuffer.CAPACITY )
+ synchronized( _dest ){ //sparse requires lock
+ lnnz += buff.size();
+ buff.flushCellBufferToMatrixBlock(_dest);
+ }
}
- }
- finally
+
+ //final buffer flush
+ synchronized( _dest ){ //sparse requires lock
+ lnnz += buff.size();
+ buff.flushCellBufferToMatrixBlock(_dest);
+ }
+ }
+ else //DENSE<-value
{
- if( reader != null )
- reader.close();
+ while( reader.next(key, value) ) {
+ st.reset( value.toString() ); //reinit tokenizer
+ row = st.nextInt()-1;
+ col = st.nextInt()-1;
+ double lvalue = st.nextDoubleForParallel();
+ _dest.setValueDenseUnsafe( row, col, lvalue );
+ lnnz += (lvalue!=0) ? 1 : 0;
+ }
}
}
- catch(Exception ex)
- {
- //central error handling (return code, message)
- _rc = false;
- _errMsg = ex.getMessage();
-
+ catch(Exception ex) {
//post-mortem error handling and bounds checking
if( row < 0 || row + 1 > _rlen || col < 0 || col + 1 > _clen )
- {
- _errMsg = "Matrix cell ["+(row+1)+","+(col+1)+"] " +
- "out of overall matrix range [1:"+_rlen+",1:"+_clen+"]. "+ex.getMessage();
- throw new RuntimeException(_errMsg, ex);
- }
+ throw new RuntimeException("Matrix cell ["+(row+1)+","+(col+1)+"] " +
+ "out of overall matrix range [1:"+_rlen+",1:"+_clen+"]. ", ex);
else
- {
- _errMsg = "Unable to read matrix in text cell format. "+ex.getMessage();
- throw new RuntimeException(_errMsg, ex );
- }
+ throw new RuntimeException("Unable to read matrix in text cell format. ", ex);
+ }
+ finally {
+ IOUtilFunctions.closeSilently(reader);
}
- return null;
+ return lnnz;
}
}
@@ -315,44 +295,39 @@ public class ReaderTextCellParallel extends MatrixReader
*/
public static class CellBuffer
{
- public static final int CAPACITY = 102400; //100K elements
+ public static final int CAPACITY = 100*1024; //100K elements
private int[] _rlen;
private int[] _clen;
private double[] _vals;
private int _pos;
- public CellBuffer( )
- {
+ public CellBuffer( ) {
_rlen = new int[CAPACITY];
_clen = new int[CAPACITY];
_vals = new double[CAPACITY];
_pos = -1;
}
- public void addCell(int rlen, int clen, double val)
- {
+ public void addCell(int rlen, int clen, double val) {
+ if( val==0 ) return;
_pos++;
_rlen[_pos] = rlen;
_clen[_pos] = clen;
_vals[_pos] = val;
}
- public void flushCellBufferToMatrixBlock( MatrixBlock dest )
- {
+ public void flushCellBufferToMatrixBlock( MatrixBlock dest ) {
for( int i=0; i<=_pos; i++ )
dest.appendValue(_rlen[i], _clen[i], _vals[i]);
-
reset();
}
- public int size()
- {
+ public int size() {
return _pos+1;
}
- public void reset()
- {
+ public void reset() {
_pos = -1;
}
}
[2/4] incubator-systemml git commit: [SYSTEMML-570] Fix mlcontext
(handling of csv format properties)
Posted by mb...@apache.org.
[SYSTEMML-570] Fix mlcontext (handling of csv format properties)
With the mlcontext API a new RDDProperties class has been introduced to
capture CSV file format properties on register input. This is absolutely
redundant as the rest of the system maintains this as
CSVFileFormatProperties (subclass of FileFormatProperties), which is
already under the control of the buffer pool.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/1b8a027b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/1b8a027b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/1b8a027b
Branch: refs/heads/master
Commit: 1b8a027b8d73b514e8855759d71be87c7e5bfabf
Parents: b580a6d
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Mar 27 17:02:18 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Mar 27 17:02:18 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/api/MLContext.java | 40 +++++++-------
.../controlprogram/caching/MatrixObject.java | 10 ----
.../spark/CSVReblockSPInstruction.java | 18 +++----
.../spark/ReblockSPInstruction.java | 21 ++++----
.../instructions/spark/data/RDDProperties.java | 57 --------------------
5 files changed, 37 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1b8a027b/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 01fd8ee..ac23559 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -66,7 +66,6 @@ import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.Instruction;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
-import org.apache.sysml.runtime.instructions.spark.data.RDDProperties;
import org.apache.sysml.runtime.instructions.spark.functions.ConvertStringToLongTextPair;
import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockPairFunction;
import org.apache.sysml.runtime.instructions.spark.functions.CopyTextInputFunction;
@@ -74,6 +73,8 @@ import org.apache.sysml.runtime.instructions.spark.functions.SparkListener;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
@@ -296,12 +297,12 @@ public class MLContext {
* @param hasHeader
* @param delim
* @param fill
- * @param missingValue
+ * @param fillValue
* @throws DMLRuntimeException
*/
public void registerInput(String varName, JavaRDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue) throws DMLRuntimeException {
- registerInput(varName, rdd, format, hasHeader, delim, fill, missingValue, -1, -1, -1);
+ String delim, boolean fill, double fillValue) throws DMLRuntimeException {
+ registerInput(varName, rdd, format, hasHeader, delim, fill, fillValue, -1, -1, -1);
}
/**
@@ -316,12 +317,12 @@ public class MLContext {
* @param hasHeader
* @param delim
* @param fill
- * @param missingValue
+ * @param fillValue
* @throws DMLRuntimeException
*/
public void registerInput(String varName, RDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD(), format, hasHeader, delim, fill, missingValue, -1, -1, -1);
+ String delim, boolean fill, double fillValue) throws DMLRuntimeException {
+ registerInput(varName, rdd.toJavaRDD(), format, hasHeader, delim, fill, fillValue, -1, -1, -1);
}
/**
@@ -336,15 +337,15 @@ public class MLContext {
* @param hasHeader
* @param delim
* @param fill
- * @param missingValue
+ * @param fillValue
* @param rlen
* @param clen
* @param nnz
* @throws DMLRuntimeException
*/
public void registerInput(String varName, RDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue, long rlen, long clen, long nnz) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD(), format, hasHeader, delim, fill, missingValue, -1, -1, -1);
+ String delim, boolean fill, double fillValue, long rlen, long clen, long nnz) throws DMLRuntimeException {
+ registerInput(varName, rdd.toJavaRDD(), format, hasHeader, delim, fill, fillValue, -1, -1, -1);
}
/**
@@ -359,20 +360,16 @@ public class MLContext {
* @param hasHeader
* @param delim
* @param fill
- * @param missingValue
+ * @param fillValue
* @param rlen
* @param clen
* @param nnz
* @throws DMLRuntimeException
*/
public void registerInput(String varName, JavaRDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue, long rlen, long clen, long nnz) throws DMLRuntimeException {
- RDDProperties properties = new RDDProperties();
- properties.setHasHeader(hasHeader);
- properties.setFill(fill);
- properties.setDelim(delim);
- properties.setMissingValue(missingValue);
- registerInput(varName, rdd.mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, nnz, properties);
+ String delim, boolean fill, double fillValue, long rlen, long clen, long nnz) throws DMLRuntimeException {
+ CSVFileFormatProperties props = new CSVFileFormatProperties(hasHeader, delim, fill, fillValue, "");
+ registerInput(varName, rdd.mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, nnz, props);
}
/**
@@ -476,7 +473,7 @@ public class MLContext {
}
// All CSV related methods call this ... It provides access to dimensions, nnz, file properties.
- private void registerInput(String varName, JavaPairRDD<LongWritable, Text> textOrCsv_rdd, String format, long rlen, long clen, long nnz, RDDProperties properties) throws DMLRuntimeException {
+ private void registerInput(String varName, JavaPairRDD<LongWritable, Text> textOrCsv_rdd, String format, long rlen, long clen, long nnz, FileFormatProperties props) throws DMLRuntimeException {
if(!(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)) {
throw new DMLRuntimeException("The registerInput functionality only supported for spark runtime. Please use MLContext(sc) instead of default constructor.");
}
@@ -510,9 +507,8 @@ public class MLContext {
}
JavaPairRDD<LongWritable, Text> rdd = textOrCsv_rdd.mapToPair(new CopyTextInputFunction());
- if(properties != null) {
- mo.setRddProperties(properties);
- }
+ if(props != null)
+ mo.setFileFormatProperties(props);
mo.setRDDHandle(new RDDObject(rdd, varName));
_variables.put(varName, mo);
_inVarnames.add(varName);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1b8a027b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index bd70b0f..cb553f7 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -35,7 +35,6 @@ import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartition
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.spark.data.BroadcastObject;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
-import org.apache.sysml.runtime.instructions.spark.data.RDDProperties;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
@@ -92,7 +91,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//for lazily evaluated RDDs, and (2) as abstraction for environments that do not necessarily have spark libraries available
private RDDObject _rddHandle = null; //RDD handle
private BroadcastObject _bcHandle = null; //Broadcast handle
- private RDDProperties _rddProperties = null;
/**
* Information relevant to partitioned matrices.
@@ -102,14 +100,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
private int _partitionSize = -1; //indicates n for BLOCKWISE_N
private String _partitionCacheName = null; //name of cache block
private MatrixBlock _partitionInMemory = null;
-
- public RDDProperties getRddProperties() {
- return _rddProperties;
- }
-
- public void setRddProperties(RDDProperties _rddProperties) {
- this._rddProperties = _rddProperties;
- }
/**
* Constructor that takes only the HDFS filename.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1b8a027b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index 19f8b5b..6003a0d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -46,11 +46,11 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
private boolean _hasHeader;
private String _delim;
private boolean _fill;
- private double _missingValue;
+ private double _fillValue;
public CSVReblockSPInstruction(Operator op, CPOperand in, CPOperand out,
int br, int bc, boolean hasHeader, String delim, boolean fill,
- double missingValue, String opcode, String instr)
+ double fillValue, String opcode, String instr)
{
super(op, in, out, opcode, instr);
_brlen = br;
@@ -58,17 +58,15 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
_hasHeader = hasHeader;
_delim = delim;
_fill = fill;
- _missingValue = missingValue;
+ _fillValue = fillValue;
}
public static CSVReblockSPInstruction parseInstruction(String str)
throws DMLRuntimeException
{
String opcode = InstructionUtils.getOpCode(str);
- if( !opcode.equals("csvrblk") ) {
- throw new DMLRuntimeException(
- "Incorrect opcode for CSVReblockSPInstruction:" + opcode);
- }
+ if( !opcode.equals("csvrblk") )
+ throw new DMLRuntimeException("Incorrect opcode for CSVReblockSPInstruction:" + opcode);
// Example parts of CSVReblockSPInstruction:
// [csvrblk, pREADmissing_val_maps·MATRIX·DOUBLE, _mVar37·MATRIX·DOUBLE,
@@ -82,10 +80,10 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
boolean hasHeader = Boolean.parseBoolean(parts[5]);
String delim = parts[6];
boolean fill = Boolean.parseBoolean(parts[7]);
- double missingValue = Double.parseDouble(parts[8]);
+ double fillValue = Double.parseDouble(parts[8]);
return new CSVReblockSPInstruction(null, in, out, brlen, bclen,
- hasHeader, delim, fill, missingValue, opcode, str);
+ hasHeader, delim, fill, fillValue, opcode, str);
}
@Override
@@ -124,7 +122,7 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
//reblock csv to binary block
JavaPairRDD<MatrixIndexes, MatrixBlock> out = RDDConverterUtils.csvToBinaryBlock(
- sec.getSparkContext(), in, mcOut, _hasHeader, _delim, _fill, _missingValue);
+ sec.getSparkContext(), in, mcOut, _hasHeader, _delim, _fill, _fillValue);
// put output RDD handle into symbol table
sec.setRDDHandleForVariable(output.getName(), out);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1b8a027b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
index abc54f1..5685e44 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
@@ -22,7 +22,6 @@ package org.apache.sysml.runtime.instructions.spark;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
-
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
@@ -31,12 +30,12 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
-import org.apache.sysml.runtime.instructions.spark.data.RDDProperties;
import org.apache.sysml.runtime.instructions.spark.functions.ExtractBlockForBinaryReblock;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
@@ -123,20 +122,22 @@ public class ReblockSPInstruction extends UnarySPInstruction
else if(iimd.getInputInfo() == InputInfo.CSVInputInfo) {
// HACK ALERT: Until we introduces the rewrite to insert csvrblock for non-persistent read
// throw new DMLRuntimeException("CSVInputInfo is not supported for ReblockSPInstruction");
- RDDProperties properties = mo.getRddProperties();
CSVReblockSPInstruction csvInstruction = null;
boolean hasHeader = false;
String delim = ",";
boolean fill = false;
- double missingValue = 0;
- if(properties != null) {
- hasHeader = properties.isHasHeader();
- delim = properties.getDelim();
- fill = properties.isFill();
- missingValue = properties.getMissingValue();
+ double fillValue = 0;
+ if(mo.getFileFormatProperties() instanceof CSVFileFormatProperties
+ && mo.getFileFormatProperties() != null )
+ {
+ CSVFileFormatProperties props = (CSVFileFormatProperties) mo.getFileFormatProperties();
+ hasHeader = props.hasHeader();
+ delim = props.getDelim();
+ fill = props.isFill();
+ fillValue = props.getFillValue();
}
- csvInstruction = new CSVReblockSPInstruction(null, input1, output, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), hasHeader, delim, fill, missingValue, "csvrblk", instString);
+ csvInstruction = new CSVReblockSPInstruction(null, input1, output, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), hasHeader, delim, fill, fillValue, "csvrblk", instString);
csvInstruction.processInstruction(sec);
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1b8a027b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
deleted file mode 100644
index eea1d99..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.data;
-
-/**
- * Class which allows the MLContext to pass properties for the RDD
- *
- */
-public class RDDProperties {
-
- private boolean hasHeader;
- private String delim;
- private boolean fill;
- private double missingValue;
-
- public boolean isHasHeader() {
- return hasHeader;
- }
- public void setHasHeader(boolean hasHeader) {
- this.hasHeader = hasHeader;
- }
- public String getDelim() {
- return delim;
- }
- public void setDelim(String delim) {
- this.delim = delim;
- }
- public boolean isFill() {
- return fill;
- }
- public void setFill(boolean fill) {
- this.fill = fill;
- }
- public double getMissingValue() {
- return missingValue;
- }
- public void setMissingValue(double missingValue) {
- this.missingValue = missingValue;
- }
-}
[3/4] incubator-systemml git commit: [SYSTEMML-510] New rewrite
'canonicalize matmult-add-scalar', incl tests
Posted by mb...@apache.org.
[SYSTEMML-510] New rewrite 'canonicalize matmult-add-scalar', incl tests
This new static simplification rewrite is a pure canonicalization in
order to bring patterns like U%*%V+eps, eps+U%*%V, and U%*%V-eps into
the normalized representation U%*%V+s which simplifies subsequent
complex rewrites (e.g., wdivmm or wcemm with epsilon). The potential
additional plus operation for U%*%V+(-eps) is unproblematic as this
happens on scalars.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b1e6f162
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b1e6f162
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b1e6f162
Branch: refs/heads/master
Commit: b1e6f162b1c3dd22c751bccc5036401dc889d626
Parents: 1b8a027
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Mar 27 20:45:24 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Mar 27 20:45:24 2016 -0700
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationStatic.java | 49 +++++++++++++++++++-
.../quaternary/WeightedDivMatrixMultTest.java | 21 ++++++++-
.../quaternary/WeightedDivMMLeftEps2.R | 38 +++++++++++++++
.../quaternary/WeightedDivMMLeftEps2.dml | 32 +++++++++++++
.../quaternary/WeightedDivMMLeftEps3.R | 38 +++++++++++++++
.../quaternary/WeightedDivMMLeftEps3.dml | 32 +++++++++++++
6 files changed, 206 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1e6f162/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index 37a0dc2..0cc9f40 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -137,6 +137,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
hi = fuseDatagenAndBinaryOperation(hop, hi, i); //e.g., rand(min=-1,max=1)*7 -> rand(min=-7,max=7)
hi = fuseDatagenAndMinusOperation(hop, hi, i); //e.g., -(rand(min=-2,max=1)) -> rand(min=-1,max=2)
hi = simplifyBinaryToUnaryOperation(hop, hi, i); //e.g., X*X -> X^2 (pow2), X+X -> X*2, (X>0)-(X<0) -> sign(X)
+ hi = canonicalizeMatrixMultScalarAdd(hi); //e.g., eps+U%*%t(V) -> U%*%t(V)+eps, U%*%t(V)-eps -> U%*%t(V)+(-eps)
hi = simplifyReverseOperation(hop, hi, i); //e.g., table(seq(1,nrow(X),1),seq(nrow(X),1,-1)) %*% X -> rev(X)
hi = simplifyMultiBinaryToBinaryOperation(hi); //e.g., 1-X*Y -> X 1-* Y
hi = simplifyDistributiveBinaryOperation(hop, hi, i);//e.g., (X-Y*X) -> (1-Y)*X
@@ -473,8 +474,8 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
}
/**
- * handle simplification of binary operations
- * (relies on previous common subexpression elimination)
+ * Handle simplification of binary operations (relies on previous common subexpression elimination).
+ * At the same time this servers as a canonicalization for more complex rewrites.
*
* X+X -> X*2, X*X -> X^2, (X>0)-(X<0) -> sign(X)
* @throws HopsException
@@ -543,6 +544,50 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
return hi;
}
+ /**
+ * Rewrite to canonicalize all patterns like U%*%V+eps, eps+U%*%V, and
+ * U%*%V-eps into the common representation U%*%V+s which simplifies
+ * subsequent rewrites (e.g., wdivmm or wcemm with epsilon).
+ *
+ * @param parent
+ * @param hi
+ * @param pos
+ * @return
+ * @throws HopsException
+ */
+ private Hop canonicalizeMatrixMultScalarAdd( Hop hi )
+ throws HopsException
+ {
+ //pattern: binary operation (+ or -) of matrix mult and scalar
+ if( hi instanceof BinaryOp )
+ {
+ BinaryOp bop = (BinaryOp)hi;
+ Hop left = hi.getInput().get(0);
+ Hop right = hi.getInput().get(1);
+
+ //pattern: (eps + U%*%V) -> (U%*%V+eps)
+ if( left.getDataType().isScalar() && right instanceof AggBinaryOp
+ && bop.getOp()==OpOp2.PLUS )
+ {
+ HopRewriteUtils.removeAllChildReferences(bop);
+ HopRewriteUtils.addChildReference(bop, right, 0);
+ HopRewriteUtils.addChildReference(bop, left, 1);
+ LOG.debug("Applied canonicalizeMatrixMultScalarAdd1 (line "+hi.getBeginLine()+").");
+ }
+ //pattern: (U%*%V - eps) -> (U%*%V + (-eps))
+ else if( right.getDataType().isScalar() && left instanceof AggBinaryOp
+ && bop.getOp() == OpOp2.MINUS )
+ {
+ bop.setOp(OpOp2.PLUS);
+ HopRewriteUtils.removeChildReferenceByPos(bop, right, 1);
+ HopRewriteUtils.addChildReference(bop,
+ HopRewriteUtils.createBinary(new LiteralOp(0), right, OpOp2.MINUS), 1);
+ LOG.debug("Applied canonicalizeMatrixMultScalarAdd2 (line "+hi.getBeginLine()+").");
+ }
+ }
+
+ return hi;
+ }
/**
* NOTE: this would be by definition a dynamic rewrite; however, we apply it as a static
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1e6f162/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
index aa7e667..6ff8de6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
@@ -58,6 +58,8 @@ public class WeightedDivMatrixMultTest extends AutomatedTestBase
private final static String TEST_NAME9 = "WeightedDivMM4MultMinusRight";
private final static String TEST_NAME10 = "WeightedDivMMLeftEps";
private final static String TEST_NAME11 = "WeightedDivMMRightEps";
+ private final static String TEST_NAME12 = "WeightedDivMMLeftEps2";
+ private final static String TEST_NAME13 = "WeightedDivMMLeftEps3";
private final static String TEST_DIR = "functions/quaternary/";
private final static String TEST_CLASS_DIR = TEST_DIR + WeightedDivMatrixMultTest.class.getSimpleName() + "/";
@@ -85,7 +87,9 @@ public class WeightedDivMatrixMultTest extends AutomatedTestBase
addTestConfiguration(TEST_NAME9,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME9,new String[]{"R"}));
addTestConfiguration(TEST_NAME10,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME10,new String[]{"R"}));
addTestConfiguration(TEST_NAME11,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME11,new String[]{"R"}));
-
+ addTestConfiguration(TEST_NAME12,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME12,new String[]{"R"}));
+ addTestConfiguration(TEST_NAME13,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME13,new String[]{"R"}));
+
if (TEST_CACHE_ENABLED) {
setOutAndExpectedDeletionDisabled(true);
}
@@ -552,6 +556,18 @@ public class WeightedDivMatrixMultTest extends AutomatedTestBase
runWeightedDivMMTest(TEST_NAME11, false, true, true, ExecType.SPARK);
}
+ //d) testcases for wdivmm w/ DIVIDE LEFT/RIGHT with Epsilon
+
+ @Test
+ public void testWeightedDivMMLeftEpsCanonicalized1SparseCP() {
+ runWeightedDivMMTest(TEST_NAME12, true, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testWeightedDivMMLeftEpsCanonicalized2SparseCP() {
+ runWeightedDivMMTest(TEST_NAME13, true, true, false, ExecType.CP);
+ }
+
/**
*
* @param sparseM1
@@ -582,7 +598,8 @@ public class WeightedDivMatrixMultTest extends AutomatedTestBase
boolean basic = testname.equals(TEST_NAME3);
boolean left = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME4)
|| testname.equals(TEST_NAME6) || testname.equals(TEST_NAME8)
- || testname.equals(TEST_NAME10);
+ || testname.equals(TEST_NAME10) || testname.equals(TEST_NAME12)
+ || testname.equals(TEST_NAME13);
double sparsity = (sparse) ? spSparse : spDense;
String TEST_NAME = testname;
String TEST_CACHE_DIR = TEST_CACHE_ENABLED ?
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1e6f162/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.R b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.R
new file mode 100644
index 0000000..8c21496
--- /dev/null
+++ b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.R
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+
+library("Matrix")
+
+W = as.matrix(readMM(paste(args[1], "W.mtx", sep="")))
+U = as.matrix(readMM(paste(args[1], "U.mtx", sep="")))
+V = as.matrix(readMM(paste(args[1], "V.mtx", sep="")))
+
+x = as.numeric(args[3])
+
+R = t(t(U) %*% (W/(x + U%*%t(V))));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
+
+
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1e6f162/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.dml b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.dml
new file mode 100644
index 0000000..4dbab66
--- /dev/null
+++ b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps2.dml
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+
+W = read($1);
+U = read($2);
+V = read($3);
+
+x = $5;
+
+R = t(t(U) %*% (W/(x + U%*%t(V))));
+
+write(R, $4);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1e6f162/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.R b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.R
new file mode 100644
index 0000000..1c014d3
--- /dev/null
+++ b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.R
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+
+library("Matrix")
+
+W = as.matrix(readMM(paste(args[1], "W.mtx", sep="")))
+U = as.matrix(readMM(paste(args[1], "U.mtx", sep="")))
+V = as.matrix(readMM(paste(args[1], "V.mtx", sep="")))
+
+x = as.numeric(args[3])
+
+R = t(t(U) %*% (W/(U%*%t(V)-x)));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
+
+
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1e6f162/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.dml b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.dml
new file mode 100644
index 0000000..520aeaa
--- /dev/null
+++ b/src/test/scripts/functions/quaternary/WeightedDivMMLeftEps3.dml
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+
+W = read($1);
+U = read($2);
+V = read($3);
+
+x = $5;
+
+R = t(t(U) %*% (W/(U%*%t(V)-x)));
+
+write(R, $4);