You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/07/11 18:25:21 UTC

incubator-systemml git commit: [SYSTEMML-569] Frame transform encode/decode/apply w/ column name specs

Repository: incubator-systemml
Updated Branches:
  refs/heads/master ab45af17c -> 468e472a6


[SYSTEMML-569] Frame transform encode/decode/apply w/ column name specs

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/468e472a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/468e472a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/468e472a

Branch: refs/heads/master
Commit: 468e472a61d9ae7e1746ca1b48ad43e7ac4d79d8
Parents: ab45af1
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jul 10 21:28:57 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Mon Jul 11 11:25:03 2016 -0700

----------------------------------------------------------------------
 ...ReturnParameterizedBuiltinCPInstruction.java |   5 +-
 .../cp/ParameterizedBuiltinCPInstruction.java   |   7 +-
 ...ReturnParameterizedBuiltinSPInstruction.java |  14 ++-
 .../ParameterizedBuiltinSPInstruction.java      |  16 +--
 .../spark/utils/FrameRDDConverterUtils.java     |  20 ++--
 .../sysml/runtime/transform/BinAgent.java       |   8 +-
 .../sysml/runtime/transform/DummycodeAgent.java |   4 +-
 .../sysml/runtime/transform/MVImputeAgent.java  |   4 +-
 .../sysml/runtime/transform/OmitAgent.java      |   5 +-
 .../sysml/runtime/transform/RecodeAgent.java    |   6 +-
 .../apache/sysml/runtime/transform/TfUtils.java |  13 ++-
 .../transform/decode/DecoderFactory.java        |   6 +-
 .../transform/encode/EncoderFactory.java        |  30 +++---
 .../runtime/transform/meta/TfMetaUtils.java     |  86 ++++++++++-----
 .../functions/jmlc/FrameReadMetaTest.java       |   2 +-
 .../TransformFrameEncodeApplyTest.java          | 107 ++++++++++++++-----
 .../TransformFrameEncodeDecodeTest.java         |  44 ++++++--
 .../input/homes3/homes.tfspec_bin2.json         |   5 +
 .../input/homes3/homes.tfspec_dummy2.json       |   2 +
 .../input/homes3/homes.tfspec_impute2.json      |  10 ++
 .../input/homes3/homes.tfspec_omit2.json        |   2 +
 .../input/homes3/homes.tfspec_recode2.json      |   2 +
 22 files changed, 276 insertions(+), 122 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
index af076f7..05cf19e 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
@@ -20,6 +20,7 @@
 package org.apache.sysml.runtime.instructions.cp;
 
 import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.sysml.parser.Expression.DataType;
 import org.apache.sysml.parser.Expression.ValueType;
@@ -81,11 +82,13 @@ public class MultiReturnParameterizedBuiltinCPInstruction extends ComputationCPI
 		//obtain and pin input frame
 		FrameBlock fin = ec.getFrameInput(input1.getName());
 		String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
+		List<String> colnames = fin.getColumnNames(); 
 		
 		//execute block transform encode
-		Encoder encoder = EncoderFactory.createEncoder(spec, fin.getNumColumns(), null);
+		Encoder encoder = EncoderFactory.createEncoder(spec, colnames, fin.getNumColumns(), null);
 		MatrixBlock data = encoder.encode(fin, new MatrixBlock(fin.getNumRows(), fin.getNumColumns(), false)); //build and apply
 		FrameBlock meta = encoder.getMetaData(new FrameBlock(fin.getNumColumns(), ValueType.STRING));
+		meta.setColumnNames(colnames);
 		
 		//release input and outputs
 		ec.releaseFrameInput(input1.getName());

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index eca1627..a245d10 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -20,6 +20,7 @@
 package org.apache.sysml.runtime.instructions.cp;
 
 import java.util.HashMap;
+import java.util.List;
 
 import org.apache.sysml.lops.Lop;
 import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
@@ -269,9 +270,10 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
 			//acquire locks
 			FrameBlock data = ec.getFrameInput(params.get("target"));
 			FrameBlock meta = ec.getFrameInput(params.get("meta"));		
+			List<String> colNames = data.getColumnNames();
 			
 			//compute transformapply
-			Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), data.getNumColumns(), meta);
+			Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, data.getNumColumns(), meta);
 			MatrixBlock mbout = encoder.apply(data, new MatrixBlock(data.getNumRows(), data.getNumColumns(), false));
 			
 			//release locks
@@ -283,9 +285,10 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
 			//acquire locks
 			MatrixBlock data = ec.getMatrixInput(params.get("target"));
 			FrameBlock meta = ec.getFrameInput(params.get("meta"));
+			List<String> colnames = meta.getColumnNames();
 			
 			//compute transformdecode
-			Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), null, meta);
+			Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta);
 			FrameBlock fbout = decoder.decode(data, new FrameBlock(meta.getNumColumns(), ValueType.STRING));
 			
 			//release locks

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
index a53f673..9b35daa 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
@@ -24,6 +24,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map.Entry;
 
 import org.apache.spark.Accumulator;
@@ -126,9 +127,11 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI
 			String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
 			MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
 			MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
-			
+			List<String> colnames = !TfMetaUtils.isIDSpecification(spec) ?
+					in.lookup(1L).get(0).getColumnNames() : null; 
+					
 			//step 1: build transform meta data
-			Encoder encoderBuild = EncoderFactory.createEncoder(spec, 
+			Encoder encoderBuild = EncoderFactory.createEncoder(spec, colnames,
 					fo.getSchema(), (int)fo.getNumColumns(), null);
 			
 			Accumulator<Long> accMax = sec.getSparkContext().accumulator(0L, new MaxAcc()); 
@@ -148,17 +151,18 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI
 			FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.TextCellInputInfo);
 			FrameBlock meta = reader.readFrameFromHDFS(fometa.getFileName(), accMax.value(), fo.getNumColumns());
 			meta.recomputeColumnCardinality(); //recompute num distinct items per column
+			meta.setColumnNames((colnames!=null)?colnames:meta.getColumnNames());
 			
 			//step 2: transform apply (similar to spark transformapply)
 			//compute omit offset map for block shifts
 			TfOffsetMap omap = null;
-			if( TfMetaUtils.containsOmitSpec(spec) ) {
+			if( TfMetaUtils.containsOmitSpec(spec, colnames) ) {
 				omap = new TfOffsetMap(SparkUtils.toIndexedLong(in.mapToPair(
-					new RDDTransformApplyOffsetFunction(spec)).collect()));
+					new RDDTransformApplyOffsetFunction(spec, colnames)).collect()));
 			}
 				
 			//create encoder broadcast (avoiding replication per task) 
-			Encoder encoder = EncoderFactory.createEncoder(spec, 
+			Encoder encoder = EncoderFactory.createEncoder(spec, colnames,
 					fo.getSchema(), (int)fo.getNumColumns(), meta);
 			mcOut.setDimension(mcIn.getRows()-((omap!=null)?omap.getNumRmRows():0), encoder.getNumCols()); 
 			Broadcast<Encoder> bmeta = sec.getSparkContext().broadcast(encoder);

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
index 25a4078..f880f57 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
@@ -21,6 +21,7 @@ package org.apache.sysml.runtime.instructions.spark;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.function.Function;
@@ -428,16 +429,18 @@ public class ParameterizedBuiltinSPInstruction  extends ComputationSPInstruction
 			FrameBlock meta = sec.getFrameInput(params.get("meta"));		
 			MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(params.get("target"));
 			MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
+			List<String> colnames = !TfMetaUtils.isIDSpecification(params.get("spec")) ?
+					in.lookup(1L).get(0).getColumnNames() : null; 
 			
 			//compute omit offset map for block shifts
 			TfOffsetMap omap = null;
-			if( TfMetaUtils.containsOmitSpec(params.get("spec")) ) {
+			if( TfMetaUtils.containsOmitSpec(params.get("spec"), colnames) ) {
 				omap = new TfOffsetMap(SparkUtils.toIndexedLong(in.mapToPair(
-					new RDDTransformApplyOffsetFunction(params.get("spec"))).collect()));
+					new RDDTransformApplyOffsetFunction(params.get("spec"), colnames)).collect()));
 			}
 				
 			//create encoder broadcast (avoiding replication per task) 
-			Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), 
+			Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colnames,
 					fo.getSchema(), (int)fo.getNumColumns(), meta);
 			mcOut.setDimension(mcIn.getRows()-((omap!=null)?omap.getNumRmRows():0), encoder.getNumCols()); 
 			Broadcast<Encoder> bmeta = sec.getSparkContext().broadcast(encoder);
@@ -460,6 +463,7 @@ public class ParameterizedBuiltinSPInstruction  extends ComputationSPInstruction
 			JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(params.get("target"));
 			MatrixCharacteristics mc = sec.getMatrixCharacteristics(params.get("target"));
 			FrameBlock meta = sec.getFrameInput(params.get("meta"));		
+			List<String> colnames = meta.getColumnNames();
 			
 			//reblock if necessary (clen > bclen)
 			if( mc.getCols() > mc.getNumColBlocks() ) {
@@ -469,7 +473,7 @@ public class ParameterizedBuiltinSPInstruction  extends ComputationSPInstruction
 			}
 			
 			//construct decoder and decode individual matrix blocks
-			Decoder decoder = DecoderFactory.createDecoder(params.get("spec"), null, meta);
+			Decoder decoder = DecoderFactory.createDecoder(params.get("spec"), colnames, null, meta);
 			JavaPairRDD<Long,FrameBlock> out = in.mapToPair(
 					new RDDTransformDecodeFunction(decoder, meta.getNumColumns(), mc.getRowsPerBlock()));
 			
@@ -769,9 +773,9 @@ public class ParameterizedBuiltinSPInstruction  extends ComputationSPInstruction
 		
 		private int[] _omitColList = null;
 		
-		public RDDTransformApplyOffsetFunction(String spec) {
+		public RDDTransformApplyOffsetFunction(String spec, List<String> colnames) {
 			try {
-				_omitColList = TfMetaUtils.parseJsonIDList(spec, TfUtils.TXMETHOD_OMIT);
+				_omitColList = TfMetaUtils.parseJsonIDList(spec, colnames, TfUtils.TXMETHOD_OMIT);
 			} 
 			catch (DMLRuntimeException e) {
 				throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index b63724a..c640d4d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Arrays;
 
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -394,10 +395,9 @@ public class FrameRDDConverterUtils
 		private String _delim = null;
 		private boolean _fill = false;
 		private int _maxRowsPerBlock = -1; 
+		private List<String> _colnames = null;
 		
-		
-		public CSVToBinaryBlockFunction(MatrixCharacteristics mc, boolean hasHeader, String delim, boolean fill)
-		{
+		public CSVToBinaryBlockFunction(MatrixCharacteristics mc, boolean hasHeader, String delim, boolean fill) {
 			_clen = mc.getCols();
 			_hasHeader = hasHeader;
 			_delim = delim;
@@ -420,10 +420,12 @@ public class FrameRDDConverterUtils
 				Tuple2<Text,Long> tmp = arg0.next();
 				String row = tmp._1().toString();
 				long rowix = tmp._2();
-				if(!_hasHeader) 	// In case there is no header, rowindex to be adjusted to base 1.
-					++rowix;
-				if(_hasHeader && rowix == 0)	//Skip header
+				if(!_hasHeader) // In case there is no header, rowindex to be adjusted to base 1.
+					rowix++;
+				if(_hasHeader && rowix == 0) { //Skip header
+					_colnames = Arrays.asList(row.split(_delim));
 					continue;
+				}
 			
 				if( iRowsInBlock == 0 || iRowsInBlock == _maxRowsPerBlock) {
 					if( iRowsInBlock == _maxRowsPerBlock )
@@ -436,7 +438,7 @@ public class FrameRDDConverterUtils
 				String[] parts = IOUtilFunctions.split(row, _delim);
 				boolean emptyFound = false;
 				mb[0].appendRow(parts);
-				++iRowsInBlock;
+				iRowsInBlock++;
 		
 				//sanity check empty cells filled w/ values
 				IOUtilFunctions.checkAndRaiseErrorCSVEmptyField(row, _fill, emptyFound);
@@ -453,7 +455,9 @@ public class FrameRDDConverterUtils
 		{
 			//compute row block index and number of column blocks
 			ix[0] = new LongWritable(rowix);
-			mb[0] = new FrameBlock((int)_clen, ValueType.STRING);		
+			mb[0] = new FrameBlock((int)_clen, ValueType.STRING);
+			if( _colnames != null )
+				mb[0].setColumnNames(_colnames);
 		}
 		
 		// Flushes current state of filled column blocks to output list.

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
index ad7cbfc..ee0e56a 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
@@ -68,10 +68,10 @@ public class BinAgent extends Encoder
 		super( null, clen );
 	}
 	
-	public BinAgent(JSONObject parsedSpec, int clen) 
+	public BinAgent(JSONObject parsedSpec, List<String> colnames, int clen) 
 		throws JSONException, IOException 
 	{
-		this(parsedSpec, clen, false);
+		this(parsedSpec, colnames, clen, false);
 	}
 	
 	/**
@@ -81,7 +81,7 @@ public class BinAgent extends Encoder
 	 * @throws JSONException
 	 * @throws IOException 
 	 */
-	public BinAgent(JSONObject parsedSpec, int clen, boolean colsOnly) 
+	public BinAgent(JSONObject parsedSpec, List<String> colnames, int clen, boolean colsOnly) 
 		throws JSONException, IOException 
 	{
 		super( null, clen );		
@@ -89,7 +89,7 @@ public class BinAgent extends Encoder
 			return;
 		
 		if( colsOnly ) {
-			List<Integer> collist = TfMetaUtils.parseBinningColIDs(parsedSpec);
+			List<Integer> collist = TfMetaUtils.parseBinningColIDs(parsedSpec, colnames);
 			initColList(ArrayUtils.toPrimitive(collist.toArray(new Integer[0])));
 		}
 		else 

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
index aeeb62b..b51d639 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
@@ -64,11 +64,11 @@ public class DummycodeAgent extends Encoder
 		super(list, clen);
 	}
 	
-	public DummycodeAgent(JSONObject parsedSpec, int clen) throws JSONException {
+	public DummycodeAgent(JSONObject parsedSpec, List<String> colnames, int clen) throws JSONException {
 		super(null, clen);
 		
 		if ( parsedSpec.containsKey(TfUtils.TXMETHOD_DUMMYCODE) ) {
-			int[] collist = TfMetaUtils.parseJsonIDList(parsedSpec, TfUtils.TXMETHOD_DUMMYCODE);
+			int[] collist = TfMetaUtils.parseJsonIDList(parsedSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE);
 			initColList(collist);
 		}
 	}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
index 68896ac..1266ced 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
@@ -98,13 +98,13 @@ public class MVImputeAgent extends Encoder
 	public KahanObject[] getMeans_scnomv()   { return _scnomvMeanList; }
 	public CM_COV_Object[] getVars_scnomv()  { return _scnomvVarList; }
 	
-	public MVImputeAgent(JSONObject parsedSpec, int clen) 
+	public MVImputeAgent(JSONObject parsedSpec, List<String> colnames, int clen) 
 		throws JSONException
 	{
 		super(null, clen);
 		
 		//handle column list
-		int[] collist = TfMetaUtils.parseJsonObjectIDList(parsedSpec, TfUtils.TXMETHOD_IMPUTE);
+		int[] collist = TfMetaUtils.parseJsonObjectIDList(parsedSpec, colnames, TfUtils.TXMETHOD_IMPUTE);
 		initColList(collist);
 	
 		//handle method list

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
index 186ac7a..de6d59f 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
@@ -21,6 +21,7 @@ package org.apache.sysml.runtime.transform;
 
 import java.io.IOException;
 import java.util.Iterator;
+import java.util.List;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -49,13 +50,13 @@ public class OmitAgent extends Encoder
 		super(list, clen);
 	}
 	
-	public OmitAgent(JSONObject parsedSpec, int clen) 
+	public OmitAgent(JSONObject parsedSpec, List<String> colnames, int clen) 
 		throws JSONException 
 	{
 		super(null, clen);
 		if (!parsedSpec.containsKey(TfUtils.TXMETHOD_OMIT))
 			return;
-		int[] collist = TfMetaUtils.parseJsonIDList(parsedSpec, TfUtils.TXMETHOD_OMIT);
+		int[] collist = TfMetaUtils.parseJsonIDList(parsedSpec, colnames, TfUtils.TXMETHOD_OMIT);
 		initColList(collist);
 	}
 	

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
index 5abe9db..edfdff4 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
@@ -59,19 +59,19 @@ public class RecodeAgent extends Encoder
 	private HashMap<Integer, HashMap<String, Long>> _rcdMaps  = new HashMap<Integer, HashMap<String, Long>>();
 	private HashMap<Integer, HashMap<String,String>> _finalMaps = null;
 	
-	public RecodeAgent(JSONObject parsedSpec, int clen)
+	public RecodeAgent(JSONObject parsedSpec, List<String> colnames, int clen)
 		throws JSONException 
 	{
 		super(null, clen);
 		int rcdCount = 0;
 		
 		if( parsedSpec.containsKey(TfUtils.TXMETHOD_RECODE) ) {
-			int[] collist = TfMetaUtils.parseJsonIDList(parsedSpec, TfUtils.TXMETHOD_RECODE);
+			int[] collist = TfMetaUtils.parseJsonIDList(parsedSpec, colnames, TfUtils.TXMETHOD_RECODE);
 			rcdCount = initColList(collist);
 		}
 		
 		if ( parsedSpec.containsKey(TfUtils.TXMETHOD_MVRCD)) {
-			_mvrcdList = TfMetaUtils.parseJsonIDList(parsedSpec, TfUtils.TXMETHOD_MVRCD);
+			_mvrcdList = TfMetaUtils.parseJsonIDList(parsedSpec, colnames, TfUtils.TXMETHOD_MVRCD);
 			rcdCount += _mvrcdList.length;
 		}
 		

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
index 24bde22..2b63797 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
@@ -23,6 +23,7 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.filecache.DistributedCache;
@@ -135,7 +136,7 @@ public class TfUtils implements Serializable{
 		}		
 		_NAstrings = TfUtils.parseNAStrings(job);
 		_spec = job.get(MRJobConfiguration.TF_SPEC);
-		_oa = new OmitAgent(new JSONObject(_spec), -1);
+		_oa = new OmitAgent(new JSONObject(_spec), null, -1);
 	}
 	
 	// called from GenTFMtdMapper, ApplyTf (Hadoop)
@@ -244,11 +245,13 @@ public class TfUtils implements Serializable{
 	private void createAgents(JSONObject spec, String[] naStrings) 
 		throws IOException, JSONException 
 	{
-		_oa = new OmitAgent(spec, _numInputCols);
+		List<String> colnames = Arrays.asList(_outputColumnNames);
+		
+		_oa = new OmitAgent(spec, colnames, _numInputCols);
 		_mia = new MVImputeAgent(spec, naStrings, _numInputCols);
-		_ra = new RecodeAgent(spec, _numInputCols);
-		_ba = new BinAgent(spec, _numInputCols);
-		_da = new DummycodeAgent(spec, _numInputCols);
+		_ra = new RecodeAgent(spec, colnames, _numInputCols);
+		_ba = new BinAgent(spec, colnames, _numInputCols);
+		_da = new DummycodeAgent(spec, colnames, _numInputCols);
 	}
 	
 	public void setupAgents(OmitAgent oa, MVImputeAgent mia, RecodeAgent ra, BinAgent ba, DummycodeAgent da)  {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
index 78f1ad2..f276015 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java
@@ -46,7 +46,7 @@ public class DecoderFactory
 	 * @throws DMLRuntimeException
 	 */
 	@SuppressWarnings("unchecked")
-	public static Decoder createDecoder(String spec, List<ValueType> schema, FrameBlock meta) 
+	public static Decoder createDecoder(String spec, List<String> colnames, List<ValueType> schema, FrameBlock meta) 
 		throws DMLRuntimeException 
 	{	
 		Decoder decoder = null;
@@ -64,9 +64,9 @@ public class DecoderFactory
 			
 			//create decoders 'recode', 'dummy' and 'pass-through'
 			List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_RECODE)));
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_RECODE)));
 			List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_DUMMYCODE))); 
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE))); 
 			rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
 			List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils
 					.subtract(UtilFunctions.getSequenceList(1, schema.size(), 1), rcIDs)); 

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
index ae98bae..8adea7b 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java
@@ -48,8 +48,8 @@ public class EncoderFactory
 	 * @return
 	 * @throws DMLRuntimeException 
 	 */
-	public static Encoder createEncoder(String spec, int clen, FrameBlock meta) throws DMLRuntimeException {
-		return createEncoder(spec, Collections.nCopies(clen, ValueType.STRING), meta);
+	public static Encoder createEncoder(String spec, List<String> colnames, int clen, FrameBlock meta) throws DMLRuntimeException {
+		return createEncoder(spec, colnames, Collections.nCopies(clen, ValueType.STRING), meta);
 	}
 	
 	/**
@@ -61,9 +61,9 @@ public class EncoderFactory
 	 * @return
 	 * @throws DMLRuntimeException
 	 */
-	public static Encoder createEncoder(String spec, List<ValueType> schema, int clen, FrameBlock meta) throws DMLRuntimeException {
+	public static Encoder createEncoder(String spec, List<String> colnames, List<ValueType> schema, int clen, FrameBlock meta) throws DMLRuntimeException {
 		List<ValueType> lschema = (schema==null) ? Collections.nCopies(clen, ValueType.STRING) : schema;
-		return createEncoder(spec, lschema, meta);
+		return createEncoder(spec, colnames, lschema, meta);
 	}
 	
 	
@@ -75,7 +75,7 @@ public class EncoderFactory
 	 * @throws DMLRuntimeException
 	 */
 	@SuppressWarnings("unchecked")
-	public static Encoder createEncoder(String spec, List<ValueType> schema, FrameBlock meta) 
+	public static Encoder createEncoder(String spec,  List<String> colnames, List<ValueType> schema, FrameBlock meta) 
 		throws DMLRuntimeException 
 	{	
 		Encoder encoder = null;
@@ -89,21 +89,21 @@ public class EncoderFactory
 			//prepare basic id lists (recode, dummycode, pass-through)
 			//note: any dummycode column requires recode as preparation
 			List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_RECODE)));
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_RECODE)));
 			List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_DUMMYCODE))); 
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE))); 
 			rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
-			List<Integer> binIDs = TfMetaUtils.parseBinningColIDs(jSpec); 
+			List<Integer> binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames); 
 			List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils.subtract(
 					CollectionUtils.subtract(UtilFunctions.getSequenceList(1, clen, 1), rcIDs), binIDs)); 
 			List<Integer> oIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_OMIT))); 
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_OMIT))); 
 			List<Integer> mvIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonObjectIDList(jSpec, TfUtils.TXMETHOD_IMPUTE))); 
+					TfMetaUtils.parseJsonObjectIDList(jSpec, colnames, TfUtils.TXMETHOD_IMPUTE))); 
 			
 			//create individual encoders
 			if( !rcIDs.isEmpty() ) {
-				RecodeAgent ra = new RecodeAgent(jSpec, clen);
+				RecodeAgent ra = new RecodeAgent(jSpec, colnames, clen);
 				ra.setColList(ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])));
 				lencoders.add(ra);	
 			}
@@ -111,13 +111,13 @@ public class EncoderFactory
 				lencoders.add(new EncoderPassThrough(
 						ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), clen));	
 			if( !dcIDs.isEmpty() )
-				lencoders.add(new DummycodeAgent(jSpec, schema.size()));
+				lencoders.add(new DummycodeAgent(jSpec, colnames, schema.size()));
 			if( !binIDs.isEmpty() )
-				lencoders.add(new BinAgent(jSpec, schema.size(), true));
+				lencoders.add(new BinAgent(jSpec, colnames, schema.size(), true));
 			if( !oIDs.isEmpty() )
-				lencoders.add(new OmitAgent(jSpec, schema.size()));
+				lencoders.add(new OmitAgent(jSpec, colnames, schema.size()));
 			if( !mvIDs.isEmpty() ) {
-				MVImputeAgent ma = new MVImputeAgent(jSpec, schema.size());
+				MVImputeAgent ma = new MVImputeAgent(jSpec, colnames, schema.size());
 				ma.initRecodeIDList(rcIDs);
 				lencoders.add(ma);
 			}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
index 82396f6..de883f3 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
@@ -57,23 +57,41 @@ public class TfMetaUtils
 	 * @return
 	 * @throws DMLRuntimeException
 	 */
-	public static boolean containsOmitSpec(String spec) throws DMLRuntimeException {
-		return (TfMetaUtils.parseJsonIDList(spec, TfUtils.TXMETHOD_OMIT).length > 0);	
+	public static boolean isIDSpecification(String spec) throws DMLRuntimeException {
+		try {
+			JSONObject jSpec = new JSONObject(spec);
+			return jSpec.containsKey("ids") && jSpec.getBoolean("ids");
+		}
+		catch(JSONException ex) {
+			throw new DMLRuntimeException(ex);
+		}
+	}
+	
+	/**
+	 * 
+	 * @param spec
+	 * @param colnames
+	 * @return
+	 * @throws DMLRuntimeException
+	 */
+	public static boolean containsOmitSpec(String spec, List<String> colnames) throws DMLRuntimeException {
+		return (TfMetaUtils.parseJsonIDList(spec, colnames, TfUtils.TXMETHOD_OMIT).length > 0);	
 	}
 	
 	/**
 	 * 
 	 * @param spec
+	 * @param colnames
 	 * @param group
 	 * @return
-	 * @throws JSONException
+	 * @throws DMLRuntimeException
 	 */
-	public static int[] parseJsonIDList(String spec, String group) 
+	public static int[] parseJsonIDList(String spec, List<String> colnames, String group) 
 		throws DMLRuntimeException 
 	{
 		try {
 			JSONObject jSpec = new JSONObject(spec);
-			return parseJsonIDList(jSpec, group);
+			return parseJsonIDList(jSpec, colnames, group);
 		}
 		catch(JSONException ex) {
 			throw new DMLRuntimeException(ex);
@@ -82,29 +100,39 @@ public class TfMetaUtils
 	
 	/**
 	 * TODO consolidate external and internal json spec definitions
-	 * 		
-	 * @param parsedSpec
+	 * 
+	 * @param spec
+	 * @param colnames
 	 * @param group
 	 * @return
-	 * @throws JSONException 
+	 * @throws JSONException
 	 */
-	public static int[] parseJsonIDList(JSONObject spec, String group) 
+	public static int[] parseJsonIDList(JSONObject spec, List<String> colnames, String group) 
 		throws JSONException
 	{
 		int[] colList = new int[0];
+		boolean ids = spec.containsKey("ids") && spec.getBoolean("ids");
 		
 		if( spec.containsKey(group) ) {
 			//parse attribute-array or plain array of IDs
 			JSONArray attrs = null;
-			if( spec.get(group) instanceof JSONObject )
+			if( spec.get(group) instanceof JSONObject ) {
 				attrs = (JSONArray) ((JSONObject)spec.get(group)).get(TfUtils.JSON_ATTRS);
+				ids = true; //file-based transform outputs ids w/o id tags
+			}
 			else
 				attrs = (JSONArray)spec.get(group);			
 			
 			//construct ID list array
 			colList = new int[attrs.size()];
-			for(int i=0; i < colList.length; i++) 
-				colList[i] = UtilFunctions.toInt(attrs.get(i));
+			for(int i=0; i < colList.length; i++) {
+				colList[i] = ids ? UtilFunctions.toInt(attrs.get(i)) : 
+					(colnames.indexOf(attrs.get(i)) + 1);
+				if( colList[i] <= 0 ) {
+					throw new RuntimeException("Specified column '" + 
+						attrs.get(i)+"' does not exist.");
+				}
+			}
 		
 			//ensure ascending order of column IDs
 			Arrays.sort(colList);
@@ -120,10 +148,11 @@ public class TfMetaUtils
 	 * @return
 	 * @throws JSONException
 	 */
-	public static int[] parseJsonObjectIDList(JSONObject spec, String group) 
+	public static int[] parseJsonObjectIDList(JSONObject spec, List<String> colnames, String group) 
 		throws JSONException
 	{
 		int[] colList = new int[0];
+		boolean ids = spec.containsKey("ids") && spec.getBoolean("ids");
 		
 		if( spec.containsKey(group) && spec.get(group) instanceof JSONArray )
 		{
@@ -131,7 +160,12 @@ public class TfMetaUtils
 			colList = new int[colspecs.size()];
 			for(int j=0; j<colspecs.size(); j++) {
 				JSONObject colspec = (JSONObject) colspecs.get(j);
-				colList[j] = colspec.getInt("id");
+				colList[j] = ids ? colspec.getInt("id") : 
+					(colnames.indexOf(colspec.get("name")) + 1);
+				if( colList[j] <= 0 ) {
+					throw new RuntimeException("Specified column '" +
+						colspec.get(ids?"id":"name")+"' does not exist.");
+				}
 			}
 			
 			//ensure ascending order of column IDs
@@ -188,8 +222,8 @@ public class TfMetaUtils
 		}
 
 		//get list of recode ids
-		List<Integer> recodeIDs = parseRecodeColIDs(spec);
-		List<Integer> binIDs = parseBinningColIDs(spec);
+		List<Integer> recodeIDs = parseRecodeColIDs(spec, colnames);
+		List<Integer> binIDs = parseBinningColIDs(spec, colnames);
 		
 		//create frame block from in-memory strings
 		return convertToTransformMetaDataFrame(rows, colnames, recodeIDs, binIDs, meta, mvmeta);
@@ -243,8 +277,8 @@ public class TfMetaUtils
 		}
 		
 		//get list of recode ids
-		List<Integer> recodeIDs = parseRecodeColIDs(spec);
-		List<Integer> binIDs = parseBinningColIDs(spec);
+		List<Integer> recodeIDs = parseRecodeColIDs(spec, colnames);
+		List<Integer> binIDs = parseBinningColIDs(spec, colnames);
 		
 		//create frame block from in-memory strings
 		return convertToTransformMetaDataFrame(rows, colnames, recodeIDs, binIDs, meta, mvmeta);
@@ -327,7 +361,7 @@ public class TfMetaUtils
 	 * @throws IOException
 	 */
 	@SuppressWarnings("unchecked")
-	private static List<Integer> parseRecodeColIDs(String spec) 
+	private static List<Integer> parseRecodeColIDs(String spec, List<String> colnames) 
 		throws IOException 
 	{	
 		if( spec == null )
@@ -339,9 +373,9 @@ public class TfMetaUtils
 			//parse json transform specification for recode col ids
 			JSONObject jSpec = new JSONObject(spec);
 			List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_RECODE)));
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_RECODE)));
 			List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(
-					TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_DUMMYCODE))); 
+					TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE))); 
 			specRecodeIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
 		}
 		catch(Exception ex) {
@@ -357,12 +391,12 @@ public class TfMetaUtils
 	 * @return
 	 * @throws IOException
 	 */
-	public static List<Integer> parseBinningColIDs(String spec) 
+	public static List<Integer> parseBinningColIDs(String spec, List<String> colnames) 
 		throws IOException 
 	{
 		try {
 			JSONObject jSpec = new JSONObject(spec);
-			return parseBinningColIDs(jSpec);
+			return parseBinningColIDs(jSpec, colnames);
 		}
 		catch(JSONException ex) {
 			throw new IOException(ex);
@@ -375,17 +409,17 @@ public class TfMetaUtils
 	 * @return
 	 * @throws IOException
 	 */
-	public static List<Integer> parseBinningColIDs(JSONObject jSpec) 
+	public static List<Integer> parseBinningColIDs(JSONObject jSpec, List<String> colnames) 
 		throws IOException 
 	{
 		try {
 			if( jSpec.containsKey(TfUtils.TXMETHOD_BIN) && jSpec.get(TfUtils.TXMETHOD_BIN) instanceof JSONArray ) {
 				return Arrays.asList(ArrayUtils.toObject(
-						TfMetaUtils.parseJsonObjectIDList(jSpec, TfUtils.TXMETHOD_BIN)));	
+						TfMetaUtils.parseJsonObjectIDList(jSpec, colnames, TfUtils.TXMETHOD_BIN)));	
 			}
 			else { //internally generates
 				return Arrays.asList(ArrayUtils.toObject(
-						TfMetaUtils.parseJsonIDList(jSpec, TfUtils.TXMETHOD_BIN)));	
+						TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_BIN)));	
 			}
 		}
 		catch(JSONException ex) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java
index 3fad934..b0eb3ce 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java
@@ -179,7 +179,7 @@ public class FrameReadMetaTest extends AutomatedTestBase
 		throws DMLRuntimeException 
 	{
 		List<Integer> collist = Arrays.asList(ArrayUtils.toObject(
-				TfMetaUtils.parseJsonIDList(spec, TfUtils.TXMETHOD_RECODE)));
+				TfMetaUtils.parseJsonIDList(spec, M.getColumnNames(), TfUtils.TXMETHOD_RECODE)));
 		HashMap<String,Long>[] ret = new HashMap[M.getNumColumns()];
 		Iterator<Object[]> iter = M.getObjectRowIterator();
 		while( iter.hasNext() ) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java
index b61060b..27d58f9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java
@@ -39,13 +39,18 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase
 	//dataset and transform tasks without missing values
 	private final static String DATASET1 	= "homes3/homes.csv";
 	private final static String SPEC1 		= "homes3/homes.tfspec_recode.json"; 
+	private final static String SPEC1b 		= "homes3/homes.tfspec_recode2.json"; 
 	private final static String SPEC2 		= "homes3/homes.tfspec_dummy.json";
+	private final static String SPEC2b 		= "homes3/homes.tfspec_dummy2.json";
 	private final static String SPEC3 		= "homes3/homes.tfspec_bin.json"; //incl recode
+	private final static String SPEC3b 		= "homes3/homes.tfspec_bin2.json"; //incl recode
 	
 	//dataset and transform tasks with missing values
 	private final static String DATASET2 	= "homes/homes.csv";
 	private final static String SPEC4 		= "homes3/homes.tfspec_impute.json";
+	private final static String SPEC4b 		= "homes3/homes.tfspec_impute2.json";
 	private final static String SPEC5 		= "homes3/homes.tfspec_omit.json";
+	private final static String SPEC5b 		= "homes3/homes.tfspec_omit2.json";
 	
 	public enum TransformType {
 		RECODE,
@@ -62,62 +67,112 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase
 	}
 	
 	@Test
-	public void testHomesRecodeSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE);
+	public void testHomesRecodeIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE, false);
 	}
 	
 	@Test
-	public void testHomesRecodeSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE);
+	public void testHomesRecodeIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE, false);
 	}
 	
 	@Test
-	public void testHomesDummycodeSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY);
+	public void testHomesDummycodeIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY, false);
 	}
 	
 	@Test
-	public void testHomesDummycodeSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY);
+	public void testHomesDummycodeIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY, false);
 	}
 	
 	@Test
-	public void testHomesBinningSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.BIN);
+	public void testHomesBinningIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.BIN, false);
 	}
 	
 	@Test
-	public void testHomesBinningSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.BIN);
+	public void testHomesBinningIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.BIN, false);
 	}
 	
 	@Test
-	public void testHomesOmitSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.OMIT);
+	public void testHomesOmitIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.OMIT, false);
 	}
 	
 	@Test
-	public void testHomesOmitSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.OMIT);
+	public void testHomesOmitIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.OMIT, false);
 	}
 	
 	@Test
-	public void testHomesImputeSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.IMPUTE);
+	public void testHomesImputeIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.IMPUTE, false);
 	}
 	
 	@Test
-	public void testHomesImputeSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.IMPUTE);
+	public void testHomesImputeIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.IMPUTE, false);
 	}
 
+	@Test
+	public void testHomesRecodeColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE, true);
+	}
+	
+	@Test
+	public void testHomesRecodeColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE, true);
+	}
+	
+	@Test
+	public void testHomesDummycodeColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY, true);
+	}
+	
+	@Test
+	public void testHomesDummycodeColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY, true);
+	}
+	
+	@Test
+	public void testHomesBinningColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.BIN, true);
+	}
+	
+	@Test
+	public void testHomesBinningColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.BIN, true);
+	}
+	
+	@Test
+	public void testHomesOmitColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.OMIT, true);
+	}
+	
+	@Test
+	public void testHomesOmitvColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.OMIT, true);
+	}
+	
+	@Test
+	public void testHomesImputeColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.IMPUTE, true);
+	}
+	
+	@Test
+	public void testHomesImputeColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.IMPUTE, true);
+	}
+	
 	/**
 	 * 
 	 * @param rt
 	 * @param ofmt
 	 * @param dataset
 	 */
-	private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type )
+	private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type, boolean colnames )
 	{
 		//set runtime platform
 		RUNTIME_PLATFORM rtold = rtplatform;
@@ -131,11 +186,11 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase
 		//set transform specification
 		String SPEC = null; String DATASET = null;
 		switch( type ) {
-			case RECODE: SPEC = SPEC1; DATASET = DATASET1; break;
-			case DUMMY:  SPEC = SPEC2; DATASET = DATASET1; break;
-			case BIN:    SPEC = SPEC3; DATASET = DATASET1; break;
-			case IMPUTE: SPEC = SPEC4; DATASET = DATASET2; break;
-			case OMIT:   SPEC = SPEC5; DATASET = DATASET2; break;
+			case RECODE: SPEC = colnames?SPEC1b:SPEC1; DATASET = DATASET1; break;
+			case DUMMY:  SPEC = colnames?SPEC2b:SPEC2; DATASET = DATASET1; break;
+			case BIN:    SPEC = colnames?SPEC3b:SPEC3; DATASET = DATASET1; break;
+			case IMPUTE: SPEC = colnames?SPEC4b:SPEC4; DATASET = DATASET2; break;
+			case OMIT:   SPEC = colnames?SPEC5b:SPEC5; DATASET = DATASET2; break;
 		}
 
 		if( !ofmt.equals("csv") )

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
index 09f497e..0bdf4da 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java
@@ -42,7 +42,9 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase
 	//dataset and transform tasks without missing values
 	private final static String DATASET1 	= "homes3/homes.csv";
 	private final static String SPEC1 		= "homes3/homes.tfspec_recode.json"; 
+	private final static String SPEC1b 		= "homes3/homes.tfspec_recode2.json"; 
 	private final static String SPEC2 		= "homes3/homes.tfspec_dummy.json";
+	private final static String SPEC2b 		= "homes3/homes.tfspec_dummy2.json";
 	
 	public enum TransformType {
 		RECODE,
@@ -60,23 +62,43 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase
 	}
 	
 	@Test
-	public void testHomesRecodeSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE);
+	public void testHomesRecodeIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE, false);
 	}
 	
 	@Test
-	public void testHomesRecodeSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE);
+	public void testHomesRecodeIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE, false);
 	}
 	
 	@Test
-	public void testHomesDummycodeSingleNodeCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY);
+	public void testHomesDummycodeIDsSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY, false);
 	}
 	
 	@Test
-	public void testHomesDummycodeSparkCSV() {
-		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY);
+	public void testHomesDummycodeIDsSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY, false);
+	}
+	
+	@Test
+	public void testHomesRecodeColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE, true);
+	}
+	
+	@Test
+	public void testHomesRecodeColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE, true);
+	}
+	
+	@Test
+	public void testHomesDummycodeColnamesSingleNodeCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY, true);
+	}
+	
+	@Test
+	public void testHomesDummycodeColnamesSparkCSV() {
+		runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY, true);
 	}
 	
 	/**
@@ -85,7 +107,7 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase
 	 * @param ofmt
 	 * @param dataset
 	 */
-	private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type )
+	private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type, boolean colnames )
 	{
 		//set runtime platform
 		RUNTIME_PLATFORM rtold = rtplatform;
@@ -99,8 +121,8 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase
 		//set transform specification
 		String SPEC = null; String DATASET = null;
 		switch( type ) {
-			case RECODE: SPEC = SPEC1; DATASET = DATASET1; break;
-			case DUMMY:  SPEC = SPEC2; DATASET = DATASET1; break;
+			case RECODE: SPEC = colnames?SPEC1b:SPEC1; DATASET = DATASET1; break;
+			case DUMMY:  SPEC = colnames?SPEC2b:SPEC2; DATASET = DATASET1; break;
 			default: throw new RuntimeException("Unsupported transform type for encode/decode test.");
 		}
 

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_bin2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_bin2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_bin2.json
new file mode 100644
index 0000000..a5a5375
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_bin2.json
@@ -0,0 +1,5 @@
+{
+ "recode": [ zipcode, "district", "view" ], "bin": [
+ { "name": "saleprice"  , "method": "equi-width", "numbins": 3 }
+ ,{ "name": "sqft", "method": "equi-width", "numbins": 4 }]
+  }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
new file mode 100644
index 0000000..917fab6
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
@@ -0,0 +1,2 @@
+{
+ "dummycode": [ "district", "view", "zipcode" ] }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_impute2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_impute2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_impute2.json
new file mode 100644
index 0000000..5fae74b
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_impute2.json
@@ -0,0 +1,10 @@
+{
+ "impute": [
+                 { "name": "zipcode", "method": "global_mode" }
+                ,{ "name": "district", "method": "constant", "value": "south" }
+                ,{ "name": "numbedrooms", "method": "constant", "value": "2" }
+                ,{ "name": "numbathrooms", "method": "constant", "value": "1" }
+                ,{ "name": "floors", "method": "constant", "value": "1" }
+                ,{ "name": "view", "method": "global_mode" }
+                ,{ "name": "askingprice", "method": "global_mean" }
+], "recode": [ "district", "view" ] }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_omit2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_omit2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_omit2.json
new file mode 100644
index 0000000..2114958
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_omit2.json
@@ -0,0 +1,2 @@
+{
+ "omit": [ "zipcode","district","numbedrooms","numbathrooms","floors","view","saleprice","askingprice" ], "recode": [ "district", "view" ] }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/468e472a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_recode2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_recode2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_recode2.json
new file mode 100644
index 0000000..d446628
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_recode2.json
@@ -0,0 +1,2 @@
+{
+ "recode": [ "zipcode", "district", "view" ] }
\ No newline at end of file