You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ma...@apache.org on 2021/03/01 17:47:00 UTC

[systemds] branch master updated: [MINOR] Fix null check in EncoderMVImpute for Global_Mean; Also:

This is an automated email from the ASF dual-hosted git repository.

markd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new d4ba502  [MINOR] Fix null check in EncoderMVImpute for Global_Mean; Also:
d4ba502 is described below

commit d4ba5028635ff2011b22828ad1e372d1811fab95
Author: Lukas Erlbacher <lu...@gmail.com>
AuthorDate: Mon Mar 1 18:46:28 2021 +0100

    [MINOR] Fix null check in EncoderMVImpute for Global_Mean; Also:
    
    * Adapted test so NaN is checked
    * removed unnecessary import
    * adapted count for mean calculation
    
    Closes #1190
---
 .../runtime/transform/encode/EncoderMVImpute.java  | 24 +++++++++++++---------
 src/test/java/org/apache/sysds/test/TestUtils.java |  7 +++++++
 .../transform/TransformFrameEncodeApplyTest.java   | 10 +++++----
 3 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderMVImpute.java b/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderMVImpute.java
index 16377ba..2effddb 100644
--- a/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderMVImpute.java
+++ b/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderMVImpute.java
@@ -48,8 +48,7 @@ import org.apache.wink.json4j.JSONArray;
 import org.apache.wink.json4j.JSONException;
 import org.apache.wink.json4j.JSONObject;
 
-public class EncoderMVImpute extends Encoder
-{
+public class EncoderMVImpute extends Encoder {
 	private static final long serialVersionUID = 9057868620144662194L;
 
 	public enum MVMethod { INVALID, GLOBAL_MEAN, GLOBAL_MODE, CONSTANT }
@@ -69,8 +68,7 @@ public class EncoderMVImpute extends Encoder
 	public KahanObject[] getMeans()   { return _meanList; }
 	
 	public EncoderMVImpute(JSONObject parsedSpec, String[] colnames, int clen, int minCol, int maxCol)
-		throws JSONException
-	{
+			throws JSONException {
 		super(null, clen);
 		
 		//handle column list
@@ -166,9 +164,15 @@ public class EncoderMVImpute extends Encoder
 				if( _mvMethodList[j] == MVMethod.GLOBAL_MEAN ) {
 					//compute global column mean (scale)
 					long off = _countList[j];
-					for( int i=0; i<in.getNumRows(); i++ )
+					for( int i=0; i<in.getNumRows(); i++ ){
+						Object key = in.get(i, colID-1);
+						if(key == null){
+							off--;
+							continue;
+						}
 						_meanFn.execute2(_meanList[j], UtilFunctions.objectToDouble(
-							in.getSchema()[colID-1], in.get(i, colID-1)), off+i+1);
+								in.getSchema()[colID-1], key), off+i+1);
+					}
 					_replacementList[j] = String.valueOf(_meanList[j]._sum);
 					_countList[j] += in.getNumRows();
 				}
@@ -241,8 +245,8 @@ public class EncoderMVImpute extends Encoder
 	}
 
 	private static void fillListsFromMap(Map<Integer, ColInfo> map, int[] colList, MVMethod[] mvMethodList,
-		String[] replacementList, KahanObject[] meanList, long[] countList,
-		HashMap<Integer, HashMap<String, Long>> hist) {
+			String[] replacementList, KahanObject[] meanList, long[] countList,
+			HashMap<Integer, HashMap<String, Long>> hist) {
 		int i = 0;
 		for(Entry<Integer, ColInfo> entry : map.entrySet()) {
 			colList[i] = entry.getKey();
@@ -267,7 +271,7 @@ public class EncoderMVImpute extends Encoder
 			for(int i = 0; i < other._colList.length; i++) {
 				int column = other._colList[i] + (col - 1);
 				ColInfo otherColInfo = new ColInfo(otherImpute._mvMethodList[i], otherImpute._replacementList[i],
-					otherImpute._meanList[i], otherImpute._countList[i], otherImpute._hist.get(i + 1));
+						otherImpute._meanList[i], otherImpute._countList[i], otherImpute._hist.get(i + 1));
 				ColInfo colInfo = map.get(column);
 				if(colInfo == null)
 					map.put(column, otherColInfo);
@@ -432,7 +436,7 @@ public class EncoderMVImpute extends Encoder
 		public void merge(ColInfo otherColInfo) {
 			if(_method != otherColInfo._method)
 				throw new DMLRuntimeException("Tried to merge two different impute methods: " + _method.name() + " vs. "
-					+ otherColInfo._method.name());
+						+ otherColInfo._method.name());
 			switch(_method) {
 				case CONSTANT:
 					assert _replacement.equals(otherColInfo._replacement);
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java
index 0d541e7..b8c6304 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -3056,4 +3056,11 @@ public class TestUtils
 
 		return y;
 	}
+
+	public static boolean containsNan(double[][] data, int col) {
+		for (double[] datum : data)
+			if (Double.isNaN(datum[col]))
+				return true;
+		return false;
+	}
 }
diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeApplyTest.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeApplyTest.java
index 18f5fd7..65b9fa8 100644
--- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeApplyTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeApplyTest.java
@@ -31,8 +31,7 @@ import org.apache.sysds.test.TestConfiguration;
 import org.apache.sysds.test.TestUtils;
 import org.apache.sysds.utils.Statistics;
 
-public class TransformFrameEncodeApplyTest extends AutomatedTestBase 
-{
+public class TransformFrameEncodeApplyTest extends AutomatedTestBase {
 	private final static String TEST_NAME1 = "TransformFrameEncodeApply";
 	private final static String TEST_DIR = "functions/transform/";
 	private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameEncodeApplyTest.class.getSimpleName() + "/";
@@ -357,8 +356,7 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase
 		runTransformTest(ExecMode.HYBRID, "csv", TransformType.HASH_RECODE, false);
 	}
 	
-	private void runTransformTest( ExecMode rt, String ofmt, TransformType type, boolean colnames )
-	{
+	private void runTransformTest( ExecMode rt, String ofmt, TransformType type, boolean colnames )	{
 		boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
 		if( rtplatform == ExecMode.SPARK || rtplatform == ExecMode.HYBRID)
 			DMLScript.USE_LOCAL_SPARK_CONFIG = true;
@@ -431,6 +429,10 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase
 							1:0, R1[i][10+j], 1e-8);
 					}
 				}
+			} else if (type == TransformType.IMPUTE){
+				// Column 8 had GLOBAL_MEAN applied
+				Assert.assertFalse(TestUtils.containsNan(R1, 8));
+				Assert.assertFalse(TestUtils.containsNan(R2, 8));
 			}
 		}
 		catch(Exception ex) {