[systemds] branch master updated: [MINOR] Fix unintended debug output, cleanup of federated workers

[mb...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 5c760c6  [MINOR] Fix unintended debug output, cleanup of federated workers
5c760c6 is described below

commit 5c760c62ebfb6eefee2852b777ba830f9c5d0bd3
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Wed Oct 13 15:36:24 2021 +0200

    [MINOR] Fix unintended debug output, cleanup of federated workers
    
    After detailed debugging I miss one of the outputs, which failed the
    federated tests because some of them check expected outputs.
    
    Furthermore, this fix also includes a minor rearrangement of the cleanup
    of federated workers, before we close the spark execution context to
    avoid issues in local test environment (which did not fail the actual
    tests but created confusing error logs)
---
 src/main/java/org/apache/sysds/api/DMLScript.java                   | 6 ++++--
 .../apache/sysds/runtime/controlprogram/caching/MatrixObject.java   | 1 -
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java
index 09c0a6e..4564f05 100644
--- a/src/main/java/org/apache/sysds/api/DMLScript.java
+++ b/src/main/java/org/apache/sysds/api/DMLScript.java
@@ -431,11 +431,13 @@ public class DMLScript
 			ScriptExecutorUtils.executeRuntimeProgram(rtprog, ec, ConfigurationManager.getDMLConfig(), STATISTICS ? STATISTICS_COUNT : 0, null);
 		}
 		finally {
+			//cleanup scratch_space and all working dirs
+			cleanupHadoopExecution(ConfigurationManager.getDMLConfig());
+			//stop spark context (after cleanup of federated workers and other pools,
+			//otherwise federated spark cleanups in local tests throw errors in same JVM)
 			if(ec != null && ec instanceof SparkExecutionContext)
 				((SparkExecutionContext) ec).close();
 			LOG.info("END DML run " + getDateTime() );
-			//cleanup scratch_space and all working dirs
-			cleanupHadoopExecution( ConfigurationManager.getDMLConfig());
 		}
 	}
 
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java
index 4da0675..ffedd65 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java
@@ -419,7 +419,6 @@ public class MatrixObject extends CacheableData<MatrixBlock> {
 		long clen = dims[1];
 		MetaDataFormat iimd = (MetaDataFormat) _metaData;
 		DataCharacteristics mc = iimd.getDataCharacteristics();
-		System.out.println(mc);
 		long begin = 0;
 
 		if(LOG.isTraceEnabled()) {