You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/01/31 07:20:07 UTC
[2/2] systemml git commit: [SYSTEMML-2101] Fix JMLC robustness for
missing fs impl classes
[SYSTEMML-2101] Fix JMLC robustness for missing fs impl classes
This patch increases the robustness of JMLC for missing file system
classes. Thrown NoClassDefFoundErrors so far passed the exception
handling leading to failures even though the persistent reads are later
replaced by in-memory inputs.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/56205d02
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/56205d02
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/56205d02
Branch: refs/heads/master
Commit: 56205d025bf049cc52acd99c87cbf287ddf2929a
Parents: 4b5b14b
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue Jan 30 22:56:18 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue Jan 30 22:56:18 2018 -0800
----------------------------------------------------------------------
.../controlprogram/parfor/opt/OptimizerRuleBased.java | 8 ++++----
.../org/apache/sysml/runtime/io/IOUtilFunctions.java | 14 +++++++++++++-
2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/56205d02/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index d551d58..0359f7f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -95,6 +95,7 @@ import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -1096,15 +1097,14 @@ public class OptimizerRuleBased extends Optimizer
//account for remaining hdfs capacity
try {
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
+ FileSystem fs = IOUtilFunctions.getFileSystem(ConfigurationManager.getCachedJobConf());
long hdfsCapacityRemain = fs.getStatus().getRemaining();
long sizeInputs = 0; //sum of all input sizes (w/o replication)
- for( String var : partitionedMatrices.keySet() )
- {
+ for( String var : partitionedMatrices.keySet() ) {
MatrixObject mo = (MatrixObject)vars.get(var);
Path fname = new Path(mo.getFileName());
if( fs.exists( fname ) ) //non-existing (e.g., CP) -> small file
- sizeInputs += fs.getContentSummary(fname).getLength();
+ sizeInputs += fs.getContentSummary(fname).getLength();
}
replication = (int) Math.min(replication, Math.floor(0.9*hdfsCapacityRemain/sizeInputs));
http://git-wip-us.apache.org/repos/asf/systemml/blob/56205d02/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 8d55eab..526ad98 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -67,8 +67,20 @@ public class IOUtilFunctions
ConfigurationManager.getCachedJobConf());
}
+ public static FileSystem getFileSystem(Configuration conf) throws IOException {
+ try{
+ return FileSystem.get(conf);
+ } catch(NoClassDefFoundError err) {
+ throw new IOException(err.getMessage());
+ }
+ }
+
public static FileSystem getFileSystem(Path fname, Configuration conf) throws IOException {
- return FileSystem.get(fname.toUri(), conf);
+ try {
+ return FileSystem.get(fname.toUri(), conf);
+ } catch(NoClassDefFoundError err) {
+ throw new IOException(err.getMessage());
+ }
}
public static boolean isSameFileScheme(Path path1, Path path2) {