You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2016/02/27 02:42:30 UTC
incubator-systemml git commit: Adding a fix for transform()
Repository: incubator-systemml
Updated Branches:
refs/heads/master 9e41461c0 -> 5d09d27ad
Adding a fix for transform()
- For scenario where txMtdPath and outputPath are in same folder, but
outputPath can have suffix.
- Also making the path checking fully qualified.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5d09d27a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5d09d27a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5d09d27a
Branch: refs/heads/master
Commit: 5d09d27ad8dc644ed930a8f8bbba70e26c13de7d
Parents: 9e41461
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Fri Feb 26 17:42:30 2016 -0800
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Fri Feb 26 17:42:30 2016 -0800
----------------------------------------------------------------------
.../sysml/runtime/transform/DataTransform.java | 31 ++++++++++++++------
1 file changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5d09d27a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
index ea2f264..2287f1b 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
@@ -826,7 +826,7 @@ public class DataTransform {
boolean isBB = (bboutputs.size() > 0);
String tmpPath = MRJobConfiguration.constructTempOutputFilename();
- checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs);
+ checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs, fs);
JobReturn retCSV = null, retBB = null;
@@ -1099,7 +1099,7 @@ public class DataTransform {
boolean isCSV = (csvoutputs.size() > 0);
boolean isBB = (bboutputs.size() > 0);
- checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs);
+ checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs, fs);
JobReturn ret = null;
@@ -1402,17 +1402,30 @@ public class DataTransform {
}
private static void checkIfOutputOverlapsWithTxMtd(MatrixObject[] outputMatrices, TransformOperands oprnds,
- boolean isCSV, boolean isBB, ArrayList<Integer> csvoutputs, ArrayList<Integer> bboutputs) throws DMLRuntimeException {
+ boolean isCSV, boolean isBB, ArrayList<Integer> csvoutputs, ArrayList<Integer> bboutputs, FileSystem fs) throws DMLRuntimeException {
if(isCSV) {
- checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[csvoutputs.get(0)].getFileName());
+ checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[csvoutputs.get(0)].getFileName(), fs);
}
else if(isBB) {
- checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[bboutputs.get(0)].getFileName());
+ checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[bboutputs.get(0)].getFileName(), fs);
}
}
- private static void checkIfOutputOverlapsWithTxMtd(String txMtdPath, String outputPath) throws DMLRuntimeException {
- if(txMtdPath.startsWith(outputPath) || outputPath.startsWith(txMtdPath)) {
+ private static void checkIfOutputOverlapsWithTxMtd(String txMtdPath, String outputPath, FileSystem fs) throws DMLRuntimeException {
+ Path path1 = new Path(txMtdPath).makeQualified(fs);
+ Path path2 = new Path(outputPath).makeQualified(fs);
+
+ String fullTxMtdPath = path1.toString();
+ String fullOutputPath = path2.toString();
+
+ if(path1.getParent().toString().compareTo(path2.getParent().toString()) == 0) {
+ // Both txMtdPath and outputPath are in same folder, but outputPath can have suffix
+ if(fullTxMtdPath.compareTo(fullOutputPath) == 0) {
+ throw new DMLRuntimeException("The transform path \'" + txMtdPath
+ + "\' cannot overlap with the output path \'" + outputPath + "\'");
+ }
+ }
+ else if(fullTxMtdPath.startsWith(fullOutputPath) || fullOutputPath.startsWith(fullTxMtdPath)) {
throw new DMLRuntimeException("The transform path \'" + txMtdPath
+ "\' cannot overlap with the output path \'" + outputPath + "\'");
}
@@ -1425,11 +1438,11 @@ public class DataTransform {
// Parse transform instruction (the first instruction) to obtain relevant fields
TransformOperands oprnds = new TransformOperands(inst, inputMatrices[0]);
- checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[0].getFileName());
-
JobConf job = new JobConf();
FileSystem fs = FileSystem.get(job);
+ checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[0].getFileName(), fs);
+
// find the first file in alphabetical ordering of partfiles in directory inputPath
String smallestFile = CSVReblockMR.findSmallestFile(job, oprnds.inputPath);