You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2016/02/18 03:00:10 UTC
incubator-systemml git commit: Provide error message when the
transform and output path overlaps.
Repository: incubator-systemml
Updated Branches:
refs/heads/master 5baebe4f7 -> f4d99cf01
Provide error message when the transform and output path overlaps.
We will throw error message for following invocation:
hadoop jar SystemML.jar -f algorithms/transform.dml -nvargs
TRANSFORM_PATH=foo/output OUTPUT_DATA_PATH=foo/output.mtx ...
However, following invocation is valid
hadoop jar SystemML.jar -f algorithms/transform.dml -nvargs
TRANSFORM_PATH=foo/maps OUTPUT_DATA_PATH=foo/data/output.mtx ...
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f4d99cf0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f4d99cf0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f4d99cf0
Branch: refs/heads/master
Commit: f4d99cf0108ab03a73d5cdfdc36951f2f762e53e
Parents: 5baebe4
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Wed Feb 17 18:00:12 2016 -0800
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Wed Feb 17 18:00:12 2016 -0800
----------------------------------------------------------------------
.../sysml/runtime/transform/DataTransform.java | 25 +++++++++++++++++++-
1 file changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f4d99cf0/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
index 1b85a3f..ea2f264 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
@@ -795,7 +795,7 @@ public class DataTransform {
// Parse transform instruction (the first instruction) to obtain relevant fields
TransformOperands oprnds = new TransformOperands(insts[0], inputMatrices[0]);
- JobConf job = new JobConf(DataTransform.class); // ConfigurationManager.getCachedJobConf());
+ JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
FileSystem fs = FileSystem.get(job);
// find the first file in alphabetical ordering of partfiles in directory inputPath
@@ -826,6 +826,8 @@ public class DataTransform {
boolean isBB = (bboutputs.size() > 0);
String tmpPath = MRJobConfiguration.constructTempOutputFilename();
+ checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs);
+
JobReturn retCSV = null, retBB = null;
if (!oprnds.isApply) {
@@ -1097,6 +1099,8 @@ public class DataTransform {
boolean isCSV = (csvoutputs.size() > 0);
boolean isBB = (bboutputs.size() > 0);
+ checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs);
+
JobReturn ret = null;
if (!oprnds.isApply) {
@@ -1397,6 +1401,23 @@ public class DataTransform {
br.close();
}
+ private static void checkIfOutputOverlapsWithTxMtd(MatrixObject[] outputMatrices, TransformOperands oprnds,
+ boolean isCSV, boolean isBB, ArrayList<Integer> csvoutputs, ArrayList<Integer> bboutputs) throws DMLRuntimeException {
+ if(isCSV) {
+ checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[csvoutputs.get(0)].getFileName());
+ }
+ else if(isBB) {
+ checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[bboutputs.get(0)].getFileName());
+ }
+ }
+
+ private static void checkIfOutputOverlapsWithTxMtd(String txMtdPath, String outputPath) throws DMLRuntimeException {
+ if(txMtdPath.startsWith(outputPath) || outputPath.startsWith(txMtdPath)) {
+ throw new DMLRuntimeException("The transform path \'" + txMtdPath
+ + "\' cannot overlap with the output path \'" + outputPath + "\'");
+ }
+ }
+
public static void spDataTransform(ParameterizedBuiltinSPInstruction inst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices, ExecutionContext ec) throws Exception {
SparkExecutionContext sec = (SparkExecutionContext)ec;
@@ -1404,6 +1425,8 @@ public class DataTransform {
// Parse transform instruction (the first instruction) to obtain relevant fields
TransformOperands oprnds = new TransformOperands(inst, inputMatrices[0]);
+ checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[0].getFileName());
+
JobConf job = new JobConf();
FileSystem fs = FileSystem.get(job);