You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2020/04/11 20:52:14 UTC
[systemml] branch master updated: [SYSTEMML-2538] Fix csv/text
output rename in forced singlenode
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push:
new 47924e6 [SYSTEMML-2538] Fix csv/text output rename in forced singlenode
47924e6 is described below
commit 47924e6aced3dac0768756c7dfec932d696b6a3f
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sat Apr 11 22:51:50 2020 +0200
[SYSTEMML-2538] Fix csv/text output rename in forced singlenode
This patch fixes an issue where an input csv/text file is directly fed
into a persistent write, which eventually just renames the input file
because it already exist on HDFS in the right format. We now explicitly
guard against persistently read inputs, which only can occur w/ forced
singelnode execution mode because other (in spark and hybrid) there is a
reblock (potentially in memory) that creates a new metadata object.
Furthermore, this also includes a minor internal refactoring for
consistently obtaining input/output infos for external format strings,
as well as a slight modification of the MatrixMatrixCellwiseTest to run
over smaller inputs (because R is taken quite a while for them).
---
.../java/org/apache/sysds/api/jmlc/Connection.java | 4 +-
.../org/apache/sysds/parser/DataExpression.java | 4 +-
.../controlprogram/caching/CacheableData.java | 2 +-
.../federated/FederatedWorkerHandler.java | 2 +-
.../instructions/cp/VariableCPInstruction.java | 38 ++-
.../sysds/runtime/io/MatrixReaderFactory.java | 2 +-
.../sysds/runtime/matrix/data/InputInfo.java | 8 +-
.../sysds/runtime/matrix/data/OutputInfo.java | 2 +-
.../org/apache/sysds/test/AutomatedTestBase.java | 14 +-
.../FullMatrixMatrixCellwiseOperationTest.java | 4 +-
.../test/functions/data/misc/NoRenameTest.java | 254 +++++++++++++++++++++
.../functions/frame/FrameMatrixReblockTest.java | 5 +-
.../test/functions/frame/FrameMatrixWriteTest.java | 2 +-
.../transform/TransformEncodeDecodeTest.java | 2 +-
src/test/scripts/functions/data/NoRenameTest1.dml | 24 ++
src/test/scripts/functions/data/NoRenameTest2.dml | 24 ++
16 files changed, 345 insertions(+), 46 deletions(-)
diff --git a/src/main/java/org/apache/sysds/api/jmlc/Connection.java b/src/main/java/org/apache/sysds/api/jmlc/Connection.java
index e1557b8..a008939 100644
--- a/src/main/java/org/apache/sysds/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysds/api/jmlc/Connection.java
@@ -372,7 +372,7 @@ public class Connection implements Closeable
long nnz = jmtd.containsKey(DataExpression.READNNZPARAM)?
jmtd.getLong(DataExpression.READNNZPARAM) : -1;
String format = jmtd.getString(DataExpression.FORMAT_TYPE);
- InputInfo iinfo = InputInfo.stringExternalToInputInfo(format);
+ InputInfo iinfo = InputInfo.fromExternalString(format);
//read matrix file
return readDoubleMatrix(fname, iinfo, rows, cols, blen, nnz);
@@ -614,7 +614,7 @@ public class Connection implements Closeable
long rows = jmtd.getLong(DataExpression.READROWPARAM);
long cols = jmtd.getLong(DataExpression.READCOLPARAM);
String format = jmtd.getString(DataExpression.FORMAT_TYPE);
- InputInfo iinfo = InputInfo.stringExternalToInputInfo(format);
+ InputInfo iinfo = InputInfo.fromExternalString(format);
//read frame file
return readStringFrame(fname, iinfo, rows, cols);
diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java
index 6ab1eb2..1b7ddc4 100644
--- a/src/main/java/org/apache/sysds/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysds/parser/DataExpression.java
@@ -220,7 +220,7 @@ public class DataExpression extends DataIdentifier
return null;
}
dataExpr.addVarParam(currName, currExpr);
- }
+ }
}
else if (functionName.equalsIgnoreCase("rand")){
@@ -1178,7 +1178,7 @@ public class DataExpression extends DataIdentifier
getOutput().setNnz(-1L);
}
- else{
+ else{
raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
index edd7519..a27318a 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
@@ -734,7 +734,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
//get object from cache
if( _data == null )
getCache();
- acquire( false, _data==null ); //incl. read matrix if evicted
+ acquire( false, _data==null ); //incl. read matrix if evicted
// b) write the matrix
try {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
index c2817c5..36bd258 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
@@ -137,7 +137,7 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
mc.setRows(mtd.getLong(DataExpression.READROWPARAM));
mc.setCols(mtd.getLong(DataExpression.READCOLPARAM));
String format = mtd.getString(DataExpression.FORMAT_TYPE);
- oi = OutputInfo.outputInfoFromStringExternal(format);
+ oi = OutputInfo.fromExternalString(format);
ii = OutputInfo.getMatchingInputInfo(oi);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
index d2e65f6..9176335 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
@@ -945,26 +945,23 @@ public class VariableCPInstruction extends CPInstruction implements LineageTrace
try {
OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
DataCharacteristics dc = (mo.getMetaData()).getDataCharacteristics();
- if(oi == OutputInfo.CSVOutputInfo) {
+ if( oi == OutputInfo.CSVOutputInfo
+ && !getInput1().getName().startsWith(org.apache.sysds.lops.Data.PREAD_PREFIX) )
+ {
WriterTextCSV writer = new WriterTextCSV((FileFormatPropertiesCSV)_formatProperties);
writer.addHeaderToCSV(mo.getFileName(), fname, dc.getRows(), dc.getCols());
}
- else if ( oi == OutputInfo.BinaryBlockOutputInfo || oi == OutputInfo.TextCellOutputInfo ) {
- mo.exportData(fname, outFmt, _formatProperties);
- }
else {
- throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into CSV format.");
+ mo.exportData(fname, outFmt, _formatProperties);
}
-
- // Write Metadata file
HDFSTool.writeMetaDataFile (fname + ".mtd", mo.getValueType(), dc, OutputInfo.CSVOutputInfo, _formatProperties);
- } catch (IOException e) {
+ }
+ catch (IOException e) {
throw new DMLRuntimeException(e);
}
}
}
-
/**
* Helper function to write MM files to HDFS.
*
@@ -980,24 +977,25 @@ public class VariableCPInstruction extends CPInstruction implements LineageTrace
mo.exportData(fname, outFmt);
}
else {
- OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
- DataCharacteristics dc = mo.getDataCharacteristics();
- if(oi == OutputInfo.TextCellOutputInfo) {
- try {
+ try {
+ OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
+ DataCharacteristics dc = mo.getDataCharacteristics();
+ if( oi == OutputInfo.TextCellOutputInfo
+ && !getInput1().getName().startsWith(org.apache.sysds.lops.Data.PREAD_PREFIX) )
+ {
WriterMatrixMarket.mergeTextcellToMatrixMarket(mo.getFileName(),
fname, dc.getRows(), dc.getCols(), dc.getNonZeros());
- } catch (IOException e) {
- throw new DMLRuntimeException(e);
+ }
+ else {
+ mo.exportData(fname, outFmt);
}
}
- else if ( oi == OutputInfo.BinaryBlockOutputInfo) {
- mo.exportData(fname, outFmt);
- }
- else {
- throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into MatrixMarket format.");
+ catch (IOException e) {
+ throw new DMLRuntimeException(e);
}
}
}
+
/**
* Helper function to write scalars to HDFS based on its value type.
*
diff --git a/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java b/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java
index d64d52c..3d2af34 100644
--- a/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java
@@ -38,7 +38,7 @@ public class MatrixReaderFactory
if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) && MatrixBlock.DEFAULT_SPARSEBLOCK == SparseBlock.Type.MCSR )
reader = new ReaderTextCellParallel( iinfo );
else
- reader = new ReaderTextCell( iinfo );
+ reader = new ReaderTextCell( iinfo );
}
else if( iinfo == InputInfo.CSVInputInfo )
{
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java b/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java
index fa7c3ce..7786d5d 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java
@@ -123,17 +123,17 @@ public class InputInfo implements Serializable
return null;
}
- public static InputInfo stringExternalToInputInfo(String str) {
+ public static InputInfo fromExternalString(String str) {
if( DataExpression.FORMAT_TYPE_VALUE_TEXT.equals(str) )
return InputInfo.TextCellInputInfo;
else if( DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET.equals(str) )
return InputInfo.MatrixMarketInputInfo;
else if( DataExpression.FORMAT_TYPE_VALUE_CSV.equals(str) )
- return InputInfo.CSVInputInfo;
+ return InputInfo.CSVInputInfo;
else if( DataExpression.FORMAT_TYPE_VALUE_LIBSVM.equals(str) )
- return InputInfo.LIBSVMInputInfo;
+ return InputInfo.LIBSVMInputInfo;
else if( DataExpression.FORMAT_TYPE_VALUE_BINARY.equals(str) ) // TODO BinaryTensorBlockInputInfo
- return InputInfo.BinaryBlockInputInfo;
+ return InputInfo.BinaryBlockInputInfo;
return null;
}
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java b/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java
index b8dce5a..00e4895 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java
@@ -153,7 +153,7 @@ public class OutputInfo implements Serializable
return "specialized";
}
- public static OutputInfo outputInfoFromStringExternal(String format) {
+ public static OutputInfo fromExternalString(String format) {
if (format.equalsIgnoreCase(FORMAT_TYPE_VALUE_TEXT))
return OutputInfo.TextCellOutputInfo;
else if (format.equalsIgnoreCase(FORMAT_TYPE_VALUE_BINARY))
diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java
index dee4ad5..c3d224f 100644
--- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java
@@ -288,15 +288,15 @@ public abstract class AutomatedTestBase {
}
protected ExecMode setExecMode(ExecType instType) {
- ExecMode platformOld = rtplatform;
switch(instType) {
- case SPARK:
- rtplatform = ExecMode.SPARK;
- break;
- default:
- rtplatform = ExecMode.HYBRID;
- break;
+ case SPARK: return setExecMode(ExecMode.SPARK);
+ default: return setExecMode(ExecMode.HYBRID);
}
+ }
+
+ protected ExecMode setExecMode(ExecMode execMode) {
+ ExecMode platformOld = rtplatform;
+ rtplatform = execMode;
if(rtplatform != ExecMode.SINGLE_NODE)
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
return platformOld;
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
index 2564ff1..b02c970 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
@@ -44,8 +44,8 @@ public class FullMatrixMatrixCellwiseOperationTest extends AutomatedTestBase
private final static String TEST_CLASS_DIR = TEST_DIR + FullMatrixMatrixCellwiseOperationTest.class.getSimpleName() + "/";
private final static double eps = 1e-10;
- private final static int rows = 1100;
- private final static int cols = 900;
+ private final static int rows = 1010;
+ private final static int cols = 300;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.1;
diff --git a/src/test/java/org/apache/sysds/test/functions/data/misc/NoRenameTest.java b/src/test/java/org/apache/sysds/test/functions/data/misc/NoRenameTest.java
new file mode 100644
index 0000000..619a325
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/data/misc/NoRenameTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.data.misc;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysds.common.Types.ExecMode;
+import org.apache.sysds.conf.ConfigurationManager;
+import org.apache.sysds.runtime.io.MatrixReader;
+import org.apache.sysds.runtime.io.MatrixReaderFactory;
+import org.apache.sysds.runtime.io.MatrixWriter;
+import org.apache.sysds.runtime.io.MatrixWriterFactory;
+import org.apache.sysds.runtime.matrix.data.InputInfo;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.matrix.data.OutputInfo;
+import org.apache.sysds.runtime.util.DataConverter;
+import org.apache.sysds.runtime.util.HDFSTool;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+
+public class NoRenameTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "NoRenameTest1";
+ private final static String TEST_NAME2 = "NoRenameTest2";
+ private final static String TEST_DIR = "functions/data/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + NoRenameTest.class.getSimpleName() + "/";
+
+ private final static int rows = 100;
+ private final static int cols = 50;
+ private final static int blocksize = ConfigurationManager.getBlocksize();
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.3;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "C" }) );
+ addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "C" }) );
+ }
+
+ @Test
+ public void testTextcellDenseSinglenode() {
+ runRenameTest("text", false, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testTextcellSparseSinglenode() {
+ runRenameTest("text", true, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testTextcsvDenseSinglenode() {
+ runRenameTest("csv", false, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testTextcsvSparseSinglenode() {
+ runRenameTest("csv", true, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testTextmmDenseSinglenode() {
+ runRenameTest("mm", false, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testTextmmSparseSinglenode() {
+ runRenameTest("mm", true, ExecMode.SINGLE_NODE);
+ }
+//
+// @Test
+// public void testTextlibsvmDenseSinglenode() {
+// runRenameTest("libsvm", false, ExecMode.SINGLE_NODE);
+// }
+//
+// @Test
+// public void testTextlibsvmSparseSinglenode() {
+// runRenameTest("libsvm", true, ExecMode.SINGLE_NODE);
+// }
+//
+ @Test
+ public void testBinaryDenseSinglenode() {
+ runRenameTest("binary", false, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testBinarySparseSinglenode() {
+ runRenameTest("binary", true, ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testTextcellDenseHybrid() {
+ runRenameTest("text", false, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testTextcellSparseHybrid() {
+ runRenameTest("text", true, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testTextcsvDenseHybrid() {
+ runRenameTest("csv", false, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testTextcsvSparseHybrid() {
+ runRenameTest("csv", true, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testTextmmDenseHybrid() {
+ runRenameTest("mm", false, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testTextmmSparseHybrid() {
+ runRenameTest("mm", true, ExecMode.HYBRID);
+ }
+//
+// @Test
+// public void testTextlibsvmDenseHybrid() {
+// runRenameTest("libsvm", false, ExecMode.HYBRID);
+// }
+//
+// @Test
+// public void testTextlibsvmSparseHybrid() {
+// runRenameTest("libsvm", true, ExecMode.HYBRID);
+// }
+
+ @Test
+ public void testBinaryDenseHybrid() {
+ runRenameTest("binary", false, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testBinarySparseHybrid() {
+ runRenameTest("binary", true, ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testTextcellDenseSpark() {
+ runRenameTest("text", false, ExecMode.SPARK);
+ }
+
+ @Test
+ public void testTextcellSparseSpark() {
+ runRenameTest("text", true, ExecMode.SPARK);
+ }
+
+ @Test
+ public void testTextcsvDenseSpark() {
+ runRenameTest("csv", false, ExecMode.SPARK);
+ }
+
+ @Test
+ public void testTextcsvSparseSpark() {
+ runRenameTest("csv", true, ExecMode.SPARK);
+ }
+
+ @Test
+ public void testTextmmDenseSpark() {
+ runRenameTest("mm", false, ExecMode.SPARK);
+ }
+
+ @Test
+ public void testTextmmSparseSpark() {
+ runRenameTest("mm", true, ExecMode.SPARK);
+ }
+//
+// @Test
+// public void testTextlibsvmDenseSpark() {
+// runRenameTest("libsvm", false, ExecMode.SPARK);
+// }
+//
+// @Test
+// public void testTextlibsvmSparseSpark() {
+// runRenameTest("libsvm", true, ExecMode.SPARK);
+// }
+
+ @Test
+ public void testBinaryDenseSpark() {
+ runRenameTest("binary", false, ExecMode.SPARK);
+ }
+
+ @Test
+ public void testBinarySparseSpark() {
+ runRenameTest("binary", true, ExecMode.SPARK);
+ }
+
+ private void runRenameTest(String fmt, boolean sparse, ExecMode et)
+ {
+ ExecMode platformOld = setExecMode(et);
+ double sparsity = (sparse) ? sparsity2 : sparsity1;
+
+ String TEST_NAME = fmt.equals("binary") ? TEST_NAME2 : TEST_NAME1;
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ config.addVariable("rows", rows);
+ config.addVariable("cols", cols);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-explain","-args", input("A"), fmt,
+ String.valueOf(rows), String.valueOf(cols), output("C")};
+
+ try {
+ //write input
+ double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
+ MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
+ MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(
+ OutputInfo.fromExternalString(fmt));
+ writer.writeMatrixToHDFS(mbA, input("A"), rows, cols, blocksize, mbA.getNonZeros());
+
+ //execute test
+ runTest(true, false, null, -1);
+ MatrixReader reader = MatrixReaderFactory.createMatrixReader(
+ InputInfo.fromExternalString(fmt));
+ MatrixBlock mbC = reader.readMatrixFromHDFS(output("C"), rows, cols, blocksize, -1);
+ double[][] C = DataConverter.convertToDoubleMatrix(mbC);
+
+ //compare matrices and check valid input
+ TestUtils.compareMatrices(A, C, rows, cols, 1e-10);
+ Assert.assertTrue(HDFSTool.existsFileOnHDFS(input("A")));
+ }
+ catch(IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ finally {
+ rtplatform = platformOld;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java
index bc342c2..8b76883 100644
--- a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java
@@ -232,16 +232,15 @@ public class FrameMatrixReblockTest extends AutomatedTestBase
//write input data
FrameWriter writer = FrameWriterFactory.createFrameWriter(
- InputInfo.getMatchingOutputInfo(InputInfo.stringExternalToInputInfo(ofmt)));
+ InputInfo.getMatchingOutputInfo(InputInfo.fromExternalString(ofmt)));
writer.writeFrameToHDFS(fb, fname, rows, cols);
}
private static double[][] readMatrixOutput(String fname, String ofmt, int rows, int cols)
throws IOException
{
- MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.stringExternalToInputInfo(ofmt));
+ MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.fromExternalString(ofmt));
MatrixBlock mb = reader.readMatrixFromHDFS(fname, rows, cols, 1000, -1);
-
return DataConverter.convertToDoubleMatrix(mb);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java
index 8a62dca..7252f84 100644
--- a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java
@@ -172,7 +172,7 @@ public class FrameMatrixWriteTest extends AutomatedTestBase
throws IOException
{
//read input data
- FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.stringExternalToInputInfo(ofmt));
+ FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.fromExternalString(ofmt));
FrameBlock fb = reader.readFrameFromHDFS(fname, rows, cols);
MatrixBlock ret = DataConverter.convertToMatrixBlock(fb);
diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java
index ff8bed4..8057b11 100644
--- a/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java
@@ -100,7 +100,7 @@ public class TransformEncodeDecodeTest extends AutomatedTestBase
getAndLoadTestConfiguration(TEST_NAME1);
//get input/output info
- InputInfo iinfo = InputInfo.stringExternalToInputInfo(fmt);
+ InputInfo iinfo = InputInfo.fromExternalString(fmt);
OutputInfo oinfo = InputInfo.getMatchingOutputInfo(iinfo);
//generate and write input data
diff --git a/src/test/scripts/functions/data/NoRenameTest1.dml b/src/test/scripts/functions/data/NoRenameTest1.dml
new file mode 100644
index 0000000..8a8956d
--- /dev/null
+++ b/src/test/scripts/functions/data/NoRenameTest1.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1, format=$2, rows=$3, cols=$4);
+print(nrow(A))
+write(A, $5, format=$2);
diff --git a/src/test/scripts/functions/data/NoRenameTest2.dml b/src/test/scripts/functions/data/NoRenameTest2.dml
new file mode 100644
index 0000000..892278f
--- /dev/null
+++ b/src/test/scripts/functions/data/NoRenameTest2.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1, format=$2, rows=$3, cols=$4, rows_in_block=1000, cols_in_block=1000);
+print(nrow(A))
+write(A, $5, format=$2);