You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/09/08 21:27:34 UTC
incubator-systemml git commit: [SYSTEMML-145] Remove crc files from
local file system
Repository: incubator-systemml
Updated Branches:
refs/heads/master 24f8c5e6d -> df4e03cb1
[SYSTEMML-145] Remove crc files from local file system
Closes #215.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/df4e03cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/df4e03cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/df4e03cb
Branch: refs/heads/master
Commit: df4e03cb13873c9d50c8a8dfc8be6f71118084f0
Parents: 24f8c5e
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Thu Sep 8 14:23:49 2016 -0700
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Thu Sep 8 14:23:49 2016 -0700
----------------------------------------------------------------------
.../sysml/runtime/io/IOUtilFunctions.java | 22 ++++++++++++++++++++
.../sysml/runtime/io/WriterBinaryBlock.java | 4 ++++
.../runtime/io/WriterBinaryBlockParallel.java | 10 +++++++++
.../sysml/runtime/io/WriterBinaryCell.java | 5 +++++
.../sysml/runtime/io/WriterMatrixMarket.java | 4 ++++
.../runtime/io/WriterMatrixMarketParallel.java | 10 +++++++++
.../apache/sysml/runtime/io/WriterTextCSV.java | 4 ++++
.../sysml/runtime/io/WriterTextCSVParallel.java | 15 +++++++++++--
.../apache/sysml/runtime/io/WriterTextCell.java | 4 ++++
.../runtime/io/WriterTextCellParallel.java | 10 +++++++++
10 files changed, 86 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 49e8098..0ec3534 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -31,6 +31,8 @@ import java.util.Comparator;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -326,4 +328,24 @@ public class IOUtilFunctions
}
return ncol;
}
+
+ /**
+ * Delete the CRC files from the local file system associated with a
+ * particular file and its metadata file.
+ *
+ * @param fs
+ * the file system
+ * @param path
+ * the path to a file
+ * @throws IOException
+ * thrown if error occurred attempting to delete crc files
+ */
+ public static void deleteCrcFilesFromLocalFileSystem(FileSystem fs, Path path) throws IOException {
+ if (fs instanceof LocalFileSystem) {
+ Path fnameCrc = new Path(path.getParent(), "." + path.getName() + ".crc");
+ fs.delete(fnameCrc, false);
+ Path fnameMtdCrc = new Path(path.getParent(), "." + path.getName() + ".mtd.crc");
+ fs.delete(fnameMtdCrc, false);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
index 4ef1334..ac5a2e4 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
@@ -65,6 +65,8 @@ public class WriterBinaryBlock extends MatrixWriter
writeDiagBinaryBlockMatrixToHDFS(path, job, fs, src, rlen, clen, brlen, bclen);
else
writeBinaryBlockMatrixToHDFS(path, job, fs, src, rlen, clen, brlen, bclen);
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
@Override
@@ -84,6 +86,8 @@ public class WriterBinaryBlock extends MatrixWriter
(int)Math.min(clen, bclen), true);
writer.append(index, block);
writer.close();
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
index 16e01a3..2456331 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
@@ -28,6 +28,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.sysml.conf.DMLConfig;
@@ -87,6 +88,15 @@ public class WriterBinaryBlockParallel extends WriterBinaryBlock
catch (Exception e) {
throw new IOException("Failed parallel write of binary block input.", e);
}
+
+ // delete crc files if written to local file system
+ if (fs instanceof LocalFileSystem) {
+ int blklen = (int)Math.ceil((double)rlen / numThreads);
+ for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+ Path newPath = new Path(path, String.format("0-m-%05d",i));
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, newPath);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
index 8c66d52..5cc8efa 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
@@ -50,6 +50,9 @@ public class WriterBinaryCell extends MatrixWriter
//core write
writeBinaryCellMatrixToHDFS(path, job, src, rlen, clen, brlen, bclen);
+
+ FileSystem fs = FileSystem.get(job);
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
@Override
@@ -68,6 +71,8 @@ public class WriterBinaryCell extends MatrixWriter
MatrixCell cell = new MatrixCell(0);
writer.append(index, cell);
writer.close();
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
index 9ae359a..d413112 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
@@ -63,6 +63,8 @@ public class WriterMatrixMarket extends MatrixWriter
//core write
writeMatrixMarketMatrixToHDFS(path, job, fs, src);
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
@Override
@@ -75,6 +77,8 @@ public class WriterMatrixMarket extends MatrixWriter
FSDataOutputStream writer = fs.create(path);
writer.writeBytes("1 1 0");
writer.close();
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
index 34d392e..3326b86 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
@@ -28,6 +28,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.sysml.conf.DMLConfig;
@@ -88,6 +89,15 @@ public class WriterMatrixMarketParallel extends WriterMatrixMarket
catch (Exception e) {
throw new IOException("Failed parallel write of text output.", e);
}
+
+ // delete crc files if written to local file system
+ if (fs instanceof LocalFileSystem) {
+ int blklen = (int)Math.ceil((double)rlen / numThreads);
+ for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+ Path newPath = new Path(path, String.format("0-m-%05d",i));
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, newPath);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
index 7e2ce90..8bf8cb1 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
@@ -76,6 +76,8 @@ public class WriterTextCSV extends MatrixWriter
//core write (sequential/parallel)
writeCSVMatrixToHDFS(path, job, fs, src, _props);
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
@Override
@@ -88,6 +90,8 @@ public class WriterTextCSV extends MatrixWriter
MatrixBlock src = new MatrixBlock((int)rlen, 1, true);
writeCSVMatrixToHDFS(path, job, fs, src, _props);
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
index 01ad579..5df9118 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
@@ -28,6 +28,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.sysml.conf.DMLConfig;
@@ -90,8 +91,18 @@ public class WriterTextCSVParallel extends WriterTextCSV
}
catch (Exception e) {
throw new IOException("Failed parallel write of csv output.", e);
- }
- }
+ }
+
+ // delete crc files if written to local file system
+ if (fs instanceof LocalFileSystem) {
+ int rlen = src.getNumRows();
+ int blklen = (int)Math.ceil((double)rlen / numThreads);
+ for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+ Path newPath = new Path(path, String.format("0-m-%05d",i));
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, newPath);
+ }
+ }
+ }
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
index e32172e..5363669 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
@@ -55,6 +55,8 @@ public class WriterTextCell extends MatrixWriter
//core write
writeTextCellMatrixToHDFS(path, job, fs, src, rlen, clen);
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
@Override
@@ -67,6 +69,8 @@ public class WriterTextCell extends MatrixWriter
FSDataOutputStream writer = fs.create(path);
writer.writeBytes("1 1 0");
writer.close();
+
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/df4e03cb/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
index b758435..09d1dbc 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
@@ -28,6 +28,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.sysml.conf.DMLConfig;
@@ -94,6 +95,15 @@ public class WriterTextCellParallel extends WriterTextCell
catch (Exception e) {
throw new IOException("Failed parallel write of text output.", e);
}
+
+ // delete crc files if written to local file system
+ if (fs instanceof LocalFileSystem) {
+ int blklen = (int)Math.ceil((double)rlen / numThreads);
+ for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+ Path newPath = new Path(path, String.format("0-m-%05d",i));
+ IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, newPath);
+ }
+ }
}