You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by tu...@apache.org on 2015/12/16 15:15:54 UTC
[1/2] incubator-apex-malhar git commit: MLHR-1931 #resolve added
exists and listFiles to FileAccess
Repository: incubator-apex-malhar
Updated Branches:
refs/heads/devel-3 f070c33bf -> 480d6ed6c
MLHR-1931 #resolve added exists and listFiles to FileAccess
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/bc9a51fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/bc9a51fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/bc9a51fb
Branch: refs/heads/devel-3
Commit: bc9a51fbc467de4843a487d8f9781600f181a045
Parents: 54106d4
Author: Chandni Singh <cs...@apache.org>
Authored: Sun Dec 6 01:20:25 2015 -0800
Committer: Chandni Singh <cs...@apache.org>
Committed: Sun Dec 6 01:20:25 2015 -0800
----------------------------------------------------------------------
.../datatorrent/lib/fileaccess/FileAccess.java | 22 ++++++++++++++++++++
.../lib/fileaccess/FileAccessFSImpl.java | 18 ++++++++++++++++
2 files changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/bc9a51fb/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java
----------------------------------------------------------------------
diff --git a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java
index 4b7f6e5..16d6d72 100644
--- a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java
+++ b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java
@@ -24,6 +24,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.TreeMap;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
+
import com.datatorrent.netlet.util.Slice;
/**
@@ -52,6 +55,25 @@ public interface FileAccess extends Closeable
long getFileSize(long bucketKey, String s) throws IOException;
/**
+ * Checks if a file exists under a bucket.
+ *
+ * @param bucketKey bucket key
+ * @param fileName file name
+ * @return true if file exists; false otherwise.
+ * @throws IOException
+ */
+ boolean exists(long bucketKey, String fileName) throws IOException;
+
+ /**
+ * Lists the files in the bucket path.
+ *
+ * @param bucketKey bucket key
+ * @return status of all the files in the bucket path if the bucket exists; null otherwise.
+ * @throws IOException
+ */
+ RemoteIterator<LocatedFileStatus> listFiles(long bucketKey) throws IOException;
+
+ /**
* Data File Format Reader
*/
interface FileReader extends Closeable
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/bc9a51fb/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java
----------------------------------------------------------------------
diff --git a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java
index 80a201a..5c7e8d3 100644
--- a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java
+++ b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java
@@ -27,8 +27,10 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import com.datatorrent.netlet.util.DTThrowable;
@@ -122,6 +124,22 @@ public abstract class FileAccessFSImpl implements FileAccess
}
@Override
+ public boolean exists(long bucketKey, String fileName) throws IOException
+ {
+ return fs.exists(getFilePath(bucketKey, fileName));
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listFiles(long bucketKey) throws IOException
+ {
+ Path bucketPath = getBucketPath(bucketKey);
+ if (!fs.exists(bucketPath)) {
+ return null;
+ }
+ return fs.listFiles(bucketPath, true);
+ }
+
+ @Override
public String toString()
{
return this.getClass().getSimpleName() + "[basePath=" + basePath + "]";
[2/2] incubator-apex-malhar git commit: Merge branch 'MLHR-1931' of
https://github.com/chandnisingh/incubator-apex-malhar into devel-3
Posted by tu...@apache.org.
Merge branch 'MLHR-1931' of https://github.com/chandnisingh/incubator-apex-malhar into devel-3
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/480d6ed6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/480d6ed6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/480d6ed6
Branch: refs/heads/devel-3
Commit: 480d6ed6c0e896dc97c1f1b6979c4729fa640714
Parents: f070c33 bc9a51f
Author: Tushar R. Gosavi <tu...@apache.org>
Authored: Wed Dec 16 18:16:22 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Wed Dec 16 18:16:22 2015 +0530
----------------------------------------------------------------------
.../datatorrent/lib/fileaccess/FileAccess.java | 22 ++++++++++++++++++++
.../lib/fileaccess/FileAccessFSImpl.java | 18 ++++++++++++++++
2 files changed, 40 insertions(+)
----------------------------------------------------------------------