You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/08/11 11:59:20 UTC
ignite git commit: IGNITE-3647 IGFS: Local secondary: Implement
listPaths() and listFiles() operations. This closes #937.
Repository: ignite
Updated Branches:
refs/heads/ignite-1926 4bfa08d02 -> 866ef6bad
IGNITE-3647 IGFS: Local secondary: Implement listPaths() and listFiles() operations. This closes #937.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/866ef6ba
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/866ef6ba
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/866ef6ba
Branch: refs/heads/ignite-1926
Commit: 866ef6bad70db9b435e84b7e03086e6f293c9352
Parents: 4bfa08d
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Thu Aug 11 14:59:05 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Aug 11 14:59:05 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/IgniteFileSystem.java | 27 ++--
.../igfs/secondary/IgfsSecondaryFileSystem.java | 2 +-
.../hadoop/fs/LocalIgfsSecondaryFileSystem.java | 128 ++++++++++---------
3 files changed, 85 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/866ef6ba/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
index f9aeb8d..8fb4fcd 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -34,6 +34,7 @@ import org.apache.ignite.lang.IgniteAsyncSupport;
import org.apache.ignite.lang.IgniteAsyncSupported;
import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.Nullable;
+import org.apache.ignite.igfs.IgfsPathNotFoundException;
/**
* <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache:
@@ -90,7 +91,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
*
* @param path Path to get information for.
* @return Summary object.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path is not found.
+ * @throws IgfsPathNotFoundException If path is not found.
* @throws IgniteException If failed.
*/
public IgfsPathSummary summary(IgfsPath path) throws IgniteException;
@@ -101,7 +102,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param path File path to read.
* @return File input stream to read data from.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public IgfsInputStream open(IgfsPath path) throws IgniteException;
@@ -112,7 +113,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
* @return File input stream to read data from.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
@@ -124,7 +125,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started.
* @return File input stream to read data from.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException;
@@ -178,7 +179,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param create Create file if it doesn't exist yet.
* @return File output stream to append data to.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist and create flag is {@code false}.
+ * @throws IgfsPathNotFoundException If path doesn't exist and create flag is {@code false}.
*/
public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException;
@@ -191,7 +192,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param props File properties to set only in case it file was just created.
* @return File output stream to append data to.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist and create flag is {@code false}.
+ * @throws IgfsPathNotFoundException If path doesn't exist and create flag is {@code false}.
*/
public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
throws IgniteException;
@@ -204,7 +205,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param accessTime Optional last access time to set. Value {@code -1} does not update access time.
* @param modificationTime Optional last modification time to set. Value {@code -1} does not update
* modification time.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If target was not found.
+ * @throws IgfsPathNotFoundException If target was not found.
* @throws IgniteException If error occurred.
*/
public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException;
@@ -218,7 +219,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param len Size of data in the file to resolve affinity for.
* @return Affinity block locations.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException;
@@ -233,7 +234,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param maxLen Maximum length of a single returned block location length.
* @return Affinity block locations.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen)
throws IgniteException;
@@ -393,7 +394,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param dest Destination file path. If destination path is a directory, then source file will be placed
* into destination directory with original name.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If source file doesn't exist.
+ * @throws IgfsPathNotFoundException If source file doesn't exist.
*/
public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
@@ -430,9 +431,9 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* Lists file paths under the specified path.
*
* @param path Path to list files under.
- * @return List of files under the specified path.
+ * @return List of paths under the specified path.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
@@ -442,7 +443,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* @param path Path to list files under.
* @return List of files under the specified path.
* @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
+ * @throws IgfsPathNotFoundException If path doesn't exist.
*/
public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/866ef6ba/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index 4d9d255..47a0dbd 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -110,7 +110,7 @@ public interface IgfsSecondaryFileSystem {
* Lists file paths under the specified path.
*
* @param path Path to list files under.
- * @return List of files under the specified path.
+ * @return List of paths under the specified path.
* @throws IgniteException In case of error.
* @throws org.apache.ignite.igfs.IgfsPathNotFoundException If path doesn't exist.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/866ef6ba/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
index d9c162c..afd8d03 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
@@ -17,7 +17,6 @@
package org.apache.ignite.hadoop.fs;
-import java.nio.file.Files;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.ParentNotDirectoryException;
@@ -37,12 +36,10 @@ import org.apache.ignite.igfs.IgfsUserContext;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProperties;
-import org.apache.ignite.internal.processors.igfs.IgfsEntryInfo;
-import org.apache.ignite.internal.processors.igfs.IgfsFileImpl;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.io.GridFilenameUtils;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
@@ -54,8 +51,9 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
-import java.util.ArrayList;
+import java.nio.file.Files;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -311,71 +309,53 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
/** {@inheritDoc} */
@Override public Collection<IgfsPath> listPaths(IgfsPath path) {
- try {
- // TODO: IGNITE-3647.
- FileStatus[] statuses = fileSystemForUser().listStatus(convert(path));
+ File[] entries = listFiles0(path);
- if (statuses == null)
- throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
-
- Collection<IgfsPath> res = new ArrayList<>(statuses.length);
+ if (F.isEmpty(entries))
+ return Collections.emptySet();
+ else {
+ Collection<IgfsPath> res = U.newHashSet(entries.length);
- for (FileStatus status : statuses)
- res.add(new IgfsPath(path, status.getPath().getName()));
+ for (File entry : entries)
+ res.add(igfsPath(entry));
return res;
}
- catch (FileNotFoundException ignored) {
- throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
- }
}
/** {@inheritDoc} */
@Override public Collection<IgfsFile> listFiles(IgfsPath path) {
- try {
- // TODO: IGNITE-3647.
- FileStatus[] statuses = fileSystemForUser().listStatus(convert(path));
-
- if (statuses == null)
- throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
-
- Collection<IgfsFile> res = new ArrayList<>(statuses.length);
-
- for (FileStatus s : statuses) {
- IgfsEntryInfo fsInfo = s.isDirectory() ?
- IgfsUtils.createDirectory(
- IgniteUuid.randomUuid(),
- null,
- properties(s),
- s.getAccessTime(),
- s.getModificationTime()
- ) :
- IgfsUtils.createFile(
- IgniteUuid.randomUuid(),
- (int)s.getBlockSize(),
- s.getLen(),
- null,
- null,
- false,
- properties(s),
- s.getAccessTime(),
- s.getModificationTime()
- );
-
- res.add(new IgfsFileImpl(new IgfsPath(path, s.getPath().getName()), fsInfo, 1));
+ File[] entries = listFiles0(path);
+
+ if (F.isEmpty(entries))
+ return Collections.emptySet();
+ else {
+ Collection<IgfsFile> res = U.newHashSet(entries.length);
+
+ for (File entry : entries) {
+ IgfsFile info = info(igfsPath(entry));
+
+ if (info != null)
+ res.add(info);
}
return res;
}
- catch (FileNotFoundException ignored) {
+ }
+
+ /**
+ * Returns an array of File object. Under the specific path.
+ *
+ * @param path IGFS path.
+ * @return Array of File objects.
+ */
+ @Nullable private File[] listFiles0(IgfsPath path) {
+ File f = fileForPath(path);
+
+ if (!f.exists())
throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
- }
+ else
+ return f.listFiles();
}
/** {@inheritDoc} */
@@ -538,6 +518,10 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
if (fsFactory instanceof LifecycleAware)
((LifecycleAware) fsFactory).start();
+
+ workDir = new File(workDir).getAbsolutePath();
+
+ assert !workDir.endsWith("/") : workDir;
}
/** {@inheritDoc} */
@@ -592,8 +576,36 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
private File fileForPath(IgfsPath path) {
if (workDir == null)
return new File(path.toString());
- else
- return new File(workDir, path.toString());
+ else {
+ if ("/".equals(path.toString()))
+ return new File(workDir);
+ else
+ return new File(workDir, path.toString());
+ }
+ }
+
+ /**
+ * Create IGFS path for file.
+ *
+ * @param f File object.
+ * @return IFGS path.
+ * @throws IgfsException If failed.
+ */
+ private IgfsPath igfsPath(File f) throws IgfsException {
+ String path = f.getAbsolutePath();
+
+ if (workDir != null) {
+ if (!path.startsWith(workDir))
+ throw new IgfsException("Path is not located in the work directory [workDir=" + workDir +
+ "path=" + path + ']');
+
+ path = path.substring(workDir.length(), path.length());
+
+ assert !path.startsWith("/") : "Path is not located in the work directory [workDir=" + workDir +
+ "path=" + f + ']';
+ }
+
+ return new IgfsPath(path);
}
/**