You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/08/15 07:38:50 UTC
[21/30] ignite git commit: IGNITE-3650: Implemented info() operation.
IGNITE-3650: Implemented info() operation.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/09982a64
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/09982a64
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/09982a64
Branch: refs/heads/ignite-1926
Commit: 09982a64fb4e01e235f3d84f23477655ca15433e
Parents: 93440cb
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Aug 15 09:55:54 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Aug 15 09:55:54 2016 +0300
----------------------------------------------------------------------
.../internal/processors/igfs/IgfsFileImpl.java | 20 ++-
.../internal/processors/igfs/IgfsImpl.java | 2 +-
.../processors/igfs/IgfsMetaManager.java | 2 +-
.../internal/processors/igfs/IgfsUtils.java | 53 ++++++++
.../processors/igfs/IgfsAbstractSelfTest.java | 6 +-
.../igfs/IgfsDualAbstractSelfTest.java | 3 +-
.../hadoop/fs/LocalFileSystemIgfsFile.java | 134 +++++++++++++++++++
.../hadoop/fs/LocalIgfsSecondaryFileSystem.java | 88 ++----------
...SecondaryFileSystemDualAbstractSelfTest.java | 9 +-
9 files changed, 229 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
index 9f79f42..984c8f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
@@ -66,6 +66,9 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
/** Last modification time. */
private long modificationTime;
+ /** Flags. */
+ private byte flags;
+
/** Properties. */
private Map<String, String> props;
@@ -81,6 +84,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
* which is specified separately.
*
* @param igfsFile The file to copy.
+ * @param grpBlockSize Group block size.
*/
public IgfsFileImpl(IgfsFile igfsFile, long grpBlockSize) {
A.notNull(igfsFile, "igfsFile");
@@ -97,25 +101,29 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
this.accessTime = igfsFile.accessTime();
this.modificationTime = igfsFile.modificationTime();
+ this.flags = IgfsUtils.flags(igfsFile.isDirectory(), igfsFile.isFile());
}
/**
* Constructs directory info.
*
* @param path Path.
+ * @param info Entry info.
+ * @param globalGrpBlockSize Global group block size.
*/
public IgfsFileImpl(IgfsPath path, IgfsEntryInfo info, long globalGrpBlockSize) {
A.notNull(path, "path");
A.notNull(info, "info");
this.path = path;
+
fileId = info.id();
+ flags = IgfsUtils.flags(info.isDirectory(), info.isFile());
+
if (info.isFile()) {
blockSize = info.blockSize();
- assert blockSize > 0; // By contract file must have blockSize > 0, while directory's blockSize == 0.
-
len = info.length();
grpBlockSize = info.affinityKey() == null ? globalGrpBlockSize :
@@ -145,12 +153,12 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
/** {@inheritDoc} */
@Override public boolean isFile() {
- return blockSize > 0;
+ return IgfsUtils.isFile(flags);
}
/** {@inheritDoc} */
@Override public boolean isDirectory() {
- return blockSize == 0;
+ return IgfsUtils.isDirectory(flags);
}
/** {@inheritDoc} */
@@ -214,6 +222,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
U.writeStringMap(out, props);
out.writeLong(accessTime);
out.writeLong(modificationTime);
+ out.writeByte(flags);
}
/**
@@ -232,6 +241,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
props = U.readStringMap(in);
accessTime = in.readLong();
modificationTime = in.readLong();
+ flags = in.readByte();
}
/** {@inheritDoc} */
@@ -245,6 +255,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
IgfsUtils.writeProperties(rawWriter, props);
rawWriter.writeLong(accessTime);
rawWriter.writeLong(modificationTime);
+ rawWriter.writeByte(flags);
}
/** {@inheritDoc} */
@@ -258,6 +269,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
props = IgfsUtils.readProperties(rawReader);
accessTime = rawReader.readLong();
modificationTime = rawReader.readLong();
+ flags = rawReader.readByte();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index e1f8e61..e4f5999 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -924,7 +924,7 @@ public final class IgfsImpl implements IgfsEx {
@Override public IgfsInputStreamAdapter open(final IgfsPath path, final int bufSize,
final int seqReadsBeforePrefetch) {
A.notNull(path, "path");
- A.ensure(bufSize >= 0, "bufSize >= 0");
+ A.ensure(bufSize >= 0, "bufSize");
A.ensure(seqReadsBeforePrefetch >= 0, "seqReadsBeforePrefetch >= 0");
return safeOp(new Callable<IgfsInputStreamAdapter>() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index 1364491..89cadce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -1919,7 +1919,7 @@ public class IgfsMetaManager extends IgfsManager {
IgfsEntryInfo newInfo = IgfsUtils.createFile(
IgniteUuid.randomUuid(),
- status.blockSize(),
+ igfsCtx.configuration().getBlockSize(),
status.length(),
affKey,
createFileLockId(false),
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
index a79d965..2e79a98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
@@ -120,6 +120,12 @@ public class IgfsUtils {
/** Separator between id and name parts in the trash name. */
private static final char TRASH_NAME_SEPARATOR = '|';
+ /** Flag: this is a directory. */
+ private static final byte FLAG_DIR = 0x1;
+
+ /** Flag: this is a file. */
+ private static final byte FLAG_FILE = 0x2;
+
/**
* Static initializer.
*/
@@ -907,4 +913,51 @@ public class IgfsUtils {
return resModes;
}
+
+ /**
+ * Create flags value.
+ *
+ * @param isDir Directory flag.
+ * @param isFile File flag.
+ * @return Result.
+ */
+ public static byte flags(boolean isDir, boolean isFile) {
+ byte res = isDir ? FLAG_DIR : 0;
+
+ if (isFile)
+ res |= FLAG_FILE;
+
+ return res;
+ }
+
+ /**
+ * Check whether passed flags represent directory.
+ *
+ * @param flags Flags.
+ * @return {@code True} if this is directory.
+ */
+ public static boolean isDirectory(byte flags) {
+ return hasFlag(flags, FLAG_DIR);
+ }
+
+ /**
+ * Check whether passed flags represent file.
+ *
+ * @param flags Flags.
+ * @return {@code True} if this is file.
+ */
+ public static boolean isFile(byte flags) {
+ return hasFlag(flags, FLAG_FILE);
+ }
+
+ /**
+ * Check whether certain flag is set.
+ *
+ * @param flags Flags.
+ * @param flag Flag to check.
+ * @return {@code True} if flag is set.
+ */
+ private static boolean hasFlag(byte flags, byte flag) {
+ return (flags & flag) == flag;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 154a202..c92eb3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -1120,6 +1120,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testRootPropertiesPersistAfterFormat() throws Exception {
+ if(!propertiesSupported())
+ return;
+
if (dual && !(igfsSecondaryFileSystem instanceof IgfsSecondaryFileSystemImpl)) {
// In case of Hadoop dual mode only user name, group name, and permission properties are updated,
// an arbitrary named property is just ignored:
@@ -2114,7 +2117,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
try {
os = igfs.append(FILE, false);
- igfs.update(FILE, props);
+ if (permissionsSupported())
+ igfs.update(FILE, props);
os.close();
} finally {
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index aea0e97..a64dc2e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -1173,7 +1173,8 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
create(igfsSecondary, paths(DIR, SUBDIR), paths(FILE));
// Write enough data to the secondary file system.
- final int blockSize = igfs.info(FILE).blockSize();
+ int blockSize0 = igfs.info(FILE).blockSize();
+ final int blockSize = blockSize0 != 0 ? blockSize0 : 8 * 1024;
int totalWritten = 0;
try (OutputStream out = igfsSecondary.openOutputStream(FILE.toString(), false)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalFileSystemIgfsFile.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalFileSystemIgfsFile.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalFileSystemIgfsFile.java
new file mode 100644
index 0000000..2ecbbf1
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalFileSystemIgfsFile.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.ignite.igfs.IgfsFile;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Implementation of the IgfsFile interface for the local filesystem.
+ */
+class LocalFileSystemIgfsFile implements IgfsFile {
+ /** Path. */
+ private final IgfsPath path;
+
+ /** Flags. */
+ private final byte flags;
+
+ /** Block size. */
+ private final int blockSize;
+
+ /** Modification time. */
+ private final long modTime;
+
+ /** Length. */
+ private final long len;
+
+ /** Properties. */
+ private final Map<String, String> props;
+
+ /**
+ * @param path IGFS path.
+ * @param isFile Path is a file.
+ * @param isDir Path is a directory.
+ * @param blockSize Block size in bytes.
+ * @param modTime Modification time in millis.
+ * @param len File length in bytes.
+ * @param props Properties.
+ */
+ LocalFileSystemIgfsFile(IgfsPath path, boolean isFile, boolean isDir, int blockSize,
+ long modTime, long len, Map<String, String> props) {
+
+ assert !isDir || blockSize == 0 : "blockSize must be 0 for dirs. [blockSize=" + blockSize + ']';
+ assert !isDir || len == 0 : "length must be 0 for dirs. [length=" + len + ']';
+
+ this.path = path;
+ this.flags = IgfsUtils.flags(isDir, isFile);
+ this.blockSize = blockSize;
+ this.modTime = modTime;
+ this.len = len;
+ this.props = props;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsPath path() {
+ return path;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isFile() {
+ return IgfsUtils.isFile(flags);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDirectory() {
+ return IgfsUtils.isDirectory(flags);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int blockSize() {
+ return blockSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long groupBlockSize() {
+ return blockSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long accessTime() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long modificationTime() {
+ return modTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String property(String name) throws IllegalArgumentException {
+ return property(name, null);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String property(String name, @Nullable String dfltVal) {
+ if (props != null) {
+ String res = props.get(name);
+
+ if (res != null)
+ return res;
+ }
+
+ return dfltVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> properties() {
+ return props != null ? props : Collections.<String, String>emptyMap();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long length() {
+ return len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
index dad47f7..5859f42 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/LocalIgfsSecondaryFileSystem.java
@@ -214,7 +214,7 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
"[src=" + src + ", dest=" + dest + ']');
}
catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to rename [src=" + src + ", dest="+ dest + ']');
+ throw handleSecondaryFsError(e, "Failed to rename [src=" + src + ", dest=" + dest + ']');
}
}
@@ -253,10 +253,7 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
}
}
- if (!dir.delete())
- return false;
-
- return true;
+ return dir.delete();
}
/** {@inheritDoc} */
@@ -377,6 +374,7 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
// TODO: IGNITE-3648.
return create0(path, overwrite, bufSize);
}
+
/** {@inheritDoc} */
@Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
@Nullable Map<String, String> props) {
@@ -402,76 +400,17 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
/** {@inheritDoc} */
@Override public IgfsFile info(final IgfsPath path) {
- try {
- // TODO: IGNITE-3650.
- final FileStatus status = fileSystemForUser().getFileStatus(convert(path));
-
- if (status == null)
- return null;
-
- final Map<String, String> props = properties(status);
-
- return new IgfsFile() {
- @Override public IgfsPath path() {
- return path;
- }
-
- @Override public boolean isFile() {
- return status.isFile();
- }
-
- @Override public boolean isDirectory() {
- return status.isDirectory();
- }
-
- @Override public int blockSize() {
- // By convention directory has blockSize == 0, while file has blockSize > 0:
- return isDirectory() ? 0 : (int)status.getBlockSize();
- }
-
- @Override public long groupBlockSize() {
- return status.getBlockSize();
- }
-
- @Override public long accessTime() {
- return status.getAccessTime();
- }
-
- @Override public long modificationTime() {
- return status.getModificationTime();
- }
-
- @Override public String property(String name) throws IllegalArgumentException {
- String val = props.get(name);
-
- if (val == null)
- throw new IllegalArgumentException("Property not found [path=" + path + ", name=" + name + ']');
-
- return val;
- }
-
- @Nullable @Override public String property(String name, @Nullable String dfltVal) {
- String val = props.get(name);
+ File f = fileForPath(path);
- return val == null ? dfltVal : val;
- }
+ if (!f.exists())
+ return null;
- @Override public long length() {
- return status.getLen();
- }
+ boolean isDir = f.isDirectory();
- /** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return props;
- }
- };
- }
- catch (FileNotFoundException ignore) {
- return null;
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
- }
+ if (isDir)
+ return new LocalFileSystemIgfsFile(path, false, true, 0, f.lastModified(), 0, null);
+ else
+ return new LocalFileSystemIgfsFile(path, f.isFile(), false, 0, f.lastModified(), f.length(), null);
}
/** {@inheritDoc} */
@@ -489,6 +428,7 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
/**
* Gets the FileSystem for the current context user.
+ *
* @return the FileSystem instance, never null.
*/
private FileSystem fileSystemForUser() {
@@ -513,7 +453,7 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
fsFactory = new CachingHadoopFileSystemFactory();
if (fsFactory instanceof LifecycleAware)
- ((LifecycleAware) fsFactory).start();
+ ((LifecycleAware)fsFactory).start();
workDir = new File(workDir).getAbsolutePath();
@@ -523,7 +463,7 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
/** {@inheritDoc} */
@Override public void stop() throws IgniteException {
if (fsFactory instanceof LifecycleAware)
- ((LifecycleAware)fsFactory).stop();
+ ((LifecycleAware)fsFactory).stop();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/09982a64/modules/hadoop/src/test/java/org/apache/ignite/igfs/LocalSecondaryFileSystemDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/LocalSecondaryFileSystemDualAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/LocalSecondaryFileSystemDualAbstractSelfTest.java
index 5043625..8efe2c9 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/LocalSecondaryFileSystemDualAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/LocalSecondaryFileSystemDualAbstractSelfTest.java
@@ -32,7 +32,9 @@ public abstract class LocalSecondaryFileSystemDualAbstractSelfTest extends IgfsD
private static final String FS_WORK_DIR = U.getIgniteHome() + File.separatorChar + "work"
+ File.separatorChar + "fs";
- /** Constructor. */
+ /** Constructor.
+ * @param mode IGFS mode.
+ */
public LocalSecondaryFileSystemDualAbstractSelfTest(IgfsMode mode) {
super(mode);
}
@@ -58,11 +60,6 @@ public abstract class LocalSecondaryFileSystemDualAbstractSelfTest extends IgfsD
}
/** {@inheritDoc} */
- @Override protected boolean appendSupported() {
- return false;
- }
-
- /** {@inheritDoc} */
@Override protected boolean permissionsSupported() {
return false;
}