You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/09/30 14:46:15 UTC
[5/7] ignite git commit: IGNITE-1515: Fixed delete.
IGNITE-1515: Fixed delete.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/54bb7d76
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/54bb7d76
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/54bb7d76
Branch: refs/heads/ignite-1534
Commit: 54bb7d760d9c1261a64fcf3d0be858be7b14444e
Parents: be906e3
Author: iveselovskiy <iv...@gridgain.com>
Authored: Wed Sep 30 10:05:19 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Sep 30 10:05:19 2015 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/igfs/IgfsPath.java | 2 +-
.../internal/processors/igfs/IgfsFileInfo.java | 2 +-
.../internal/processors/igfs/IgfsImpl.java | 57 +----
.../processors/igfs/IgfsMetaManager.java | 251 +++++++++++++++----
.../ignite/igfs/IgfsFragmentizerSelfTest.java | 2 +-
.../processors/igfs/IgfsAbstractSelfTest.java | 80 ++++--
.../igfs/IgfsMetaManagerSelfTest.java | 6 -
.../processors/igfs/IgfsMetricsSelfTest.java | 2 +-
.../processors/igfs/IgfsProcessorSelfTest.java | 29 +--
.../igfs/UniversalFileSystemAdapter.java | 1 -
.../processors/hadoop/igfs/HadoopIgfsUtils.java | 36 +++
...oopFileSystemUniversalFileSystemAdapter.java | 4 +-
.../HadoopIgfs20FileSystemAbstractSelfTest.java | 7 +-
.../IgniteHadoopFileSystemAbstractSelfTest.java | 5 +-
14 files changed, 345 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
index a99c1ee..fb0621c 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
@@ -50,7 +50,7 @@ public final class IgfsPath implements Comparable<IgfsPath>, Externalizable {
private static final char SLASH_CHAR = '/';
/** The directory separator. */
- private static final String SLASH = "/";
+ public static final String SLASH = "/";
/** URI representing this path. Should never change after object creation or de-serialization. */
private String path;
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
index 116d585..8564500 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
@@ -216,7 +216,7 @@ public final class IgfsFileInfo implements Externalizable {
* @param listing New directory listing.
* @param old Old file info.
*/
- IgfsFileInfo(Map<String, IgfsListingEntry> listing, IgfsFileInfo old) {
+ IgfsFileInfo(@Nullable Map<String, IgfsListingEntry> listing, IgfsFileInfo old) {
this(old.isDirectory(), old.id, old.blockSize, old.len, old.affKey, listing, old.props, old.fileMap(),
old.lockId, false, old.accessTime, old.modificationTime, old.evictExclude());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 0dd0307..d5ba95f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -53,7 +53,6 @@ import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.IgfsEvent;
import org.apache.ignite.igfs.IgfsBlockLocation;
-import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsInvalidPathException;
import org.apache.ignite.igfs.IgfsMetrics;
@@ -712,6 +711,9 @@ public final class IgfsImpl implements IgfsEx {
if (log.isDebugEnabled())
log.debug("Deleting file [path=" + path + ", recursive=" + recursive + ']');
+ if (IgfsPath.SLASH.equals(path.toString()))
+ return false;
+
IgfsMode mode = resolveMode(path);
Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);
@@ -721,8 +723,11 @@ public final class IgfsImpl implements IgfsEx {
FileDescriptor desc = getFileDescriptor(path);
if (childrenModes.contains(PRIMARY)) {
- if (desc != null)
- res = delete0(desc, path.parent(), recursive);
+ if (desc != null) {
+ IgniteUuid deletedId = meta.softDelete(path, recursive);
+
+ res = deletedId != null;
+ }
else if (mode == PRIMARY)
checkConflictWithPrimary(path);
}
@@ -750,48 +755,6 @@ public final class IgfsImpl implements IgfsEx {
});
}
- /**
- * Internal procedure for (optionally) recursive file and directory deletion.
- *
- * @param desc File descriptor of file or directory to delete.
- * @param parentPath Parent path. If specified, events will be fired for each deleted file
- * or directory. If not specified, events will not be fired.
- * @param recursive Recursive deletion flag.
- * @return {@code True} if file was successfully deleted. If directory is not empty and
- * {@code recursive} flag is false, will return {@code false}.
- * @throws IgniteCheckedException In case of error.
- */
- private boolean delete0(FileDescriptor desc, @Nullable IgfsPath parentPath, boolean recursive)
- throws IgniteCheckedException {
- IgfsPath curPath = parentPath == null ? new IgfsPath() : new IgfsPath(parentPath, desc.fileName);
-
- if (desc.isFile) {
- deleteFile(curPath, desc, true);
-
- return true;
- }
- else {
- if (recursive) {
- meta.softDelete(desc.parentId, desc.fileName, desc.fileId);
-
- return true;
- }
- else {
- Map<String, IgfsListingEntry> infoMap = meta.directoryListing(desc.fileId);
-
- if (F.isEmpty(infoMap)) {
- deleteFile(curPath, desc, true);
-
- return true;
- }
- else
- // Throw exception if not empty and not recursive.
- throw new IgfsDirectoryNotEmptyException("Failed to remove directory (directory is not empty " +
- "and recursive flag is not set)");
- }
- }
- }
-
/** {@inheritDoc} */
@Override public void mkdirs(IgfsPath path) {
mkdirs(path, null);
@@ -1454,7 +1417,7 @@ public final class IgfsImpl implements IgfsEx {
*/
IgniteInternalFuture<?> formatAsync() {
try {
- IgniteUuid id = meta.softDelete(null, null, ROOT_ID);
+ IgniteUuid id = meta.format();
if (id == null)
return new GridFinishedFuture<Object>();
@@ -1526,6 +1489,8 @@ public final class IgfsImpl implements IgfsEx {
* @throws IgniteCheckedException If failed.
*/
@Nullable private FileDescriptor getFileDescriptor(IgfsPath path) throws IgniteCheckedException {
+ assert path != null;
+
List<IgniteUuid> ids = meta.fileIds(path);
IgfsFileInfo fileInfo = meta.info(ids.get(ids.size() - 1));
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index d283b64..bb6404c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -34,6 +34,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
@@ -95,6 +96,20 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA
*/
@SuppressWarnings("all")
public class IgfsMetaManager extends IgfsManager {
+ /** Comparator for Id sorting. */
+ private static final Comparator<IgniteUuid> PATH_ID_SORTING_COMPARATOR
+ = new Comparator<IgniteUuid>() {
+ @Override public int compare(IgniteUuid u1, IgniteUuid u2) {
+ if (u1 == u2)
+ return 0;
+
+ if (u1 == null)
+ return -1;
+
+ return u1.compareTo(u2);
+ }
+ };
+
/** IGFS configuration. */
private FileSystemConfiguration cfg;
@@ -376,6 +391,7 @@ public class IgfsMetaManager extends IgfsManager {
/**
* Gets file info by its ID.
+ * NB: this method is used both in Tx and out of Tx.
*
* @param fileId File ID to get details for.
* @return File info.
@@ -593,12 +609,39 @@ public class IgfsMetaManager extends IgfsManager {
}
/**
+ * Answers if the collection is sorted.
+ *
+ * @param col The collection to check.
+ * @param <T> The type of the collection elements.
+ * @return If the collection sorted.
+ */
+ private static <T extends Comparable<T>> boolean isSorted(Collection<T> col) {
+ T prev = null;
+
+ for (T t: col) {
+ if (t == null)
+ throw new NullPointerException("Collections should not contain nulls");
+
+ if (prev != null && prev.compareTo(t) > 0)
+ return false; // disordered.
+
+ prev = t;
+ }
+
+ return true;
+ }
+
+ /**
* Lock file IDs.
+ *
* @param fileIds File IDs (sorted).
* @return Map with lock info.
* @throws IgniteCheckedException If failed.
*/
private Map<IgniteUuid, IgfsFileInfo> lockIds(Collection<IgniteUuid> fileIds) throws IgniteCheckedException {
+ assert isSorted(fileIds);
+ assert validTxState(true);
+
if (log.isDebugEnabled())
log.debug("Locking file ids: " + fileIds);
@@ -608,19 +651,34 @@ public class IgfsMetaManager extends IgfsManager {
if (log.isDebugEnabled())
log.debug("Locked file ids: " + fileIds);
- // Force root ID always exist in cache.
- if (fileIds.contains(ROOT_ID) && !map.containsKey(ROOT_ID)) {
- IgfsFileInfo info = new IgfsFileInfo();
+ // Force root & trash IDs always exist in cache.
+ addInfoIfNeeded(fileIds, map, ROOT_ID);
+ addInfoIfNeeded(fileIds, map, TRASH_ID);
- id2InfoPrj.putIfAbsent(ROOT_ID, info);
+ // Returns detail's map for locked IDs.
+ return map;
+ }
- map = new GridLeanMap<>(map);
+ /**
+ * Adds FileInfo into the cache if it is requested in fileIds and is not present in the map.
+ *
+ * @param fileIds A list that may contain the id.
+ * @param map The map that may not contain the id.
+ * @param id The id to check.
+ * @throws IgniteCheckedException On error.
+ */
+ private void addInfoIfNeeded(Collection<IgniteUuid> fileIds, Map<IgniteUuid, IgfsFileInfo> map, IgniteUuid id) throws IgniteCheckedException {
+ assert validTxState(true);
- map.put(ROOT_ID, info);
- }
+ if (fileIds.contains(id) && !map.containsKey(id)) {
+ IgfsFileInfo info = new IgfsFileInfo(id);
- // Returns detail's map for locked IDs.
- return map;
+ assert info.listing() != null;
+
+ id2InfoPrj.putIfAbsent(id, info);
+
+ map.put(id, info);
+ }
}
/**
@@ -779,13 +837,11 @@ public class IgfsMetaManager extends IgfsManager {
log.debug("Locking parent id [parentId=" + parentId + ", fileName=" + fileName + ", newFileInfo=" +
newFileInfo + ']');
- validTxState(true);
+ assert validTxState(true);
// Lock only parent file ID.
IgfsFileInfo parentInfo = info(parentId);
- assert validTxState(true);
-
if (parentInfo == null)
throw fsException(new IgfsPathNotFoundException("Failed to lock parent directory (not found): " + parentId));
@@ -798,8 +854,6 @@ public class IgfsMetaManager extends IgfsManager {
IgfsListingEntry entry = parentListing.get(fileName);
- assert validTxState(true);
-
if (entry != null)
return entry.fileId();
@@ -832,18 +886,7 @@ public class IgfsMetaManager extends IgfsManager {
List<IgniteUuid> srcPathIds = fileIds(srcPath);
List<IgniteUuid> dstPathIds = fileIds(dstPath);
- final Set<IgniteUuid> allIds = new TreeSet<>(new Comparator<IgniteUuid>() {
- @Override
- public int compare(IgniteUuid u1, IgniteUuid u2) {
- if (u1 == u2)
- return 0;
-
- if (u1 == null)
- return -1;
-
- return u1.compareTo(u2);
- }
- });
+ final Set<IgniteUuid> allIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR);
allIds.addAll(srcPathIds);
@@ -1009,6 +1052,7 @@ public class IgfsMetaManager extends IgfsManager {
private void moveNonTx(IgniteUuid fileId, @Nullable String srcFileName, IgniteUuid srcParentId, String destFileName,
IgniteUuid destParentId) throws IgniteCheckedException {
assert validTxState(true);
+
assert fileId != null;
assert srcFileName != null;
assert srcParentId != null;
@@ -1026,8 +1070,6 @@ public class IgfsMetaManager extends IgfsManager {
// Lock file ID and parent IDs for this transaction.
Map<IgniteUuid, IgfsFileInfo> infoMap = lockIds(srcParentId, fileId, destParentId);
- validTxState(true);
-
IgfsFileInfo srcInfo = infoMap.get(srcParentId);
if (srcInfo == null)
@@ -1196,6 +1238,66 @@ public class IgfsMetaManager extends IgfsManager {
}
/**
+ * Deletes (moves to TRASH) all elements under the root folder.
+ *
+ * @return The new Id if the artificially created folder containing all former root
+ * elements moved to TRASH folder.
+ * @throws IgniteCheckedException On error.
+ */
+ IgniteUuid format() throws IgniteCheckedException {
+ if (busyLock.enterBusy()) {
+ try {
+ assert validTxState(false);
+
+ final IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ);
+
+ try {
+ // NB: We may lock root because its id is less than any other id:
+ final IgfsFileInfo rootInfo = lockIds(ROOT_ID, TRASH_ID).get(ROOT_ID);
+
+ assert rootInfo != null;
+
+ Map<String, IgfsListingEntry> rootListingMap = rootInfo.listing();
+
+ assert rootListingMap != null;
+
+ if (rootListingMap.isEmpty())
+ return null; // Root is empty, nothing to do.
+
+ // Construct new info and move locked entries from root to it.
+ Map<String, IgfsListingEntry> transferListing = new HashMap<>(rootListingMap);
+
+ IgfsFileInfo newInfo = new IgfsFileInfo(transferListing);
+
+ id2InfoPrj.put(newInfo.id(), newInfo);
+
+ // Add new info to trash listing.
+ id2InfoPrj.invoke(TRASH_ID, new UpdateListing(newInfo.id().toString(),
+ new IgfsListingEntry(newInfo), false));
+
+ // Remove listing entries from root.
+ // Note that root directory properties and other attributes are preserved:
+ id2InfoPrj.put(ROOT_ID, new IgfsFileInfo(null/*listing*/, rootInfo));
+
+ tx.commit();
+
+ delWorker.signal();
+
+ return newInfo.id();
+ }
+ finally {
+ tx.close();
+ }
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to perform format because Grid is stopping.");
+ }
+
+ /**
* Move path to the trash directory.
*
* @param parentId Parent ID.
@@ -1204,26 +1306,86 @@ public class IgfsMetaManager extends IgfsManager {
* @return ID of an entry located directly under the trash directory.
* @throws IgniteCheckedException If failed.
*/
- IgniteUuid softDelete(@Nullable IgniteUuid parentId, @Nullable String pathName, IgniteUuid pathId) throws IgniteCheckedException {
+ IgniteUuid softDelete(final IgfsPath path, final boolean recursive) throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
assert validTxState(false);
- IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ);
+ final SortedSet<IgniteUuid> allIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR);
+
+ List<IgniteUuid> pathIdList = fileIds(path);
+
+ assert pathIdList.size() > 1;
+
+ final IgniteUuid victimId = pathIdList.get(pathIdList.size() - 1);
+
+ assert !TRASH_ID.equals(victimId) : "TRASH does not have path, it cannot ever be deletion victim.";
+ assert !ROOT_ID.equals(victimId); // root deletion is prevented in earlier stages.
+
+ allIds.addAll(pathIdList);
+
+ if (allIds.remove(null))
+ return null; // A fragment of the path no longer exists.
+
+ boolean added = allIds.add(TRASH_ID);
+ assert added;
+
+ final IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ);
try {
- if (parentId == null)
- lockIds(pathId, TRASH_ID);
- else
- lockIds(parentId, pathId, TRASH_ID);
+ final Map<IgniteUuid, IgfsFileInfo> infoMap = lockIds(allIds);
+
+ // Directory starure was changed concurrently, so the original path no longer exists:
+ if (!verifyPathIntegrity(path, pathIdList, infoMap))
+ return null;
+
+ final IgfsFileInfo victimInfo = infoMap.get(victimId);
+
+ if (!recursive && victimInfo.isDirectory() && !victimInfo.listing().isEmpty())
+ // Throw exception if not empty and not recursive.
+ throw new IgfsDirectoryNotEmptyException("Failed to remove directory (directory is not " +
+ "empty and recursive flag is not set).");
+
+ IgfsFileInfo destInfo = infoMap.get(TRASH_ID);
- IgniteUuid resId = softDeleteNonTx(parentId, pathName, pathId);
+ assert destInfo != null;
+
+ final String srcFileName = path.name();
+
+ final String destFileName = victimId.toString();
+
+ assert destInfo.listing().get(destFileName) == null : "Failed to add file name into the " +
+ "destination directory (file already exists) [destName=" + destFileName + ']';
+
+ IgfsFileInfo srcParentInfo = infoMap.get(pathIdList.get(pathIdList.size() - 2));
+
+ assert srcParentInfo != null;
+
+ IgniteUuid srcParentId = srcParentInfo.id();
+ assert srcParentId.equals(pathIdList.get(pathIdList.size() - 2));
+
+ IgfsListingEntry srcEntry = srcParentInfo.listing().get(srcFileName);
+
+ assert srcEntry != null : "Deletion victim not found in parent listing [path=" + path +
+ ", name=" + srcFileName + ", listing=" + srcParentInfo.listing() + ']';
+
+ assert victimId.equals(srcEntry.fileId());
+
+ id2InfoPrj.invoke(srcParentId, new UpdateListing(srcFileName, srcEntry, true));
+
+ // Add listing entry into the destination parent listing.
+ id2InfoPrj.invoke(TRASH_ID, new UpdateListing(destFileName, srcEntry, false));
+
+ if (victimInfo.isFile())
+ // Update a file info of the removed file with a file path,
+ // which will be used by delete worker for event notifications.
+ id2InfoPrj.invoke(victimId, new UpdatePath(path));
tx.commit();
delWorker.signal();
- return resId;
+ return victimId;
}
finally {
tx.close();
@@ -1234,8 +1396,8 @@ public class IgfsMetaManager extends IgfsManager {
}
}
else
- throw new IllegalStateException("Failed to perform soft delete because Grid is stopping [parentId=" +
- parentId + ", pathName=" + pathName + ", pathId=" + pathId + ']');
+ throw new IllegalStateException("Failed to perform soft delete because Grid is " +
+ "stopping [path=" + path + ']');
}
/**
@@ -1316,6 +1478,7 @@ public class IgfsMetaManager extends IgfsManager {
/**
* Remove listing entries of the given parent.
+ * This operation actually deletes directories from TRASH, is used solely by IgfsDeleteWorker.
*
* @param parentId Parent ID.
* @param listing Listing entries.
@@ -1403,6 +1566,7 @@ public class IgfsMetaManager extends IgfsManager {
/**
* Remove entry from the metadata listing.
+ * Used solely by IgfsDeleteWorker.
*
* @param parentId Parent ID.
* @param name Name.
@@ -2394,17 +2558,18 @@ public class IgfsMetaManager extends IgfsManager {
if (busyLock.enterBusy()) {
try {
SynchronizationTask<IgfsFileInfo> task = new SynchronizationTask<IgfsFileInfo>() {
- @Override public IgfsFileInfo onSuccess(Map<IgfsPath, IgfsFileInfo> infos)
- throws Exception {
+ @Override public IgfsFileInfo onSuccess(Map<IgfsPath, IgfsFileInfo> infos) throws Exception {
if (infos.get(path) == null)
return null;
fs.update(path, props);
- assert path.parent() == null || infos.get(path.parent()) != null;
+ IgfsFileInfo parentInfo = infos.get(path.parent());
+
+ assert path.parent() == null || parentInfo != null;
- return updatePropertiesNonTx(infos.get(path.parent()).id(), infos.get(path).id(), path.name(),
- props);
+ return updatePropertiesNonTx(parentInfo == null ? null : parentInfo.id(),
+ infos.get(path).id(), path.name(), props);
}
@Override public IgfsFileInfo onFailure(@Nullable Exception err) throws IgniteCheckedException {
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
index ebf91e0..fd4ec17 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
@@ -237,7 +237,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
U.sleep(200);
}
- igfs.delete(new IgfsPath("/"), true);
+ igfs.format();
igfs.awaitDeletesAsync().get();
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index cfa99ff..7e73859 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -101,10 +101,10 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
protected static final long BLOCK_SIZE = 32 * 1024 * 1024;
/** Default repeat count. */
- protected static final int REPEAT_CNT = 5;
+ protected static final int REPEAT_CNT = 5; // Diagnostic: ~100; Regression: 5
/** Concurrent operations count. */
- protected static final int OPS_CNT = 16;
+ protected static final int OPS_CNT = 16; // Diagnostic: ~160; Regression: 16
/** Seed. */
protected static final long SEED = System.currentTimeMillis();
@@ -252,6 +252,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
clear(igfs, igfsSecondary);
+
+ assert igfs.listFiles(new IgfsPath("/")).isEmpty();
}
/** {@inheritDoc} */
@@ -923,6 +925,23 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
}
/**
+ * Tests that root directory properties persist afetr the #format() operation.
+ *
+ * @throws Exception If failed.
+ */
+ public void testRootPropertiesPersistAfterFormat() throws Exception {
+ igfs.update(new IgfsPath("/"), Collections.singletonMap("foo", "moo"));
+
+ igfs.format();
+
+ IgfsFile file = igfs.info(new IgfsPath("/"));
+
+ Map<String,String> props = file.properties();
+
+ assertEquals("moo", props.get("foo"));
+ }
+
+ /**
* Test regular file open.
*
* @throws Exception If failed.
@@ -1600,6 +1619,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testConcurrentMkdirsDelete() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-1541");
+
for (int i = 0; i < REPEAT_CNT; i++) {
final CyclicBarrier barrier = new CyclicBarrier(2);
@@ -1881,18 +1902,10 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testDeadlocksRename() throws Exception {
for (int i = 0; i < REPEAT_CNT; i++) {
try {
- info(">>>>>> Start deadlock test.");
-
checkDeadlocks(5, 2, 2, 2, OPS_CNT, 0, 0, 0, 0);
-
- info(">>>>>> End deadlock test.");
}
finally {
- info(">>>>>> Start cleanup.");
-
clear(igfs, igfsSecondary);
-
- info(">>>>>> End cleanup.");
}
}
}
@@ -1903,8 +1916,6 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testDeadlocksDelete() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-1515");
-
for (int i = 0; i < REPEAT_CNT; i++) {
try {
checkDeadlocks(5, 2, 2, 2, 0, OPS_CNT, 0, 0, 0);
@@ -1948,6 +1959,42 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
}
/**
+ * Ensure that deadlocks do not occur during concurrent delete & rename operations.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDeadlocksDeleteRename() throws Exception {
+ for (int i = 0; i < REPEAT_CNT; i++) {
+ try {
+ checkDeadlocks(5, 2, 2, 2,
+ OPS_CNT, OPS_CNT, 0, 0, 0);
+ }
+ finally {
+ clear(igfs, igfsSecondary);
+ }
+ }
+ }
+
+ /**
+ * Ensure that deadlocks do not occur during concurrent delete & rename operations.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDeadlocksDeleteMkdirs() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-1541");
+
+ for (int i = 0; i < REPEAT_CNT; i++) {
+ try {
+ checkDeadlocks(5, 2, 2, 2,
+ 0, OPS_CNT, 0, OPS_CNT, 0);
+ }
+ finally {
+ clear(igfs, igfsSecondary);
+ }
+ }
+ }
+
+ /**
* Ensure that deadlocks do not occur during concurrent file creation operations.
*
* @throws Exception If failed.
@@ -1969,11 +2016,16 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testDeadlocks() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-1515");
+ fail("https://issues.apache.org/jira/browse/IGNITE-1541");
for (int i = 0; i < REPEAT_CNT; i++) {
try {
- checkDeadlocks(5, 2, 2, 2, OPS_CNT, OPS_CNT, OPS_CNT, OPS_CNT, OPS_CNT);
+ checkDeadlocks(5, 2, 2, 2,
+ OPS_CNT, // rename
+ OPS_CNT, // delete
+ OPS_CNT, // update
+ OPS_CNT, // mkdirs
+ OPS_CNT); // create
}
finally {
clear(igfs, igfsSecondary);
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
index 206c9fe..4072636 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
@@ -300,25 +300,19 @@ public class IgfsMetaManagerSelfTest extends IgfsCommonAbstractTest {
System.out.println("b: " + mgr.directoryListing(b.id()));
System.out.println("f3: " + mgr.directoryListing(f3.id()));
- //mgr.move(a.id(), "a", ROOT_ID, "a2", ROOT_ID);
mgr.move(path("/a"), path("/a2"));
- //mgr.move(b.id(), "b", a.id(), "b2", a.id());
mgr.move(path("/a2/b"), path("/a2/b2"));
assertNotNull(mgr.info(b.id()));
- //mgr.move(f3.id(), "f3", b.id(), "f3-2", a.id());
mgr.move(path("/a2/b2/f3"), path("/a2/b2/f3-2"));
assertNotNull(mgr.info(b.id()));
- //mgr.move(f3.id(), "f3-2", a.id(), "f3", b.id());
mgr.move(path("/a2/b2/f3-2"), path("/a2/b2/f3"));
- //mgr.move(b.id(), "b2", a.id(), "b", a.id());
mgr.move(path("/a2/b2"), path("/a2/b"));
- //mgr.move(a.id(), "a2", ROOT_ID, "a", ROOT_ID);
mgr.move(path("/a2"), path("/a"));
// Validate 'remove' operation.
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
index 8a2e5bf..fb1d6f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
@@ -348,7 +348,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
assertEquals(0, m.filesOpenedForRead());
assertEquals(0, m.filesOpenedForWrite());
- fs.delete(new IgfsPath("/"), true);
+ fs.format();
m = fs.metrics();
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
index cb134f8..9c4d832 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
@@ -389,15 +389,6 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
assert paths.size() == 3 : "Unexpected paths: " + paths;
- // Delete.
- GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfs.delete(path("/"), false);
-
- return null;
- }
- }, IgfsException.class, null);
-
igfs.delete(path("/A1/B1/C1"), false);
assertNull(igfs.info(path("/A1/B1/C1")));
@@ -416,19 +407,19 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
assertEquals(Arrays.asList(path("/A"), path("/A1"), path("/A2")), sorted(igfs.listPaths(path("/"))));
- GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfs.delete(path("/"), false);
-
- return null;
- }
- }, IgfsException.class, null);
- assertEquals(Arrays.asList(path("/A"), path("/A1"), path("/A2")), sorted(igfs.listPaths(path("/"))));
-
+ // Delete root when it is not empty:
igfs.delete(path("/"), true);
+ igfs.delete(path("/"), false);
+
+ igfs.delete(path("/A"), true);
+ igfs.delete(path("/A1"), true);
+ igfs.delete(path("/A2"), true);
assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(path("/")));
+ // Delete root when it is empty:
igfs.delete(path("/"), false);
+ igfs.delete(path("/"), true);
+
assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(path("/")));
for (Cache.Entry<Object, Object> e : metaCache)
@@ -608,7 +599,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
assertEquals(text, read("/b"));
// Cleanup.
- igfs.delete(root, true);
+ igfs.format();
assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(root));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
index 3bf70e2..ba8c164 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
@@ -28,7 +28,6 @@ import java.util.Map;
* To be used solely in tests.
*/
public interface UniversalFileSystemAdapter {
-
/**
* Gets name of the FS.
* @return name of this file system.
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
index 3913cbd..fa5cbc5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
@@ -20,7 +20,11 @@ package org.apache.ignite.internal.processors.hadoop.igfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathExistsException;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.ignite.IgniteCheckedException;
@@ -130,6 +134,38 @@ public class HadoopIgfsUtils {
}
/**
+ * Deletes all files from the given file system.
+ *
+ * @param fs The file system to clean up.
+ * @throws IOException On error.
+ */
+ public static void clear(FileSystem fs) throws IOException {
+ // Delete root contents:
+ FileStatus[] statuses = fs.listStatus(new Path("/"));
+
+ if (statuses != null) {
+ for (FileStatus stat: statuses)
+ fs.delete(stat.getPath(), true);
+ }
+ }
+
+ /**
+ * Deletes all files from the given file system.
+ *
+ * @param fs The file system to clean up.
+ * @throws IOException On error.
+ */
+ public static void clear(AbstractFileSystem fs) throws IOException {
+ // Delete root contents:
+ FileStatus[] statuses = fs.listStatus(new Path("/"));
+
+ if (statuses != null) {
+ for (FileStatus stat: statuses)
+ fs.delete(stat.getPath(), true);
+ }
+ }
+
+ /**
* Constructor.
*/
private HadoopIgfsUtils() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
index 03f0066..608bd25 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils;
import org.apache.ignite.internal.processors.igfs.IgfsEx;
import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter;
@@ -33,7 +34,6 @@ import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter;
* Universal adapter wrapping {@link org.apache.hadoop.fs.FileSystem} instance.
*/
public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFileSystemAdapter {
-
/** The wrapped filesystem. */
private final FileSystem fileSys;
@@ -70,7 +70,7 @@ public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFile
/** {@inheritDoc} */
@Override public void format() throws IOException {
- fileSys.delete(new Path("/"), true);
+ HadoopIgfsUtils.clear(fileSys);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
index 1235786..c938571 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
@@ -59,6 +59,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.FileSystemConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils;
import org.apache.ignite.internal.processors.igfs.IgfsCommonAbstractTest;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.typedef.F;
@@ -352,7 +353,7 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
try {
- fs.delete(new Path("/"), true);
+ HadoopIgfsUtils.clear(fs);
}
catch (Exception ignore) {
// No-op.
@@ -627,7 +628,9 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
Path root = new Path(fsHome, "/");
- assertTrue(fs.delete(root, true));
+ assertFalse(fs.delete(root, true));
+
+ assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true));
assertPathDoesNotExist(fs, someDir3);
assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 5ea841e..2626ebb 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -300,7 +300,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
try {
- fs.delete(new Path("/"), true);
+ HadoopIgfsUtils.clear(fs);
}
catch (Exception ignore) {
// No-op.
@@ -783,7 +783,8 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
Path root = new Path(fsHome, "/");
- assertTrue(fs.delete(root, true));
+ assertFalse(fs.delete(root, true));
+ assertTrue(fs.delete(new Path("/someDir1"), true));
assertPathDoesNotExist(fs, someDir3);
assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));