You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ak...@apache.org on 2016/08/31 06:33:07 UTC
[17/38] ignite git commit: IGNITE-3670 IGFS: Improved symlink
handling for delete operation and added more tests. This closes #975.
IGNITE-3670 IGFS: Improved symlink handling for delete operation and added more tests. This closes #975.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b5757642
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b5757642
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b5757642
Branch: refs/heads/ignite-3443
Commit: b5757642e135908d9baa027a605035dd0d4acfc9
Parents: 92f18bf
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Fri Aug 26 15:47:02 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Fri Aug 26 15:47:02 2016 +0300
----------------------------------------------------------------------
.../local/LocalIgfsSecondaryFileSystem.java | 38 +-
.../igfs/IgfsAbstractBaseSelfTest.java | 1067 ++++++++++++++++++
.../processors/igfs/IgfsAbstractSelfTest.java | 1012 +----------------
...SecondaryFileSystemDualAbstractSelfTest.java | 143 +++
4 files changed, 1239 insertions(+), 1021 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5757642/modules/core/src/main/java/org/apache/ignite/igfs/secondary/local/LocalIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/local/LocalIgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/local/LocalIgfsSecondaryFileSystem.java
index 3d3a350..519f472 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/local/LocalIgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/local/LocalIgfsSecondaryFileSystem.java
@@ -41,6 +41,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
@@ -108,35 +110,43 @@ public class LocalIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, Li
@Override public boolean delete(IgfsPath path, boolean recursive) {
File f = fileForPath(path);
- if (!recursive || !f.isDirectory())
+ if (!recursive)
return f.delete();
else
- return deleteDirectory(f);
+ return deleteRecursive(f);
}
/**
* Delete directory recursively.
*
- * @param dir Directory.
+ * @param f Directory.
* @return {@code true} if successful.
*/
- private boolean deleteDirectory(File dir) {
- File[] entries = dir.listFiles();
+ private boolean deleteRecursive(File f) {
+ BasicFileAttributes attrs;
+
+ try {
+ attrs = Files.readAttributes(f.toPath(), BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+ }
+ catch (IOException ignore) {
+ return false;
+ }
+
+ if (!attrs.isDirectory() || attrs.isSymbolicLink())
+ return f.delete();
+
+ File[] entries = f.listFiles();
if (entries != null) {
for (File entry : entries) {
- if (entry.isDirectory())
- deleteDirectory(entry);
- else if (entry.isFile()) {
- if (!entry.delete())
- return false;
- }
- else
- throw new UnsupportedOperationException("Symlink deletion is not yet supported: " + entry);
+ boolean res = deleteRecursive(entry);
+
+ if (!res)
+ return false;
}
}
- return dir.delete();
+ return f.delete();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5757642/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java
new file mode 100644
index 0000000..9575bd0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java
@@ -0,0 +1,1067 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.FileSystemConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper;
+import org.apache.ignite.igfs.IgfsInputStream;
+import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
+import org.apache.ignite.igfs.IgfsIpcEndpointType;
+import org.apache.ignite.igfs.IgfsMode;
+import org.apache.ignite.igfs.IgfsOutputStream;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMemoryMode.ONHEAP_TIERED;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.igfs.IgfsMode.PRIMARY;
+import static org.apache.ignite.igfs.IgfsMode.PROXY;
+
+/**
+ * Test fo regular igfs operations.
+ */
+@SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "ConstantConditions"})
+public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest {
+ /** IGFS block size. */
+ protected static final int IGFS_BLOCK_SIZE = 512 * 1024;
+
+ /** Default block size (32Mb). */
+ protected static final long BLOCK_SIZE = 32 * 1024 * 1024;
+
+ /** Default repeat count. */
+ protected static final int REPEAT_CNT = 5; // Diagnostic: up to 500; Regression: 5
+
+ /** Concurrent operations count. */
+ protected static final int OPS_CNT = 16;
+
+ /** Renames count. */
+ protected static final int RENAME_CNT = OPS_CNT;
+
+ /** Deletes count. */
+ protected static final int DELETE_CNT = OPS_CNT;
+
+ /** Updates count. */
+ protected static final int UPDATE_CNT = OPS_CNT;
+
+ /** Mkdirs count. */
+ protected static final int MKDIRS_CNT = OPS_CNT;
+
+ /** Create count. */
+ protected static final int CREATE_CNT = OPS_CNT;
+
+ /** Time to wait until the caches get empty after format. */
+ private static final long CACHE_EMPTY_TIMEOUT = 30_000L;
+
+ /** Seed to generate random numbers. */
+ protected static final long SEED = System.currentTimeMillis();
+
+ /** Amount of blocks to prefetch. */
+ protected static final int PREFETCH_BLOCKS = 1;
+
+ /** Amount of sequential block reads before prefetch is triggered. */
+ protected static final int SEQ_READS_BEFORE_PREFETCH = 2;
+
+ /** Primary file system REST endpoint configuration map. */
+ protected static final IgfsIpcEndpointConfiguration PRIMARY_REST_CFG;
+
+ /** Secondary file system REST endpoint configuration map. */
+ protected static final IgfsIpcEndpointConfiguration SECONDARY_REST_CFG;
+
+ /** Directory. */
+ protected static final IgfsPath DIR = new IgfsPath("/dir");
+
+ /** Sub-directory. */
+ protected static final IgfsPath SUBDIR = new IgfsPath(DIR, "subdir");
+
+ /** Another sub-directory in the same directory. */
+ protected static final IgfsPath SUBDIR2 = new IgfsPath(DIR, "subdir2");
+
+ /** Sub-directory of the sub-directory. */
+ protected static final IgfsPath SUBSUBDIR = new IgfsPath(SUBDIR, "subsubdir");
+
+ /** File. */
+ protected static final IgfsPath FILE = new IgfsPath(SUBDIR, "file");
+
+ /** Another file in the same directory. */
+ protected static final IgfsPath FILE2 = new IgfsPath(SUBDIR, "file2");
+
+ /** Other directory. */
+ protected static final IgfsPath DIR_NEW = new IgfsPath("/dirNew");
+
+ /** Other subdirectory. */
+ protected static final IgfsPath SUBDIR_NEW = new IgfsPath(DIR_NEW, "subdirNew");
+
+ /** Other sub-directory of the sub-directory. */
+ protected static final IgfsPath SUBSUBDIR_NEW = new IgfsPath(SUBDIR_NEW, "subsubdirNew");
+
+ /** Other file. */
+ protected static final IgfsPath FILE_NEW = new IgfsPath(SUBDIR_NEW, "fileNew");
+
+ /** Default data chunk (128 bytes). */
+ protected static final byte[] chunk = createChunk(128);
+
+ /** Primary IGFS. */
+ protected static IgfsImpl igfs;
+
+ /** Secondary IGFS */
+ protected static IgfsSecondaryFileSystem igfsSecondaryFileSystem;
+
+ /** Secondary file system lower layer "backdoor" wrapped in UniversalFileSystemAdapter: */
+ protected static IgfsSecondaryFileSystemTestAdapter igfsSecondary;
+
+ /** IGFS mode. */
+ protected final IgfsMode mode;
+
+ /** Dual mode flag. */
+ protected final boolean dual;
+
+ /** Memory mode. */
+ protected final CacheMemoryMode memoryMode;
+
+ /** IP finder for primary topology. */
+ protected final TcpDiscoveryVmIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** IP finder for secondary topology. */
+ protected final TcpDiscoveryVmIpFinder secondaryIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Ignite nodes of cluster, excluding the secondary file system node, if any. */
+ protected Ignite[] nodes;
+
+ static {
+ PRIMARY_REST_CFG = new IgfsIpcEndpointConfiguration();
+
+ PRIMARY_REST_CFG.setType(IgfsIpcEndpointType.TCP);
+ PRIMARY_REST_CFG.setPort(10500);
+
+ SECONDARY_REST_CFG = new IgfsIpcEndpointConfiguration();
+
+ SECONDARY_REST_CFG.setType(IgfsIpcEndpointType.TCP);
+ SECONDARY_REST_CFG.setPort(11500);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ */
+ protected IgfsAbstractBaseSelfTest(IgfsMode mode) {
+ this(mode, ONHEAP_TIERED);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ * @param memoryMode Memory mode.
+ */
+ protected IgfsAbstractBaseSelfTest(IgfsMode mode, CacheMemoryMode memoryMode) {
+ assert mode != null && mode != PROXY;
+
+ this.mode = mode;
+ this.memoryMode = memoryMode;
+
+ dual = mode != PRIMARY;
+ }
+
+ /**
+ * @return Relaxed consistency flag.
+ */
+ protected boolean relaxedConsistency() {
+ return false;
+ }
+
+ /**
+ * @return Relaxed consistency flag.
+ */
+ protected boolean initializeDefaultPathModes() {
+ return false;
+ }
+
+ /**
+ * @return Client flag.
+ */
+ protected boolean client() {
+ return false;
+ }
+
+ /**
+ * @return Use optimized marshaller flag.
+ */
+ protected boolean useOptimizedMarshaller() {
+ return false;
+ }
+
+ /**
+ * @return Whether append is supported.
+ */
+ protected boolean appendSupported() {
+ return true;
+ }
+
+ /**
+ * @return Whether permissions are supported.
+ */
+ protected boolean permissionsSupported() {
+ return true;
+ }
+
+ /**
+ * @return Whether properties are supported.
+ */
+ protected boolean propertiesSupported() {
+ return true;
+ }
+
+ /**
+ * @return Whether times are supported.
+ */
+ protected boolean timesSupported() {
+ return true;
+ }
+
+ /**
+ * @return Amount of nodes to start.
+ */
+ protected int nodeCount() {
+ return 1;
+ }
+
+ /**
+ * Data chunk.
+ *
+ * @param len Length.
+ * @return Data chunk.
+ */
+ static byte[] createChunk(int len) {
+ byte[] chunk = new byte[len];
+
+ for (int i = 0; i < chunk.length; i++)
+ chunk[i] = (byte)i;
+
+ return chunk;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ igfsSecondaryFileSystem = createSecondaryFileSystemStack();
+
+ nodes = new Ignite[nodeCount()];
+
+ for (int i = 0; i < nodes.length; i++) {
+ String nodeName = i == 0 ? "ignite" : "ignite" + i;
+
+ nodes[i] = startGridWithIgfs(nodeName, "igfs", mode, igfsSecondaryFileSystem, PRIMARY_REST_CFG,
+ primaryIpFinder);
+ }
+
+ igfs = (IgfsImpl) nodes[0].fileSystem("igfs");
+
+ if (client()) {
+ // Start client.
+ Ignition.setClientMode(true);
+
+ try {
+ Ignite ignite = startGridWithIgfs("ignite-client", "igfs", mode, igfsSecondaryFileSystem,
+ PRIMARY_REST_CFG, primaryIpFinder);
+
+ igfs = (IgfsImpl) ignite.fileSystem("igfs");
+ }
+ finally {
+ Ignition.setClientMode(false);
+ }
+ }
+ }
+
+ /**
+ * Creates secondary file system stack.
+ *
+ * @return The secondary file system.
+ * @throws Exception On error.
+ */
+ protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
+ Ignite igniteSecondary = startGridWithIgfs("ignite-secondary", "igfs-secondary", PRIMARY, null,
+ SECONDARY_REST_CFG, secondaryIpFinder);
+
+ IgfsEx secondaryIgfsImpl = (IgfsEx) igniteSecondary.fileSystem("igfs-secondary");
+
+ igfsSecondary = new DefaultIgfsSecondaryFileSystemTestAdapter(secondaryIgfsImpl);
+
+ return secondaryIgfsImpl.asSecondary();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ clear(igfs, igfsSecondary);
+
+ assert igfs.listFiles(new IgfsPath("/")).isEmpty();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+ }
+
+ /**
+ * Start grid with IGFS.
+ *
+ * @param gridName Grid name.
+ * @param igfsName IGFS name
+ * @param mode IGFS mode.
+ * @param secondaryFs Secondary file system (optional).
+ * @param restCfg Rest configuration string (optional).
+ * @param ipFinder IP finder.
+ * @return Started grid instance.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
+ @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg,
+ TcpDiscoveryIpFinder ipFinder) throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("dataCache");
+ igfsCfg.setMetaCacheName("metaCache");
+ igfsCfg.setName(igfsName);
+ igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
+ igfsCfg.setDefaultMode(mode);
+ igfsCfg.setIpcEndpointConfiguration(restCfg);
+ igfsCfg.setSecondaryFileSystem(secondaryFs);
+ igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
+ igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
+ igfsCfg.setRelaxedConsistency(relaxedConsistency());
+
+ igfsCfg.setInitializeDefaultPathModes(initializeDefaultPathModes());
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setNearConfiguration(null);
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setNearConfiguration(null);
+ dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+ dataCacheCfg.setMemoryMode(memoryMode);
+ dataCacheCfg.setOffHeapMaxMemory(0);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setNearConfiguration(null);
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ if (useOptimizedMarshaller())
+ cfg.setMarshaller(new OptimizedMarshaller());
+
+ cfg.setGridName(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(ipFinder);
+
+ prepareCacheConfigurations(dataCacheCfg, metaCacheCfg);
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ return G.start(cfg);
+ }
+
+ /**
+ * Prepare cache configuration.
+ *
+ * @param dataCacheCfg Data cache configuration.
+ * @param metaCacheCfg Meta cache configuration.
+ */
+ protected void prepareCacheConfigurations(CacheConfiguration dataCacheCfg, CacheConfiguration metaCacheCfg) {
+ // Noop
+ }
+
+ /**
+ * Execute provided task in a separate thread.
+ *
+ * @param task Task to execute.
+ * @return Result.
+ */
+ protected static <T> IgniteInternalFuture<T> execute(final Callable<T> task) {
+ final GridFutureAdapter<T> fut = new GridFutureAdapter<>();
+
+ new Thread(new Runnable() {
+ @Override public void run() {
+ try {
+ fut.onDone(task.call());
+ }
+ catch (Throwable e) {
+ fut.onDone(e);
+ }
+ }
+ }).start();
+
+ return fut;
+ }
+
+
+ /**
+ * Create the given directories and files in the given IGFS.
+ *
+ * @param igfs IGFS.
+ * @param dirs Directories.
+ * @param files Files.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("EmptyTryBlock")
+ public static void create(IgfsImpl igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files) throws Exception {
+ if (dirs != null) {
+ for (IgfsPath dir : dirs)
+ igfs.mkdirs(dir);
+ }
+
+ if (files != null) {
+ for (IgfsPath file : files) {
+ try (OutputStream ignored = igfs.create(file, true)) {
+ // No-op.
+ }
+
+ igfs.await(file);
+ }
+ }
+ }
+
+ /**
+ * Creates specified files/directories
+ *
+ * @param uni The file system to operate on.
+ * @param dirs The directories to create.
+ * @param files The files to create.
+ * @throws Exception On error.
+ */
+ @SuppressWarnings("EmptyTryBlock")
+ public void create(IgfsSecondaryFileSystemTestAdapter uni, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
+ throws Exception {
+ if (dirs != null) {
+ for (IgfsPath dir : dirs)
+ uni.mkdirs(dir.toString());
+ }
+
+ if (files != null) {
+ for (IgfsPath file : files)
+ try (OutputStream ignore = uni.openOutputStream(file.toString(), false)) {
+ // No-op
+ }
+ }
+ }
+
+ /**
+ * Create the file in the given IGFS and write provided data chunks to it.
+ *
+ * @param igfs IGFS.
+ * @param file File.
+ * @param overwrite Overwrite flag.
+ * @param chunks Data chunks.
+ * @throws IOException In case of IO exception.
+ */
+ protected static void createFile(IgfsEx igfs, IgfsPath file, boolean overwrite, @Nullable byte[]... chunks)
+ throws IOException {
+ OutputStream os = null;
+
+ try {
+ os = igfs.create(file, overwrite);
+
+ writeFileChunks(os, chunks);
+ }
+ finally {
+ U.closeQuiet(os);
+
+ awaitFileClose(igfs, file);
+ }
+ }
+
+ /**
+ * Create the file in the given IGFS and write provided data chunks to it.
+ *
+ * @param file File.
+ * @param chunks Data chunks.
+ * @throws IOException In case of IO exception.
+ */
+ protected static void createFile(IgfsSecondaryFileSystemTestAdapter uni, IgfsPath file, @Nullable byte[]... chunks)
+ throws IOException {
+ OutputStream os = null;
+
+ try {
+ os = uni.openOutputStream(file.toString(), false);
+
+ writeFileChunks(os, chunks);
+ }
+ finally {
+ U.closeQuiet(os);
+
+ IgfsEx igfsEx = uni.igfs();
+
+ if (igfsEx != null)
+ awaitFileClose(igfsEx, file);
+ }
+ }
+
+ /**
+ * Create the file in the given IGFS and write provided data chunks to it.
+ *
+ * @param igfs IGFS.
+ * @param file File.
+ * @param overwrite Overwrite flag.
+ * @param blockSize Block size.
+ * @param chunks Data chunks.
+ * @throws Exception If failed.
+ */
+ protected static void createFile(IgfsImpl igfs, IgfsPath file, boolean overwrite, long blockSize,
+ @Nullable byte[]... chunks) throws Exception {
+ IgfsOutputStream os = null;
+
+ try {
+ os = igfs.create(file, 256, overwrite, null, 0, blockSize, null);
+
+ writeFileChunks(os, chunks);
+ }
+ finally {
+ U.closeQuiet(os);
+
+ awaitFileClose(igfs, file);
+ }
+ }
+
+ /**
+ * Append to the file in the given IGFS provided data chunks.
+ *
+ * @param igfs IGFS.
+ * @param file File.
+ * @param chunks Data chunks.
+ * @throws Exception If failed.
+ */
+ protected static void appendFile(IgfsImpl igfs, IgfsPath file, @Nullable byte[]... chunks)
+ throws Exception {
+ IgfsOutputStream os = null;
+
+ try {
+ os = igfs.append(file, false);
+
+ writeFileChunks(os, chunks);
+ }
+ finally {
+ U.closeQuiet(os);
+
+ awaitFileClose(igfs, file);
+ }
+ }
+
+ /**
+ * Write provided data chunks to the file output stream.
+ *
+ * @param os Output stream.
+ * @param chunks Data chunks.
+ * @throws IOException If failed.
+ */
+ protected static void writeFileChunks(OutputStream os, @Nullable byte[]... chunks) throws IOException {
+ if (chunks != null && chunks.length > 0) {
+ for (byte[] chunk : chunks)
+ os.write(chunk);
+ }
+ }
+
+ /**
+ * Await for previously opened output stream to close. This is achieved by requesting dummy update on the file.
+ *
+ * @param igfs IGFS.
+ * @param file File.
+ */
+ public static void awaitFileClose(IgfsSecondaryFileSystem igfs, IgfsPath file) {
+ try {
+ igfs.update(file, Collections.singletonMap("prop", "val"));
+ }
+ catch (IgniteException ignore) {
+ // No-op.
+ }
+ }
+
+ /**
+ * Await for previously opened output stream to close.
+ *
+ * @param igfs IGFS.
+ * @param file File.
+ */
+ public static void awaitFileClose(@Nullable IgfsEx igfs, IgfsPath file) {
+ igfs.await(file);
+ }
+
+ /**
+ * Ensure that the given paths exist in the given IGFSs.
+ *
+ * @param igfs First IGFS.
+ * @param igfsSecondary Second IGFS.
+ * @param paths Paths.
+ * @throws Exception If failed.
+ */
+ protected void checkExist(IgfsImpl igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary, IgfsPath... paths)
+ throws Exception {
+ checkExist(igfs, paths);
+
+ if (dual)
+ checkExist(igfsSecondary, paths);
+ }
+
+ /**
+ * Ensure that the given paths exist in the given IGFS.
+ *
+ * @param igfs IGFS.
+ * @param paths Paths.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected static void checkExist(IgfsImpl igfs, IgfsPath... paths) throws IgniteCheckedException {
+ for (IgfsPath path : paths)
+ assert igfs.exists(path) : "Path doesn't exist [igfs=" + igfs.name() + ", path=" + path + ']';
+ }
+
+ /**
+ * Ensure that the given paths exist in the given IGFS.
+ *
+ * @param uni filesystem.
+ * @param paths Paths.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected void checkExist(IgfsSecondaryFileSystemTestAdapter uni, IgfsPath... paths) throws IgniteCheckedException {
+ IgfsEx ex = uni.igfs();
+
+ for (IgfsPath path : paths) {
+ if (ex != null)
+ assert ex.context().meta().fileId(path) != null : "Path doesn't exist [igfs=" + ex.name() +
+ ", path=" + path + ']';
+
+ try {
+ assert uni.exists(path.toString()) : "Path doesn't exist [igfs=" + uni.name() + ", path=" + path + ']';
+ }
+ catch (IOException ioe) {
+ throw new IgniteCheckedException(ioe);
+ }
+ }
+ }
+
+ /**
+ * Ensure that the given paths don't exist in the given IGFSs.
+ *
+ * @param igfs First IGFS.
+ * @param igfsSecondary Second IGFS.
+ * @param paths Paths.
+ * @throws Exception If failed.
+ */
+ protected void checkNotExist(IgfsImpl igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary, IgfsPath... paths)
+ throws Exception {
+ checkNotExist(igfs, paths);
+
+ if (dual)
+ checkNotExist(igfsSecondary, paths);
+ }
+
+ /**
+ * Ensure that the given paths don't exist in the given IGFS.
+ *
+ * @param igfs IGFS.
+ * @param paths Paths.
+ * @throws Exception If failed.
+ */
+ protected void checkNotExist(IgfsImpl igfs, IgfsPath... paths) throws Exception {
+ for (IgfsPath path : paths)
+ assert !igfs.exists(path) : "Path exists [igfs=" + igfs.name() + ", path=" + path + ']';
+ }
+
+ /**
+ * Ensure that the given paths don't exist in the given IGFS.
+ *
+ * @param uni secondary FS.
+ * @param paths Paths.
+ * @throws Exception If failed.
+ */
+ protected void checkNotExist(IgfsSecondaryFileSystemTestAdapter uni, IgfsPath... paths) throws Exception {
+ IgfsEx ex = uni.igfs();
+
+ for (IgfsPath path : paths) {
+ if (ex != null)
+ assert !ex.exists(path) : "Path exists [igfs=" + ex.name() + ", path=" + path + ']';
+
+ assert !uni.exists(path.toString()) : "Path exists [igfs=" + uni.name() + ", path=" + path + ']';
+ }
+ }
+
+ /**
+ * Ensure that the given file exists in the given IGFSs and that it has exactly the same content as provided in the
+ * "data" parameter.
+ *
+ * @param igfs First IGFS.
+ * @param igfsSecondary Second IGFS.
+ * @param file File.
+ * @param chunks Expected data.
+ * @throws Exception If failed.
+ */
+ protected void checkFile(@Nullable IgfsImpl igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary, IgfsPath file,
+ @Nullable byte[]... chunks) throws Exception {
+ if (igfs != null) {
+ checkExist(igfs, file);
+ checkFileContent(igfs, file, chunks);
+ }
+
+ if (dual) {
+ checkExist(igfsSecondary, file);
+ checkFileContent(igfsSecondary, file.toString(), chunks);
+ }
+ }
+
+ /**
+ * Ensure that the given file has exactly the same content as provided in the "data" parameter.
+ *
+ * @param igfs IGFS.
+ * @param file File.
+ * @param chunks Expected data.
+ * @throws IOException In case of IO exception.
+ * @throws IgniteCheckedException In case of Grid exception.
+ */
+ protected static void checkFileContent(IgfsImpl igfs, IgfsPath file, @Nullable byte[]... chunks)
+ throws IOException, IgniteCheckedException {
+ if (chunks != null && chunks.length > 0) {
+ IgfsInputStream is = null;
+
+ try {
+ is = igfs.open(file);
+
+ int chunkIdx = 0;
+ int pos = 0;
+
+ for (byte[] chunk : chunks) {
+ byte[] buf = new byte[chunk.length];
+
+ is.readFully(pos, buf);
+
+ assert Arrays.equals(chunk, buf) : "Bad chunk [igfs=" + igfs.name() + ", chunkIdx=" + chunkIdx +
+ ", expected=" + Arrays.toString(chunk) + ", actual=" + Arrays.toString(buf) + ']';
+
+ chunkIdx++;
+ pos += chunk.length;
+ }
+
+ is.close();
+ }
+ finally {
+ U.closeQuiet(is);
+ }
+ }
+ }
+
+ /**
+ * Ensure that the given file has exactly the same content as provided in the "data" parameter.
+ *
+ * @param uni FS.
+ * @param path File.
+ * @param chunks Expected data.
+ * @throws IOException In case of IO exception.
+ * @throws IgniteCheckedException In case of Grid exception.
+ */
+ protected void checkFileContent(IgfsSecondaryFileSystemTestAdapter uni, String path, @Nullable byte[]... chunks)
+ throws IOException, IgniteCheckedException {
+ if (chunks != null && chunks.length > 0) {
+ InputStream is = null;
+
+ try {
+ is = uni.openInputStream(path);
+
+ int chunkIdx = 0;
+
+ int read;
+ for (byte[] chunk: chunks) {
+ byte[] buf = new byte[chunk.length];
+
+ read = 0;
+
+ while (true) {
+ int r = is.read(buf, read, buf.length - read);
+
+ read += r;
+
+ if (read == buf.length || r <= 0)
+ break;
+ }
+
+ assert read == chunk.length : "Chunk #" + chunkIdx + " was not read fully:" +
+ " read=" + read + ", expected=" + chunk.length;
+ assert Arrays.equals(chunk, buf) : "Bad chunk [igfs=" + uni.name() + ", chunkIdx=" + chunkIdx +
+ ", expected=" + Arrays.toString(chunk) + ", actual=" + Arrays.toString(buf) + ']';
+
+ chunkIdx++;
+ }
+
+ is.close();
+ }
+ finally {
+ U.closeQuiet(is);
+ }
+ }
+ }
+
+ /**
+ * Create map with properties.
+ *
+ * @param username User name.
+ * @param grpName Group name.
+ * @param perm Permission.
+ * @return Map with properties.
+ */
+ protected Map<String, String> properties(@Nullable String username, @Nullable String grpName,
+ @Nullable String perm) {
+ Map<String, String> props = new HashMap<>();
+
+ if (username != null)
+ props.put(IgfsUtils.PROP_USER_NAME, username);
+
+ if (grpName != null)
+ props.put(IgfsUtils.PROP_GROUP_NAME, grpName);
+
+ if (perm != null)
+ props.put(IgfsUtils.PROP_PERMISSION, perm);
+
+ return props;
+ }
+
+ /**
+ * Convenient method to group paths.
+ *
+ * @param paths Paths to group.
+ * @return Paths as array.
+ */
+ protected static IgfsPath[] paths(IgfsPath... paths) {
+ return paths;
+ }
+
+ /**
+ * Safely clear IGFSs.
+ *
+ * @param igfs First IGFS.
+ * @param igfsSecondary Second IGFS.
+ * @throws Exception If failed.
+ */
+ protected void clear(IgniteFileSystem igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary) throws Exception {
+ clear(igfs);
+
+ if (dual)
+ clear(igfsSecondary);
+ }
+
+ /**
+ * Gets the data cache instance for this IGFS instance.
+ *
+ * @param igfs The IGFS unstance.
+ * @return The data cache.
+ */
+ protected static GridCacheAdapter<IgfsBlockKey, byte[]> getDataCache(IgniteFileSystem igfs) {
+ String dataCacheName = igfs.configuration().getDataCacheName();
+
+ IgniteEx igniteEx = ((IgfsEx)igfs).context().kernalContext().grid();
+
+ return ((IgniteKernal)igniteEx).internalCache(dataCacheName);
+ }
+
+ /**
+ * Gets meta cache.
+ *
+ * @param igfs The IGFS instance.
+ * @return The data cache.
+ */
+ protected static GridCacheAdapter<IgniteUuid, IgfsEntryInfo> getMetaCache(IgniteFileSystem igfs) {
+ String dataCacheName = igfs.configuration().getMetaCacheName();
+
+ IgniteEx igniteEx = ((IgfsEx)igfs).context().kernalContext().grid();
+
+ return ((IgniteKernal)igniteEx).internalCache(dataCacheName);
+ }
+
+ /**
+ * Clear particular IGFS.
+ *
+ * @param igfs IGFS.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public static void clear(IgniteFileSystem igfs) throws Exception {
+ Field workerMapFld = IgfsImpl.class.getDeclaredField("workerMap");
+
+ workerMapFld.setAccessible(true);
+
+ // Wait for all workers to finish.
+ Map<IgfsPath, IgfsFileWorkerBatch> workerMap = (Map<IgfsPath, IgfsFileWorkerBatch>)workerMapFld.get(igfs);
+
+ for (Map.Entry<IgfsPath, IgfsFileWorkerBatch> entry : workerMap.entrySet()) {
+ entry.getValue().cancel();
+
+ try {
+ entry.getValue().await();
+ }
+ catch (IgniteCheckedException e) {
+ if (!(e instanceof IgfsFileWorkerBatchCancelledException))
+ throw e;
+ }
+ }
+
+ // Clear igfs.
+ igfs.format();
+
+ int prevDifferentSize = Integer.MAX_VALUE; // Previous different size.
+ int constCnt = 0, totalCnt = 0;
+ final int constThreshold = 20;
+ final long sleepPeriod = 500L;
+ final long totalThreshold = CACHE_EMPTY_TIMEOUT / sleepPeriod;
+
+ while (true) {
+ int metaSize = 0;
+
+ for (IgniteUuid metaId : getMetaCache(igfs).keySet()) {
+ if (!IgfsUtils.isRootOrTrashId(metaId))
+ metaSize++;
+ }
+
+ int dataSize = getDataCache(igfs).size();
+
+ int size = metaSize + dataSize;
+
+ if (size <= 2)
+ return; // Caches are cleared, we're done. (2 because ROOT & TRASH always exist).
+
+ X.println("Sum size: " + size);
+
+ if (size > prevDifferentSize) {
+ X.println("Summary cache size has grown unexpectedly: size=" + size + ", prevSize=" + prevDifferentSize);
+
+ break;
+ }
+
+ if (totalCnt > totalThreshold) {
+ X.println("Timeout exceeded.");
+
+ break;
+ }
+
+ if (size == prevDifferentSize) {
+ constCnt++;
+
+ if (constCnt == constThreshold) {
+ X.println("Summary cache size stays unchanged for too long: size=" + size);
+
+ break;
+ }
+ } else {
+ constCnt = 0;
+
+ prevDifferentSize = size; // renew;
+ }
+
+ Thread.sleep(sleepPeriod);
+
+ totalCnt++;
+ }
+
+ dumpCache("MetaCache" , getMetaCache(igfs));
+
+ dumpCache("DataCache" , getDataCache(igfs));
+
+ fail("Caches are not empty.");
+ }
+
+ /**
+ * Dumps given cache for diagnostic purposes.
+ *
+ * @param cacheName Name.
+ * @param cache The cache.
+ */
+ private static void dumpCache(String cacheName, GridCacheAdapter<?,?> cache) {
+ X.println("=============================== " + cacheName + " cache dump: ");
+
+ Iterable<? extends GridCacheEntryEx> entries = cache.entries();
+
+ for (GridCacheEntryEx e: entries)
+ X.println("Lost " + cacheName + " entry = " + e);
+ }
+
+ /**
+ * Clear particular {@link IgfsSecondaryFileSystemTestAdapter}.
+ *
+ * @param uni IGFS.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public static void clear(IgfsSecondaryFileSystemTestAdapter uni) throws Exception {
+ IgfsEx igfsEx = uni.igfs();
+
+ if (igfsEx != null)
+ clear(igfsEx);
+
+ // Clear the filesystem.
+ uni.format();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ clear(igfs, igfsSecondary);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5757642/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 86c2449..c9b08d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -17,55 +17,32 @@
package org.apache.ignite.internal.processors.igfs;
-import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteFileSystem;
-import org.apache.ignite.Ignition;
import org.apache.ignite.cache.CacheMemoryMode;
import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.FileSystemConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
-import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper;
import org.apache.ignite.igfs.IgfsInputStream;
-import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
-import org.apache.ignite.igfs.IgfsIpcEndpointType;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsOutputStream;
import org.apache.ignite.igfs.IgfsParentNotDirectoryException;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.igfs.IgfsPathNotFoundException;
-import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
-import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
-import org.jetbrains.annotations.Nullable;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.lang.reflect.Field;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
@@ -84,142 +61,18 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheMemoryMode.ONHEAP_TIERED;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-import static org.apache.ignite.cache.CacheMode.REPLICATED;
-import static org.apache.ignite.igfs.IgfsMode.PRIMARY;
-import static org.apache.ignite.igfs.IgfsMode.PROXY;
-
/**
* Test fo regular igfs operations.
*/
@SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "ConstantConditions"})
-public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
- /** IGFS block size. */
- protected static final int IGFS_BLOCK_SIZE = 512 * 1024;
-
- /** Default block size (32Mb). */
- protected static final long BLOCK_SIZE = 32 * 1024 * 1024;
-
- /** Default repeat count. */
- protected static final int REPEAT_CNT = 5; // Diagnostic: up to 500; Regression: 5
-
- /** Concurrent operations count. */
- protected static final int OPS_CNT = 16;
-
- /** Renames count. */
- protected static final int RENAME_CNT = OPS_CNT;
-
- /** Deletes count. */
- protected static final int DELETE_CNT = OPS_CNT;
-
- /** Updates count. */
- protected static final int UPDATE_CNT = OPS_CNT;
-
- /** Mkdirs count. */
- protected static final int MKDIRS_CNT = OPS_CNT;
-
- /** Create count. */
- protected static final int CREATE_CNT = OPS_CNT;
-
- /** Time to wait until the caches get empty after format. */
- private static final long CACHE_EMPTY_TIMEOUT = 30_000L;
-
- /** Seed to generate random numbers. */
- protected static final long SEED = System.currentTimeMillis();
-
- /** Amount of blocks to prefetch. */
- protected static final int PREFETCH_BLOCKS = 1;
-
- /** Amount of sequential block reads before prefetch is triggered. */
- protected static final int SEQ_READS_BEFORE_PREFETCH = 2;
-
- /** Primary file system REST endpoint configuration map. */
- protected static final IgfsIpcEndpointConfiguration PRIMARY_REST_CFG;
-
- /** Secondary file system REST endpoint configuration map. */
- protected static final IgfsIpcEndpointConfiguration SECONDARY_REST_CFG;
-
- /** Directory. */
- protected static final IgfsPath DIR = new IgfsPath("/dir");
-
- /** Sub-directory. */
- protected static final IgfsPath SUBDIR = new IgfsPath(DIR, "subdir");
-
- /** Another sub-directory in the same directory. */
- protected static final IgfsPath SUBDIR2 = new IgfsPath(DIR, "subdir2");
-
- /** Sub-directory of the sub-directory. */
- protected static final IgfsPath SUBSUBDIR = new IgfsPath(SUBDIR, "subsubdir");
-
- /** File. */
- protected static final IgfsPath FILE = new IgfsPath(SUBDIR, "file");
-
- /** Another file in the same directory. */
- protected static final IgfsPath FILE2 = new IgfsPath(SUBDIR, "file2");
-
- /** Other directory. */
- protected static final IgfsPath DIR_NEW = new IgfsPath("/dirNew");
-
- /** Other subdirectory. */
- protected static final IgfsPath SUBDIR_NEW = new IgfsPath(DIR_NEW, "subdirNew");
-
- /** Other sub-directory of the sub-directory. */
- protected static final IgfsPath SUBSUBDIR_NEW = new IgfsPath(SUBDIR_NEW, "subsubdirNew");
-
- /** Other file. */
- protected static final IgfsPath FILE_NEW = new IgfsPath(SUBDIR_NEW, "fileNew");
-
- /** Default data chunk (128 bytes). */
- protected static final byte[] chunk = createChunk(128);
-
- /** Primary IGFS. */
- protected static IgfsImpl igfs;
-
- /** Secondary IGFS */
- protected static IgfsSecondaryFileSystem igfsSecondaryFileSystem;
-
- /** Secondary file system lower layer "backdoor" wrapped in UniversalFileSystemAdapter: */
- protected static IgfsSecondaryFileSystemTestAdapter igfsSecondary;
-
- /** IGFS mode. */
- protected final IgfsMode mode;
-
- /** Dual mode flag. */
- protected final boolean dual;
-
- /** Memory mode. */
- protected final CacheMemoryMode memoryMode;
-
- /** IP finder for primary topology. */
- protected final TcpDiscoveryVmIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** IP finder for secondary topology. */
- protected final TcpDiscoveryVmIpFinder secondaryIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Ignite nodes of cluster, excluding the secondary file system node, if any. */
- protected Ignite[] nodes;
-
- static {
- PRIMARY_REST_CFG = new IgfsIpcEndpointConfiguration();
-
- PRIMARY_REST_CFG.setType(IgfsIpcEndpointType.TCP);
- PRIMARY_REST_CFG.setPort(10500);
-
- SECONDARY_REST_CFG = new IgfsIpcEndpointConfiguration();
-
- SECONDARY_REST_CFG.setType(IgfsIpcEndpointType.TCP);
- SECONDARY_REST_CFG.setPort(11500);
- }
-
+public abstract class IgfsAbstractSelfTest extends IgfsAbstractBaseSelfTest {
/**
* Constructor.
*
* @param mode IGFS mode.
*/
protected IgfsAbstractSelfTest(IgfsMode mode) {
- this(mode, ONHEAP_TIERED);
+ super(mode);
}
/**
@@ -229,258 +82,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param memoryMode Memory mode.
*/
protected IgfsAbstractSelfTest(IgfsMode mode, CacheMemoryMode memoryMode) {
- assert mode != null && mode != PROXY;
-
- this.mode = mode;
- this.memoryMode = memoryMode;
-
- dual = mode != PRIMARY;
- }
-
- /**
- * @return Relaxed consistency flag.
- */
- protected boolean relaxedConsistency() {
- return false;
- }
-
- /**
- * @return Relaxed consistency flag.
- */
- protected boolean initializeDefaultPathModes() {
- return false;
- }
-
- /**
- * @return Client flag.
- */
- protected boolean client() {
- return false;
- }
-
- /**
- * @return Use optimized marshaller flag.
- */
- protected boolean useOptimizedMarshaller() {
- return false;
- }
-
- /**
- * @return Whether append is supported.
- */
- protected boolean appendSupported() {
- return true;
- }
-
- /**
- * @return Whether permissions are supported.
- */
- protected boolean permissionsSupported() {
- return true;
- }
-
- /**
- * @return Whether properties are supported.
- */
- protected boolean propertiesSupported() {
- return true;
- }
-
- /**
- * @return Whether times are supported.
- */
- protected boolean timesSupported() {
- return true;
- }
-
- /**
- * @return Amount of nodes to start.
- */
- protected int nodeCount() {
- return 1;
- }
-
- /**
- * Data chunk.
- *
- * @param len Length.
- * @return Data chunk.
- */
- static byte[] createChunk(int len) {
- byte[] chunk = new byte[len];
-
- for (int i = 0; i < chunk.length; i++)
- chunk[i] = (byte)i;
-
- return chunk;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- igfsSecondaryFileSystem = createSecondaryFileSystemStack();
-
- nodes = new Ignite[nodeCount()];
-
- for (int i = 0; i < nodes.length; i++) {
- String nodeName = i == 0 ? "ignite" : "ignite" + i;
-
- nodes[i] = startGridWithIgfs(nodeName, "igfs", mode, igfsSecondaryFileSystem, PRIMARY_REST_CFG,
- primaryIpFinder);
- }
-
- igfs = (IgfsImpl) nodes[0].fileSystem("igfs");
-
- if (client()) {
- // Start client.
- Ignition.setClientMode(true);
-
- try {
- Ignite ignite = startGridWithIgfs("ignite-client", "igfs", mode, igfsSecondaryFileSystem,
- PRIMARY_REST_CFG, primaryIpFinder);
-
- igfs = (IgfsImpl) ignite.fileSystem("igfs");
- }
- finally {
- Ignition.setClientMode(false);
- }
- }
- }
-
- /**
- * Creates secondary file system stack.
- *
- * @return The secondary file system.
- * @throws Exception On error.
- */
- protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
- Ignite igniteSecondary = startGridWithIgfs("ignite-secondary", "igfs-secondary", PRIMARY, null,
- SECONDARY_REST_CFG, secondaryIpFinder);
-
- IgfsEx secondaryIgfsImpl = (IgfsEx) igniteSecondary.fileSystem("igfs-secondary");
-
- igfsSecondary = new DefaultIgfsSecondaryFileSystemTestAdapter(secondaryIgfsImpl);
-
- return secondaryIgfsImpl.asSecondary();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- clear(igfs, igfsSecondary);
-
- assert igfs.listFiles(new IgfsPath("/")).isEmpty();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- G.stopAll(true);
- }
-
- /**
- * Start grid with IGFS.
- *
- * @param gridName Grid name.
- * @param igfsName IGFS name
- * @param mode IGFS mode.
- * @param secondaryFs Secondary file system (optional).
- * @param restCfg Rest configuration string (optional).
- * @param ipFinder IP finder.
- * @return Started grid instance.
- * @throws Exception If failed.
- */
- @SuppressWarnings("unchecked")
- protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg,
- TcpDiscoveryIpFinder ipFinder) throws Exception {
- FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
-
- igfsCfg.setDataCacheName("dataCache");
- igfsCfg.setMetaCacheName("metaCache");
- igfsCfg.setName(igfsName);
- igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
- igfsCfg.setDefaultMode(mode);
- igfsCfg.setIpcEndpointConfiguration(restCfg);
- igfsCfg.setSecondaryFileSystem(secondaryFs);
- igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
- igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
- igfsCfg.setRelaxedConsistency(relaxedConsistency());
-
- igfsCfg.setInitializeDefaultPathModes(initializeDefaultPathModes());
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("dataCache");
- dataCacheCfg.setNearConfiguration(null);
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setNearConfiguration(null);
- dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
- dataCacheCfg.setMemoryMode(memoryMode);
- dataCacheCfg.setOffHeapMaxMemory(0);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("metaCache");
- metaCacheCfg.setNearConfiguration(null);
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- if (useOptimizedMarshaller())
- cfg.setMarshaller(new OptimizedMarshaller());
-
- cfg.setGridName(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(ipFinder);
-
- prepareCacheConfigurations(dataCacheCfg, metaCacheCfg);
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setFileSystemConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- return G.start(cfg);
- }
-
- /**
- * Prepare cache configuration.
- *
- * @param dataCacheCfg Data cache configuration.
- * @param metaCacheCfg Meta cache configuration.
- */
- protected void prepareCacheConfigurations(CacheConfiguration dataCacheCfg, CacheConfiguration metaCacheCfg) {
- // Noop
- }
-
- /**
- * Execute provided task in a separate thread.
- *
- * @param task Task to execute.
- * @return Result.
- */
- protected static <T> IgniteInternalFuture<T> execute(final Callable<T> task) {
- final GridFutureAdapter<T> fut = new GridFutureAdapter<>();
-
- new Thread(new Runnable() {
- @Override public void run() {
- try {
- fut.onDone(task.call());
- }
- catch (Throwable e) {
- fut.onDone(e);
- }
- }
- }).start();
-
- return fut;
+ super(mode, memoryMode);
}
/**
@@ -1138,8 +740,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
}
/**
+ * Check root property update.
*
- * @throws Exception
+ * @throws Exception If failed.
*/
private void checkRootPropertyUpdate(String prop, String setVal, String expGetVal) throws Exception {
final IgfsPath rootPath = new IgfsPath("/");
@@ -2945,609 +2548,4 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
U.joinThreads(threads, null);
}
-
- /**
- * Create the given directories and files in the given IGFS.
- *
- * @param igfs IGFS.
- * @param dirs Directories.
- * @param files Files.
- * @throws Exception If failed.
- */
- @SuppressWarnings("EmptyTryBlock")
- public static void create(IgfsImpl igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files) throws Exception {
- if (dirs != null) {
- for (IgfsPath dir : dirs)
- igfs.mkdirs(dir);
- }
-
- if (files != null) {
- for (IgfsPath file : files) {
- try (OutputStream os = igfs.create(file, true)) {
- // No-op.
- }
-
- igfs.await(file);
- }
- }
- }
-
- /**
- * Creates specified files/directories
- *
- * @param uni The file system to operate on.
- * @param dirs The directories to create.
- * @param files The files to create.
- * @throws Exception On error.
- */
- @SuppressWarnings("EmptyTryBlock")
- public void create(IgfsSecondaryFileSystemTestAdapter uni, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
- throws Exception {
- if (dirs != null) {
- for (IgfsPath dir : dirs)
- uni.mkdirs(dir.toString());
- }
-
- if (files != null) {
- for (IgfsPath file : files)
- try (OutputStream ignore = uni.openOutputStream(file.toString(), false)) {
- // No-op
- }
- }
- }
-
- /**
- * Create the file in the given IGFS and write provided data chunks to it.
- *
- * @param igfs IGFS.
- * @param file File.
- * @param overwrite Overwrite flag.
- * @param chunks Data chunks.
- * @throws IOException In case of IO exception.
- */
- protected static void createFile(IgfsEx igfs, IgfsPath file, boolean overwrite, @Nullable byte[]... chunks)
- throws IOException {
- OutputStream os = null;
-
- try {
- os = igfs.create(file, overwrite);
-
- writeFileChunks(os, chunks);
- }
- finally {
- U.closeQuiet(os);
-
- awaitFileClose(igfs, file);
- }
- }
-
- /**
- * Create the file in the given IGFS and write provided data chunks to it.
- *
- * @param file File.
- * @param chunks Data chunks.
- * @throws IOException In case of IO exception.
- */
- protected static void createFile(IgfsSecondaryFileSystemTestAdapter uni, IgfsPath file, @Nullable byte[]... chunks)
- throws IOException {
- OutputStream os = null;
-
- try {
- os = uni.openOutputStream(file.toString(), false);
-
- writeFileChunks(os, chunks);
- }
- finally {
- U.closeQuiet(os);
-
- IgfsEx igfsEx = uni.igfs();
-
- if (igfsEx != null)
- awaitFileClose(igfsEx, file);
- }
- }
-
- /**
- * Create the file in the given IGFS and write provided data chunks to it.
- *
- * @param igfs IGFS.
- * @param file File.
- * @param overwrite Overwrite flag.
- * @param blockSize Block size.
- * @param chunks Data chunks.
- * @throws Exception If failed.
- */
- protected static void createFile(IgfsImpl igfs, IgfsPath file, boolean overwrite, long blockSize,
- @Nullable byte[]... chunks) throws Exception {
- IgfsOutputStream os = null;
-
- try {
- os = igfs.create(file, 256, overwrite, null, 0, blockSize, null);
-
- writeFileChunks(os, chunks);
- }
- finally {
- U.closeQuiet(os);
-
- awaitFileClose(igfs, file);
- }
- }
-
- /**
- * Append to the file in the given IGFS provided data chunks.
- *
- * @param igfs IGFS.
- * @param file File.
- * @param chunks Data chunks.
- * @throws Exception If failed.
- */
- protected static void appendFile(IgfsImpl igfs, IgfsPath file, @Nullable byte[]... chunks)
- throws Exception {
- IgfsOutputStream os = null;
-
- try {
- os = igfs.append(file, false);
-
- writeFileChunks(os, chunks);
- }
- finally {
- U.closeQuiet(os);
-
- awaitFileClose(igfs, file);
- }
- }
-
- /**
- * Write provided data chunks to the file output stream.
- *
- * @param os Output stream.
- * @param chunks Data chunks.
- * @throws IOException If failed.
- */
- protected static void writeFileChunks(OutputStream os, @Nullable byte[]... chunks) throws IOException {
- if (chunks != null && chunks.length > 0) {
- for (byte[] chunk : chunks)
- os.write(chunk);
- }
- }
-
- /**
- * Await for previously opened output stream to close. This is achieved by requesting dummy update on the file.
- *
- * @param igfs IGFS.
- * @param file File.
- */
- public static void awaitFileClose(IgfsSecondaryFileSystem igfs, IgfsPath file) {
- try {
- igfs.update(file, Collections.singletonMap("prop", "val"));
- }
- catch (IgniteException ignore) {
- // No-op.
- }
- }
-
- /**
- * Await for previously opened output stream to close.
- *
- * @param igfs IGFS.
- * @param file File.
- */
- public static void awaitFileClose(@Nullable IgfsEx igfs, IgfsPath file) {
- igfs.await(file);
- }
-
- /**
- * Ensure that the given paths exist in the given IGFSs.
- *
- * @param igfs First IGFS.
- * @param igfsSecondary Second IGFS.
- * @param paths Paths.
- * @throws Exception If failed.
- */
- protected void checkExist(IgfsImpl igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary, IgfsPath... paths)
- throws Exception {
- checkExist(igfs, paths);
-
- if (dual)
- checkExist(igfsSecondary, paths);
- }
-
- /**
- * Ensure that the given paths exist in the given IGFS.
- *
- * @param igfs IGFS.
- * @param paths Paths.
- * @throws IgniteCheckedException If failed.
- */
- protected static void checkExist(IgfsImpl igfs, IgfsPath... paths) throws IgniteCheckedException {
- for (IgfsPath path : paths)
- assert igfs.exists(path) : "Path doesn't exist [igfs=" + igfs.name() + ", path=" + path + ']';
- }
-
- /**
- * Ensure that the given paths exist in the given IGFS.
- *
- * @param uni filesystem.
- * @param paths Paths.
- * @throws IgniteCheckedException If failed.
- */
- protected void checkExist(IgfsSecondaryFileSystemTestAdapter uni, IgfsPath... paths) throws IgniteCheckedException {
- IgfsEx ex = uni.igfs();
-
- for (IgfsPath path : paths) {
- if (ex != null)
- assert ex.context().meta().fileId(path) != null : "Path doesn't exist [igfs=" + ex.name() +
- ", path=" + path + ']';
-
- try {
- assert uni.exists(path.toString()) : "Path doesn't exist [igfs=" + uni.name() + ", path=" + path + ']';
- }
- catch (IOException ioe) {
- throw new IgniteCheckedException(ioe);
- }
- }
- }
-
- /**
- * Ensure that the given paths don't exist in the given IGFSs.
- *
- * @param igfs First IGFS.
- * @param igfsSecondary Second IGFS.
- * @param paths Paths.
- * @throws Exception If failed.
- */
- protected void checkNotExist(IgfsImpl igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary, IgfsPath... paths)
- throws Exception {
- checkNotExist(igfs, paths);
-
- if (dual)
- checkNotExist(igfsSecondary, paths);
- }
-
- /**
- * Ensure that the given paths don't exist in the given IGFS.
- *
- * @param igfs IGFS.
- * @param paths Paths.
- * @throws Exception If failed.
- */
- protected void checkNotExist(IgfsImpl igfs, IgfsPath... paths) throws Exception {
- for (IgfsPath path : paths)
- assert !igfs.exists(path) : "Path exists [igfs=" + igfs.name() + ", path=" + path + ']';
- }
-
- /**
- * Ensure that the given paths don't exist in the given IGFS.
- *
- * @param uni secondary FS.
- * @param paths Paths.
- * @throws Exception If failed.
- */
- protected void checkNotExist(IgfsSecondaryFileSystemTestAdapter uni, IgfsPath... paths) throws Exception {
- IgfsEx ex = uni.igfs();
-
- for (IgfsPath path : paths) {
- if (ex != null)
- assert !ex.exists(path) : "Path exists [igfs=" + ex.name() + ", path=" + path + ']';
-
- assert !uni.exists(path.toString()) : "Path exists [igfs=" + uni.name() + ", path=" + path + ']';
- }
- }
-
- /**
- * Ensure that the given file exists in the given IGFSs and that it has exactly the same content as provided in the
- * "data" parameter.
- *
- * @param igfs First IGFS.
- * @param igfsSecondary Second IGFS.
- * @param file File.
- * @param chunks Expected data.
- * @throws Exception If failed.
- */
- protected void checkFile(@Nullable IgfsImpl igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary, IgfsPath file,
- @Nullable byte[]... chunks) throws Exception {
- if (igfs != null) {
- checkExist(igfs, file);
- checkFileContent(igfs, file, chunks);
- }
-
- if (dual) {
- checkExist(igfsSecondary, file);
- checkFileContent(igfsSecondary, file.toString(), chunks);
- }
- }
-
- /**
- * Ensure that the given file has exactly the same content as provided in the "data" parameter.
- *
- * @param igfs IGFS.
- * @param file File.
- * @param chunks Expected data.
- * @throws IOException In case of IO exception.
- * @throws IgniteCheckedException In case of Grid exception.
- */
- protected static void checkFileContent(IgfsImpl igfs, IgfsPath file, @Nullable byte[]... chunks)
- throws IOException, IgniteCheckedException {
- if (chunks != null && chunks.length > 0) {
- IgfsInputStream is = null;
-
- try {
- is = igfs.open(file);
-
- int chunkIdx = 0;
- int pos = 0;
-
- for (byte[] chunk : chunks) {
- byte[] buf = new byte[chunk.length];
-
- is.readFully(pos, buf);
-
- assert Arrays.equals(chunk, buf) : "Bad chunk [igfs=" + igfs.name() + ", chunkIdx=" + chunkIdx +
- ", expected=" + Arrays.toString(chunk) + ", actual=" + Arrays.toString(buf) + ']';
-
- chunkIdx++;
- pos += chunk.length;
- }
-
- is.close();
- }
- finally {
- U.closeQuiet(is);
- }
- }
- }
-
- /**
- * Ensure that the given file has exactly the same content as provided in the "data" parameter.
- *
- * @param uni FS.
- * @param path File.
- * @param chunks Expected data.
- * @throws IOException In case of IO exception.
- * @throws IgniteCheckedException In case of Grid exception.
- */
- protected void checkFileContent(IgfsSecondaryFileSystemTestAdapter uni, String path, @Nullable byte[]... chunks)
- throws IOException, IgniteCheckedException {
- if (chunks != null && chunks.length > 0) {
- InputStream is = null;
-
- try {
- is = uni.openInputStream(path);
-
- int chunkIdx = 0;
-
- int read;
- for (byte[] chunk: chunks) {
- byte[] buf = new byte[chunk.length];
-
- read = 0;
-
- while (true) {
- int r = is.read(buf, read, buf.length - read);
-
- read += r;
-
- if (read == buf.length || r <= 0)
- break;
- }
-
- assert read == chunk.length : "Chunk #" + chunkIdx + " was not read fully:" +
- " read=" + read + ", expected=" + chunk.length;
- assert Arrays.equals(chunk, buf) : "Bad chunk [igfs=" + uni.name() + ", chunkIdx=" + chunkIdx +
- ", expected=" + Arrays.toString(chunk) + ", actual=" + Arrays.toString(buf) + ']';
-
- chunkIdx++;
- }
-
- is.close();
- }
- finally {
- U.closeQuiet(is);
- }
- }
- }
-
- /**
- * Create map with properties.
- *
- * @param username User name.
- * @param grpName Group name.
- * @param perm Permission.
- * @return Map with properties.
- */
- protected Map<String, String> properties(@Nullable String username, @Nullable String grpName,
- @Nullable String perm) {
- Map<String, String> props = new HashMap<>();
-
- if (username != null)
- props.put(IgfsUtils.PROP_USER_NAME, username);
-
- if (grpName != null)
- props.put(IgfsUtils.PROP_GROUP_NAME, grpName);
-
- if (perm != null)
- props.put(IgfsUtils.PROP_PERMISSION, perm);
-
- return props;
- }
-
- /**
- * Convenient method to group paths.
- *
- * @param paths Paths to group.
- * @return Paths as array.
- */
- protected static IgfsPath[] paths(IgfsPath... paths) {
- return paths;
- }
-
- /**
- * Safely clear IGFSs.
- *
- * @param igfs First IGFS.
- * @param igfsSecondary Second IGFS.
- * @throws Exception If failed.
- */
- protected void clear(IgniteFileSystem igfs, IgfsSecondaryFileSystemTestAdapter igfsSecondary) throws Exception {
- clear(igfs);
-
- if (dual)
- clear(igfsSecondary);
- }
-
- /**
- * Gets the data cache instance for this IGFS instance.
- *
- * @param igfs The IGFS unstance.
- * @return The data cache.
- */
- protected static GridCacheAdapter<IgfsBlockKey, byte[]> getDataCache(IgniteFileSystem igfs) {
- String dataCacheName = igfs.configuration().getDataCacheName();
-
- IgniteEx igniteEx = ((IgfsEx)igfs).context().kernalContext().grid();
-
- return ((IgniteKernal)igniteEx).internalCache(dataCacheName);
- }
-
- /**
- * Gets meta cache.
- *
- * @param igfs The IGFS instance.
- * @return The data cache.
- */
- protected static GridCacheAdapter<IgniteUuid, IgfsEntryInfo> getMetaCache(IgniteFileSystem igfs) {
- String dataCacheName = igfs.configuration().getMetaCacheName();
-
- IgniteEx igniteEx = ((IgfsEx)igfs).context().kernalContext().grid();
-
- return ((IgniteKernal)igniteEx).internalCache(dataCacheName);
- }
-
- /**
- * Clear particular IGFS.
- *
- * @param igfs IGFS.
- * @throws Exception If failed.
- */
- @SuppressWarnings("unchecked")
- public static void clear(IgniteFileSystem igfs) throws Exception {
- Field workerMapFld = IgfsImpl.class.getDeclaredField("workerMap");
-
- workerMapFld.setAccessible(true);
-
- // Wait for all workers to finish.
- Map<IgfsPath, IgfsFileWorkerBatch> workerMap = (Map<IgfsPath, IgfsFileWorkerBatch>)workerMapFld.get(igfs);
-
- for (Map.Entry<IgfsPath, IgfsFileWorkerBatch> entry : workerMap.entrySet()) {
- entry.getValue().cancel();
-
- try {
- entry.getValue().await();
- }
- catch (IgniteCheckedException e) {
- if (!(e instanceof IgfsFileWorkerBatchCancelledException))
- throw e;
- }
- }
-
- // Clear igfs.
- igfs.format();
-
- int prevDifferentSize = Integer.MAX_VALUE; // Previous different size.
- int constCnt = 0, totalCnt = 0;
- final int constThreshold = 20;
- final long sleepPeriod = 500L;
- final long totalThreshold = CACHE_EMPTY_TIMEOUT / sleepPeriod;
-
- while (true) {
- int metaSize = 0;
-
- for (IgniteUuid metaId : getMetaCache(igfs).keySet()) {
- if (!IgfsUtils.isRootOrTrashId(metaId))
- metaSize++;
- }
-
- int dataSize = getDataCache(igfs).size();
-
- int size = metaSize + dataSize;
-
- if (size <= 2)
- return; // Caches are cleared, we're done. (2 because ROOT & TRASH always exist).
-
- X.println("Sum size: " + size);
-
- if (size > prevDifferentSize) {
- X.println("Summary cache size has grown unexpectedly: size=" + size + ", prevSize=" + prevDifferentSize);
-
- break;
- }
-
- if (totalCnt > totalThreshold) {
- X.println("Timeout exceeded.");
-
- break;
- }
-
- if (size == prevDifferentSize) {
- constCnt++;
-
- if (constCnt == constThreshold) {
- X.println("Summary cache size stays unchanged for too long: size=" + size);
-
- break;
- }
- } else {
- constCnt = 0;
-
- prevDifferentSize = size; // renew;
- }
-
- Thread.sleep(sleepPeriod);
-
- totalCnt++;
- }
-
- dumpCache("MetaCache" , getMetaCache(igfs));
-
- dumpCache("DataCache" , getDataCache(igfs));
-
- fail("Caches are not empty.");
- }
-
- /**
- * Dumps given cache for diagnostic purposes.
- *
- * @param cacheName Name.
- * @param cache The cache.
- */
- private static void dumpCache(String cacheName, GridCacheAdapter<?,?> cache) {
- X.println("=============================== " + cacheName + " cache dump: ");
-
- Iterable<? extends GridCacheEntryEx> entries = cache.entries();
-
- for (GridCacheEntryEx e: entries)
- X.println("Lost " + cacheName + " entry = " + e);
- }
-
- /**
- * Clear particular {@link IgfsSecondaryFileSystemTestAdapter}.
- *
- * @param uni IGFS.
- * @throws Exception If failed.
- */
- @SuppressWarnings("unchecked")
- public static void clear(IgfsSecondaryFileSystemTestAdapter uni) throws Exception {
- IgfsEx igfsEx = uni.igfs();
-
- if (igfsEx != null)
- clear(igfsEx);
-
- // Clear the filesystem.
- uni.format();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- clear(igfs, igfsSecondary);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5757642/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsLocalSecondaryFileSystemDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsLocalSecondaryFileSystemDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsLocalSecondaryFileSystemDualAbstractSelfTest.java
index c2f5633..1d1ce8d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsLocalSecondaryFileSystemDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsLocalSecondaryFileSystemDualAbstractSelfTest.java
@@ -17,12 +17,21 @@
package org.apache.ignite.internal.processors.igfs;
+import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsMode;
+import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.local.LocalIgfsSecondaryFileSystem;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.Collection;
/**
* Abstract test for Hadoop 1.0 file system stack.
@@ -32,6 +41,24 @@ public abstract class IgfsLocalSecondaryFileSystemDualAbstractSelfTest extends I
private static final String FS_WORK_DIR = U.getIgniteHome() + File.separatorChar + "work"
+ File.separatorChar + "fs";
+ /** */
+ private static final String FS_EXT_DIR = U.getIgniteHome() + File.separatorChar + "work"
+ + File.separatorChar + "ext";
+
+ /** */
+ private final File dirLinkDest = new File(FS_EXT_DIR + File.separatorChar + "extdir");
+
+ /** */
+ private final File fileLinkDest =
+ new File(FS_EXT_DIR + File.separatorChar + "extdir" + File.separatorChar + "filedest");
+
+ /** */
+ private final File dirLinkSrc = new File(FS_WORK_DIR + File.separatorChar + "dir");
+
+ /** */
+ private final File fileLinkSrc = new File(FS_WORK_DIR + File.separatorChar + "file");
+
+
/** Constructor.
* @param mode IGFS mode.
*/
@@ -39,6 +66,19 @@ public abstract class IgfsLocalSecondaryFileSystemDualAbstractSelfTest extends I
super(mode);
}
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ final File extDir = new File(FS_EXT_DIR);
+
+ if (!extDir.exists())
+ assert extDir.mkdirs();
+ else
+ cleanDirectory(extDir);
+ }
+
+
/**
* Creates secondary filesystems.
* @return IgfsSecondaryFileSystem
@@ -73,4 +113,107 @@ public abstract class IgfsLocalSecondaryFileSystemDualAbstractSelfTest extends I
@Override protected boolean timesSupported() {
return false;
}
+
+ /**
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public void testListPathForSymlink() throws Exception {
+ if (U.isWindows())
+ return;
+
+ createSymlinks();
+
+ assertTrue(igfs.info(DIR).isDirectory());
+
+ Collection<IgfsPath> pathes = igfs.listPaths(DIR);
+ Collection<IgfsFile> files = igfs.listFiles(DIR);
+
+ assertEquals(1, pathes.size());
+ assertEquals(1, files.size());
+
+ assertEquals("filedest", F.first(pathes).name());
+ assertEquals("filedest", F.first(files).path().name());
+ }
+
+ /**
+ *
+ * @throws Exception If failed.
+ */
+ public void testDeleteSymlinkDir() throws Exception {
+ if (U.isWindows())
+ return;
+
+ createSymlinks();
+
+ // Only symlink must be deleted. Destination content must be exist.
+ igfs.delete(DIR, true);
+
+ assertTrue(fileLinkDest.exists());
+ }
+
+ /**
+ *
+ * @throws Exception If failed.
+ */
+ public void testSymlinkToFile() throws Exception {
+ if (U.isWindows())
+ return;
+
+ createSymlinks();
+
+ checkFileContent(igfs, new IgfsPath("/file"), chunk);
+ }
+
+ /**
+ *
+ * @throws Exception If failed.
+ */
+ private void createSymlinks() throws Exception {
+ assert dirLinkDest.mkdir();
+
+ createFile(fileLinkDest, true, chunk);
+
+ Files.createSymbolicLink(dirLinkSrc.toPath(), dirLinkDest.toPath());
+ Files.createSymbolicLink(fileLinkSrc.toPath(), fileLinkDest.toPath());
+ }
+
+ /**
+ * @param dir Directory to clean.
+ */
+ private static void cleanDirectory(File dir){
+ File[] entries = dir.listFiles();
+
+ if (entries != null) {
+ for (File entry : entries) {
+ if (entry.isDirectory()) {
+ cleanDirectory(entry);
+
+ assert entry.delete();
+ }
+ else
+ assert entry.delete();
+ }
+ }
+ }
+
+ /**
+ * @param f File object.
+ * @param overwrite Overwrite flag.
+ * @param chunks File content.
+ * @throws IOException If failed.
+ */
+ private static void createFile(File f, boolean overwrite, @Nullable byte[]... chunks) throws IOException {
+ OutputStream os = null;
+
+ try {
+ os = new FileOutputStream(f, overwrite);
+
+ writeFileChunks(os, chunks);
+ }
+ finally {
+ U.closeQuiet(os);
+ }
+ }
}
\ No newline at end of file