You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2015/03/04 16:35:10 UTC
[08/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
new file mode 100644
index 0000000..3b4c5c2
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheDistributionMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * Tests for IGFS file system handshake.
+ */
+public class IgniteHadoopFileSystemHandshakeSelfTest extends IgfsCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Grid name. */
+ private static final String GRID_NAME = "grid";
+
+ /** IGFS name. */
+ private static final String IGFS_NAME = "igfs";
+
+ /** IGFS path. */
+ private static final IgfsPath PATH = new IgfsPath("/path");
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /**
+ * Tests for Grid and IGFS having normal names.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshake() throws Exception {
+ startUp(false, false);
+
+ checkValid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(IGFS_NAME + "@");
+ checkInvalid(IGFS_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(":" + GRID_NAME + "@");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid("");
+ checkInvalid("127.0.0.1");
+ checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Tests for Grid having {@code null} name and IGFS having normal name.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshakeDefaultGrid() throws Exception {
+ startUp(true, false);
+
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkValid(IGFS_NAME + "@");
+ checkValid(IGFS_NAME + "@127.0.0.1");
+ checkValid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(":" + GRID_NAME + "@");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid("");
+ checkInvalid("127.0.0.1");
+ checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Tests for Grid having normal name and IGFS having {@code null} name.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshakeDefaultIgfs() throws Exception {
+ startUp(false, true);
+
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(IGFS_NAME + "@");
+ checkInvalid(IGFS_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkValid(":" + GRID_NAME + "@");
+ checkValid(":" + GRID_NAME + "@127.0.0.1");
+ checkValid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid("");
+ checkInvalid("127.0.0.1");
+ checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Tests for Grid having {@code null} name and IGFS having {@code null} name.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshakeDefaultGridDefaultIgfs() throws Exception {
+ startUp(true, true);
+
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(IGFS_NAME + "@");
+ checkInvalid(IGFS_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(":" + GRID_NAME + "@");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkValid("");
+ checkValid("127.0.0.1");
+ checkValid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Perform startup.
+ *
+ * @param dfltGridName Default Grid name.
+ * @param dfltIgfsName Default IGFS name.
+ * @throws Exception If failed.
+ */
+ private void startUp(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
+ Ignite ignite = G.start(gridConfiguration(dfltGridName, dfltIgfsName));
+
+ IgniteFileSystem igfs = ignite.fileSystem(dfltIgfsName ? null : IGFS_NAME);
+
+ igfs.mkdirs(PATH);
+ }
+
+ /**
+ * Create Grid configuration.
+ *
+ * @param dfltGridName Default Grid name.
+ * @param dfltIgfsName Default IGFS name.
+ * @return Grid configuration.
+ * @throws Exception If failed.
+ */
+ private IgniteConfiguration gridConfiguration(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(dfltGridName ? null : GRID_NAME);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ cfg.setCommunicationSpi(commSpi);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("partitioned");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ cfg.setCacheConfiguration(metaCacheCfg, dataCacheCfg);
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName(dfltIgfsName ? null : IGFS_NAME);
+ igfsCfg.setPrefetchBlocks(1);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", String.valueOf(DFLT_IPC_PORT));
+ }});
+
+ igfsCfg.setManagementPort(-1);
+ igfsCfg.setBlockSize(512 * 1024);
+
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ return cfg;
+ }
+
+ /**
+ * Check valid file system endpoint.
+ *
+ * @param authority Authority.
+ * @throws Exception If failed.
+ */
+ private void checkValid(String authority) throws Exception {
+ FileSystem fs = fileSystem(authority);
+
+ assert fs.exists(new Path(PATH.toString()));
+ }
+
+ /**
+ * Check invalid file system endpoint.
+ *
+ * @param authority Authority.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ private void checkInvalid(final String authority) throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fileSystem(authority);
+
+ return null;
+ }
+ }, IOException.class, null);
+ }
+
+ /**
+ *
+ *
+ * @param authority Authority.
+ * @return File system.
+ * @throws Exception If failed.
+ */
+ private static FileSystem fileSystem(String authority) throws Exception {
+ return FileSystem.get(new URI("igfs://" + authority + "/"), configuration(authority));
+ }
+
+ /**
+ * Create configuration for test.
+ *
+ * @param authority Authority.
+ * @return Configuration.
+ */
+ private static Configuration configuration(String authority) {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://" + authority + "/");
+ cfg.set("fs.igfs.impl", org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
+ cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java
new file mode 100644
index 0000000..135a488
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.ipc.shmem.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+
+import java.lang.reflect.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.events.EventType.*;
+
+/**
+ * IPC cache test.
+ */
+public class IgniteHadoopFileSystemIpcCacheSelfTest extends IgfsCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Path to test hadoop configuration. */
+ private static final String HADOOP_FS_CFG = "modules/core/src/test/config/hadoop/core-site.xml";
+
+ /** Group size. */
+ public static final int GRP_SIZE = 128;
+
+ /** Started grid counter. */
+ private static int cnt;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setManagementPort(FileSystemConfiguration.DFLT_MGMT_PORT + cnt);
+
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "shmem");
+ put("port", String.valueOf(IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt));
+ }});
+
+ igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
+
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setCacheConfiguration(cacheConfiguration());
+
+ cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ cfg.setCommunicationSpi(commSpi);
+
+ cnt++;
+
+ return cfg;
+ }
+
+ /**
+ * Gets cache configuration.
+ *
+ * @return Cache configuration.
+ */
+ private CacheConfiguration[] cacheConfiguration() {
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrids(4);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+ }
+
+ /**
+ * Test how IPC cache map works.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public void testIpcCache() throws Exception {
+ Field cacheField = HadoopIgfsIpcIo.class.getDeclaredField("ipcCache");
+
+ cacheField.setAccessible(true);
+
+ Field activeCntField = HadoopIgfsIpcIo.class.getDeclaredField("activeCnt");
+
+ activeCntField.setAccessible(true);
+
+ Map<String, HadoopIgfsIpcIo> cache = (Map<String, HadoopIgfsIpcIo>)cacheField.get(null);
+
+ String name = "igfs:" + getTestGridName(0) + "@";
+
+ Configuration cfg = new Configuration();
+
+ cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG));
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, name), true);
+
+ // Ensure that existing IO is reused.
+ FileSystem fs1 = FileSystem.get(new URI("igfs://" + name + "/"), cfg);
+
+ assertEquals(1, cache.size());
+
+ HadoopIgfsIpcIo io = null;
+
+ System.out.println("CACHE: " + cache);
+
+ for (String key : cache.keySet()) {
+ if (key.contains("10500")) {
+ io = cache.get(key);
+
+ break;
+ }
+ }
+
+ assert io != null;
+
+ assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get());
+
+ // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
+ FileSystem fs2 = FileSystem.get(new URI("igfs://" + name + "/abc"), cfg);
+
+ assertEquals(1, cache.size());
+ assertEquals(2, ((AtomicInteger)activeCntField.get(io)).get());
+
+ fs2.close();
+
+ assertEquals(1, cache.size());
+ assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get());
+
+ Field stopField = HadoopIgfsIpcIo.class.getDeclaredField("stopping");
+
+ stopField.setAccessible(true);
+
+ assert !(Boolean)stopField.get(io);
+
+ // Ensure that IO is stopped when nobody else is need it.
+ fs1.close();
+
+ assert cache.isEmpty();
+
+ assert (Boolean)stopField.get(io);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java
new file mode 100644
index 0000000..8d43e08
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.igfs.common.IgfsLogger.*;
+
+/**
+ * Grid IGFS client logger test.
+ */
+public class IgniteHadoopFileSystemLoggerSelfTest extends IgfsCommonAbstractTest {
+ /** Path string. */
+ private static final String PATH_STR = "/dir1/dir2/file;test";
+
+ /** Path string with escaped semicolons. */
+ private static final String PATH_STR_ESCAPED = PATH_STR.replace(';', '~');
+
+ /** Path. */
+ private static final IgfsPath PATH = new IgfsPath(PATH_STR);
+
+ /** IGFS name. */
+ private static final String IGFS_NAME = "igfs";
+
+ /** Log file path. */
+ private static final String LOG_DIR = U.getIgniteHome();
+
+ /** Endpoint address. */
+ private static final String ENDPOINT = "localhost:10500";
+
+ /** Log file name. */
+ private static final String LOG_FILE = LOG_DIR + File.separator + "igfs-log-" + IGFS_NAME + "-" + U.jvmPid() +
+ ".csv";
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ removeLogs();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ removeLogs();
+ }
+
+ /**
+ * Remove existing logs.
+ *
+ * @throws Exception If failed.
+ */
+ private void removeLogs() throws Exception {
+ File dir = new File(LOG_DIR);
+
+ File[] logs = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.startsWith("igfs-log-");
+ }
+ });
+
+ for (File log : logs)
+ log.delete();
+ }
+
+ /**
+ * Ensure correct static loggers creation/removal as well as file creation.
+ *
+ * @throws Exception If failed.
+ */
+ public void testCreateDelete() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ IgfsLogger sameLog0 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ // Loggers for the same endpoint must be the same object.
+ assert log == sameLog0;
+
+ IgfsLogger otherLog = IgfsLogger.logger("other" + ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ // Logger for another endpoint must be different.
+ assert log != otherLog;
+
+ otherLog.close();
+
+ log.logDelete(PATH, PRIMARY, false);
+
+ log.close();
+
+ File logFile = new File(LOG_FILE);
+
+ // When there are multiple loggers, closing one must not force flushing.
+ assert !logFile.exists();
+
+ IgfsLogger sameLog1 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ assert sameLog0 == sameLog1;
+
+ sameLog0.close();
+
+ assert !logFile.exists();
+
+ sameLog1.close();
+
+ // When we cloe the last logger, it must flush data to disk.
+ assert logFile.exists();
+
+ logFile.delete();
+
+ IgfsLogger sameLog2 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ // This time we expect new logger instance to be created.
+ assert sameLog0 != sameLog2;
+
+ sameLog2.close();
+
+ // As we do not add any records to the logger, we do not expect flushing.
+ assert !logFile.exists();
+ }
+
+ /**
+ * Test read operations logging.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLogRead() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ log.logOpen(1, PATH, PRIMARY, 2, 3L);
+ log.logRandomRead(1, 4L, 5);
+ log.logSeek(1, 6L);
+ log.logSkip(1, 7L);
+ log.logMark(1, 8L);
+ log.logReset(1);
+ log.logCloseIn(1, 9L, 10L, 11);
+
+ log.close();
+
+ checkLog(
+ new SB().a(U.jvmPid() + d() + TYPE_OPEN_IN + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() + 2 +
+ d() + 3 + d(14)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_RANDOM_READ + d(3) + 1 + d(7) + 4 + d() + 5 + d(8)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_SEEK + d(3) + 1 + d(7) + 6 + d(9)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_SKIP + d(3) + 1 + d(9) + 7 + d(7)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_MARK + d(3) + 1 + d(10) + 8 + d(6)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_RESET + d(3) + 1 + d(16)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_CLOSE_IN + d(3) + 1 + d(11) + 9 + d() + 10 + d() + 11 + d(3)).toString()
+ );
+ }
+
+ /**
+ * Test write operations logging.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLogWrite() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ log.logCreate(1, PATH, PRIMARY, true, 2, new Integer(3).shortValue(), 4L);
+ log.logAppend(2, PATH, PRIMARY, 8);
+ log.logCloseOut(2, 9L, 10L, 11);
+
+ log.close();
+
+ checkLog(
+ new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() +
+ 2 + d(2) + 0 + d() + 1 + d() + 3 + d() + 4 + d(10)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 2 + d() +
+ 8 + d(2) + 1 + d(13)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_CLOSE_OUT + d(3) + 2 + d(11) + 9 + d() + 10 + d() + 11 + d(3))
+ .toString()
+ );
+ }
+
+ /**
+ * Test miscellaneous operations logging.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("TooBroadScope")
+ public void testLogMisc() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ String newFile = "/dir3/file.test";
+ String file1 = "/dir3/file1.test";
+ String file2 = "/dir3/file1.test";
+
+ log.logMakeDirectory(PATH, PRIMARY);
+ log.logRename(PATH, PRIMARY, new IgfsPath(newFile));
+ log.logListDirectory(PATH, PRIMARY, new String[] { file1, file2 });
+ log.logDelete(PATH, PRIMARY, false);
+
+ log.close();
+
+ checkLog(
+ new SB().a(U.jvmPid() + d() + TYPE_DIR_MAKE + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_RENAME + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(15) + newFile +
+ d(2)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_DIR_LIST + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17) + file1 +
+ DELIM_FIELD_VAL + file2).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_DELETE + d(1) + PATH_STR_ESCAPED + d() + PRIMARY + d(16) + 0 +
+ d()).toString()
+ );
+ }
+
+ /**
+ * Create IGFS file with the given path.
+ *
+ * @param path File path.
+ * @return IGFS file instance.
+ */
+ private IgfsFile file(String path) {
+ return new IgfsFileImpl(new IgfsPath(path), new IgfsFileInfo(), 64 * 1024 * 1024);
+ }
+
+ /**
+ * Ensure that log file has only the following lines.
+ *
+ * @param lines Expected lines.
+ */
+ private void checkLog(String... lines) throws Exception {
+ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(LOG_FILE)));
+
+ List<String> logLines = new ArrayList<>(lines.length);
+
+ String nextLogLine;
+
+ while ((nextLogLine = br.readLine()) != null)
+ logLines.add(nextLogLine);
+
+ U.closeQuiet(br);
+
+ assertEquals(lines.length + 1, logLines.size());
+
+ assertEquals(logLines.get(0), HDR);
+
+ for (int i = 0; i < lines.length; i++) {
+ String logLine = logLines.get(i + 1);
+
+ logLine = logLine.substring(logLine.indexOf(DELIM_FIELD, logLine.indexOf(DELIM_FIELD) + 1) + 1);
+
+ assertEquals(lines[i], logLine);
+ }
+ }
+
+ /**
+ * Return single field delimiter.
+ *
+ * @return Single field delimiter.
+ */
+ private String d() {
+ return d(1);
+ }
+
+ /**
+ * Return a bunch of field delimiters.
+ *
+ * @param cnt Amount of field delimiters.
+ * @return Field delimiters.
+ */
+ private String d(int cnt) {
+ SB buf = new SB();
+
+ for (int i = 0; i < cnt; i++)
+ buf.a(DELIM_FIELD);
+
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java
new file mode 100644
index 0000000..1f6a204
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+
+import java.lang.reflect.*;
+import java.net.*;
+import java.nio.file.*;
+import java.util.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
+
+/**
+ * Ensures that sampling is really turned on/off.
+ */
+public class IgniteHadoopFileSystemLoggerStateSelfTest extends IgfsCommonAbstractTest {
+ /** IGFS. */
+ private IgfsEx igfs;
+
+ /** File system. */
+ private FileSystem fs;
+
+ /** Whether logging is enabled in FS configuration. */
+ private boolean logging;
+
+ /** whether sampling is enabled. */
+ private Boolean sampling;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ U.closeQuiet(fs);
+
+ igfs = null;
+ fs = null;
+
+ G.stopAll(true);
+
+ logging = false;
+ sampling = null;
+ }
+
+ /**
+ * Startup the grid and instantiate the file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startUp() throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "10500");
+ }});
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("igfs-grid");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ Ignite g = G.start(cfg);
+
+ igfs = (IgfsEx)g.fileSystem("igfs");
+
+ igfs.globalSampling(sampling);
+
+ fs = fileSystem();
+ }
+
+ /**
+ * When logging is disabled and sampling is not set no-op logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingDisabledSamplingNotSet() throws Exception {
+ startUp();
+
+ assert !logEnabled();
+ }
+
+ /**
+ * When logging is enabled and sampling is not set file logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingEnabledSamplingNotSet() throws Exception {
+ logging = true;
+
+ startUp();
+
+ assert logEnabled();
+ }
+
+ /**
+ * When logging is disabled and sampling is disabled no-op logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingDisabledSamplingDisabled() throws Exception {
+ sampling = false;
+
+ startUp();
+
+ assert !logEnabled();
+ }
+
+ /**
+ * When logging is enabled and sampling is disabled no-op logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingEnabledSamplingDisabled() throws Exception {
+ logging = true;
+ sampling = false;
+
+ startUp();
+
+ assert !logEnabled();
+ }
+
+ /**
+ * When logging is disabled and sampling is enabled file logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingDisabledSamplingEnabled() throws Exception {
+ sampling = true;
+
+ startUp();
+
+ assert logEnabled();
+ }
+
+ /**
+ * When logging is enabled and sampling is enabled file logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingEnabledSamplingEnabled() throws Exception {
+ logging = true;
+ sampling = true;
+
+ startUp();
+
+ assert logEnabled();
+ }
+
+ /**
+ * Ensure sampling change through API causes changes in logging on subsequent client connections.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSamplingChange() throws Exception {
+ // Start with sampling not set.
+ startUp();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // "Not set" => true transition.
+ igfs.globalSampling(true);
+
+ fs = fileSystem();
+
+ assert logEnabled();
+
+ fs.close();
+
+ // True => "not set" transition.
+ igfs.globalSampling(null);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ // "Not-set" => false transition.
+ igfs.globalSampling(false);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // False => "not=set" transition.
+ igfs.globalSampling(null);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // True => false transition.
+ igfs.globalSampling(true);
+ igfs.globalSampling(false);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // False => true transition.
+ igfs.globalSampling(true);
+
+ fs = fileSystem();
+
+ assert logEnabled();
+ }
+
+ /**
+ * Ensure that log directory is set to IGFS when client FS connects.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public void testLogDirectory() throws Exception {
+ startUp();
+
+ assertEquals(Paths.get(U.getIgniteHome()).normalize().toString(),
+ igfs.clientLogDirectory());
+ }
+
+ /**
+ * Instantiate new file system.
+ *
+ * @return New file system.
+ * @throws Exception If failed.
+ */
+ private IgniteHadoopFileSystem fileSystem() throws Exception {
+ Configuration fsCfg = new Configuration();
+
+ fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));
+
+ fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ if (logging)
+ fsCfg.setBoolean(String.format(PARAM_IGFS_LOG_ENABLED, "igfs:igfs-grid@"), logging);
+
+ fsCfg.setStrings(String.format(PARAM_IGFS_LOG_DIR, "igfs:igfs-grid@"), U.getIgniteHome());
+
+ return (IgniteHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
+ }
+
+ /**
+ * Ensure that real logger is used by the file system.
+ *
+ * @return {@code True} in case path is secondary.
+ * @throws Exception If failed.
+ */
+ private boolean logEnabled() throws Exception {
+ assert fs != null;
+
+ Field field = fs.getClass().getDeclaredField("clientLog");
+
+ field.setAccessible(true);
+
+ return ((IgfsLogger)field.get(fs)).isLogEnabled();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java
new file mode 100644
index 0000000..e33cde7
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import java.util.*;
+
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test.
+ */
+public abstract class IgniteHadoopFileSystemLoopbackAbstractSelfTest extends
+ IgniteHadoopFileSystemAbstractSelfTest {
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ * @param skipEmbed Skip embedded mode flag.
+ */
+ protected IgniteHadoopFileSystemLoopbackAbstractSelfTest(IgfsMode mode, boolean skipEmbed) {
+ super(mode, skipEmbed, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
+ return new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
+ }};
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
new file mode 100644
index 0000000..d46001d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest() {
+ super(DUAL_ASYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
new file mode 100644
index 0000000..fa64734
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest() {
+ super(DUAL_SYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
new file mode 100644
index 0000000..9035acd
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest() {
+ super(PRIMARY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
new file mode 100644
index 0000000..8198c52
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest() {
+ super(PROXY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
new file mode 100644
index 0000000..246c516
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest() {
+ super(DUAL_ASYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
new file mode 100644
index 0000000..b5140af
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest() {
+ super(DUAL_SYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java
new file mode 100644
index 0000000..572ac4b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest() {
+ super(PRIMARY, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java
new file mode 100644
index 0000000..006c271
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest() {
+ super(PROXY, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
new file mode 100644
index 0000000..3f20070
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+
+import java.net.*;
+import java.util.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Ensures correct modes resolution for SECONDARY paths.
+ */
+public class IgniteHadoopFileSystemSecondaryModeSelfTest extends IgfsCommonAbstractTest {
+ /** Path to check. */
+ private static final Path PATH = new Path("/dir");
+
+ /** Pattern matching the path. */
+ private static final String PATTERN_MATCHES = "/dir";
+
+ /** Pattern doesn't matching the path. */
+ private static final String PATTERN_NOT_MATCHES = "/files";
+
+ /** Default IGFS mode. */
+ private IgfsMode mode;
+
+ /** Path modes. */
+ private Map<String, IgfsMode> pathModes;
+
+ /** File system. */
+ private IgniteHadoopFileSystem fs;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ mode = null;
+ pathModes = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ U.closeQuiet(fs);
+
+ fs = null;
+
+ G.stopAll(true);
+ }
+
+ /**
+ * Perform initial startup.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("NullableProblems")
+ private void startUp() throws Exception {
+ startUpSecondary();
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setDefaultMode(mode);
+ igfsCfg.setPathModes(pathModes);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "10500");
+ }});
+
+ igfsCfg.setManagementPort(-1);
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
+ "igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/",
+ "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("igfs-grid");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+
+ G.start(cfg);
+
+ Configuration fsCfg = new Configuration();
+
+ fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));
+
+ fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ fs = (IgniteHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
+ }
+
+ /**
+ * Startup secondary file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startUpSecondary() throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs-secondary");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "11500");
+ }});
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("igfs-grid-secondary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+
+ G.start(cfg);
+ }
+
+ /**
+ * Check path resolution when secondary mode is not default and there are no other exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryNotDefaultNoExclusions() throws Exception {
+ mode = PRIMARY;
+
+ startUp();
+
+ assert !secondary(PATH);
+ assert !secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is not default and there is no matching exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryNotDefaultNonMatchingExclusion() throws Exception {
+ mode = PRIMARY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PROXY));
+
+ startUp();
+
+ assert !secondary(PATH);
+ assert !secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is not default and there is matching exclusion path.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryNotDefaultMatchingExclusion() throws Exception {
+ mode = PRIMARY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PROXY), F.t(PATTERN_MATCHES, PROXY));
+
+ startUp();
+
+ assert secondary(PATH);
+ assert secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is default and there is no exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryDefaultNoExclusions() throws Exception {
+ mode = PROXY;
+
+ startUp();
+
+ assert secondary(PATH);
+ assert secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is default and there is no matching exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryDefaultNonMatchingExclusion() throws Exception {
+ mode = PROXY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY));
+
+ startUp();
+
+ assert secondary(PATH);
+ assert secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is default and there is no matching exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryDefaultMatchingExclusion() throws Exception {
+ mode = PROXY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY), F.t(PATTERN_MATCHES, PRIMARY));
+
+ startUp();
+
+ assert !secondary(PATH);
+ assert !secondary(PATH);
+ }
+
+ /**
+ * Set IGFS modes for particular paths.
+ *
+ * @param modes Modes.
+ */
+ @SafeVarargs
+ final void pathModes(IgniteBiTuple<String, IgfsMode>... modes) {
+ assert modes != null;
+
+ pathModes = new LinkedHashMap<>(modes.length, 1.0f);
+
+ for (IgniteBiTuple<String, IgfsMode> mode : modes)
+ pathModes.put(mode.getKey(), mode.getValue());
+ }
+
+ /**
+ * Check whether the given path is threaten as SECONDARY in the file system.
+ *
+ * @param path Path to check.
+ * @return {@code True} in case path is secondary.
+ * @throws Exception If failed.
+ */
+ private boolean secondary(Path path) throws Exception {
+ return fs.mode(path) == PROXY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
new file mode 100644
index 0000000..991045b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.util.ipc.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.testframework.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * IGFS Hadoop file system IPC self test.
+ */
+public abstract class IgniteHadoopFileSystemShmemAbstractSelfTest extends IgniteHadoopFileSystemAbstractSelfTest {
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ * @param skipEmbed Skip embedded mode flag.
+ */
+ protected IgniteHadoopFileSystemShmemAbstractSelfTest(IgfsMode mode, boolean skipEmbed) {
+ super(mode, skipEmbed, false);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
+ return new HashMap<String, String>() {{
+ put("type", "shmem");
+ put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
+ }};
+ }
+
+ /**
+ * Checks correct behaviour in case when we run out of system
+ * resources.
+ *
+ * @throws Exception If error occurred.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testOutOfResources() throws Exception {
+ final Collection<IpcEndpoint> eps = new LinkedList<>();
+
+ try {
+ IgniteCheckedException e = (IgniteCheckedException)GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @SuppressWarnings("InfiniteLoopStatement")
+ @Override public Object call() throws Exception {
+ while (true) {
+ IpcEndpoint ep = IpcEndpointFactory.connectEndpoint("shmem:10500", log);
+
+ eps.add(ep);
+ }
+ }
+ }, IgniteCheckedException.class, null);
+
+ assertNotNull(e);
+
+ String msg = e.getMessage();
+
+ assertTrue("Invalid exception: " + X.getFullStackTrace(e),
+ msg.contains("(error code: 28)") ||
+ msg.contains("(error code: 24)") ||
+ msg.contains("(error code: 12)"));
+ }
+ finally {
+ for (IpcEndpoint ep : eps)
+ ep.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
new file mode 100644
index 0000000..ed34398
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest() {
+ super(DUAL_ASYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
new file mode 100644
index 0000000..4b7bfb1
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest() {
+ super(DUAL_SYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
new file mode 100644
index 0000000..c1393d3
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest() {
+ super(PRIMARY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
new file mode 100644
index 0000000..2c97e7b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest() {
+ super(PROXY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java
new file mode 100644
index 0000000..6d9973f
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest() {
+ super(DUAL_ASYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java
new file mode 100644
index 0000000..6ec084d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalDualSyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalDualSyncSelfTest() {
+ super(DUAL_SYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java
new file mode 100644
index 0000000..03f3fee
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalPrimarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalPrimarySelfTest() {
+ super(PRIMARY, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java
new file mode 100644
index 0000000..bc27674
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalSecondarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalSecondarySelfTest() {
+ super(PROXY, true);
+ }
+}