You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2017/03/02 11:30:58 UTC
[21/50] [abbrv] ignite git commit: IGNITE-2356: Added
high-availability feature to IGFS client. From now on it is possible to
connect to IGFS through Ignite client. This way in case one server fails,
client will still be able to work with other nodes. Th
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml
new file mode 100644
index 0000000..072c6db
--- /dev/null
+++ b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="gridName" value="IGFS-cli-0"/>
+
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11211"/>
+ </bean>
+ </property>
+
+ <!--
+ Configure one IGFS file system instance named "igfs" on this node.
+ -->
+ <property name="fileSystemConfiguration">
+ <list>
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs-0"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="PRIMARY"/>
+ <property name="ipcEndpointEnabled" value="false"/>
+
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ <property name="localAddress" value="127.0.0.1"/>
+ <property name="localPort" value="47500"/>
+ <property name="joinTimeout" value="5000"/>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml
new file mode 100644
index 0000000..e52921d
--- /dev/null
+++ b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="gridName" value="IGFS-cli-1"/>
+
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11212"/>
+ </bean>
+ </property>
+
+ <!--
+ Configure one IGFS file system instance named "igfs" on this node.
+ -->
+ <property name="fileSystemConfiguration">
+ <list>
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs-1"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="PRIMARY"/>
+ <property name="ipcEndpointEnabled" value="false"/>
+
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47600..47609</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ <property name="localAddress" value="127.0.0.1"/>
+ <property name="localPort" value="47600"/>
+ <property name="joinTimeout" value="5000"/>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml
new file mode 100644
index 0000000..9344e99
--- /dev/null
+++ b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="gridName" value="IGFS-cli-2"/>
+
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11213"/>
+ </bean>
+ </property>
+
+ <!--
+ Configure one IGFS file system instance named "igfs" on this node.
+ -->
+ <property name="fileSystemConfiguration">
+ <list>
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs-2"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="PRIMARY"/>
+ <property name="ipcEndpointEnabled" value="false"/>
+
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47700..47709</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ <property name="localAddress" value="127.0.0.1"/>
+ <property name="localPort" value="47700"/>
+ <property name="joinTimeout" value="5000"/>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml b/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml
new file mode 100644
index 0000000..6fc38cd
--- /dev/null
+++ b/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="clientMode" value="true"/>
+ <property name="gridName" value="test-IGFS-cli"/>
+
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11211"/>
+ </bean>
+ </property>
+
+ <property name="fileSystemConfiguration">
+ <list>
+ <!--
+ Configure PRIMARY IGFS file system instance named "igfs"
+ -->
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="DUAL_ASYNC"/>
+
+ <property name="secondaryFileSystem">
+ <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">
+ <property name="defaultUserName" value="secondary-default"/>
+
+ <property name="fileSystemFactory">
+ <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory">
+ <property name="uri" value="igfs://igfs_secondary@127.0.0.1:11500/"/>
+ <property name="configPaths" value="/work/core-site-test.xml"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml b/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml
new file mode 100644
index 0000000..fcd8d82
--- /dev/null
+++ b/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="clientMode" value="true"/>
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11211"/>
+ </bean>
+ </property>
+
+ <property name="fileSystemConfiguration">
+ <list>
+ <!--
+ Configure PRIMARY IGFS file system instance named "igfs"
+ -->
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="DUAL_SYNC"/>
+
+ <property name="secondaryFileSystem">
+ <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">
+ <property name="defaultUserName" value="secondary-default"/>
+
+ <property name="fileSystemFactory">
+ <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory">
+ <property name="uri" value="igfs://igfs_secondary@127.0.0.1:11500/"/>
+ <property name="configPaths" value="/work/core-site-test.xml"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-primary.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/igfs-cli-config-primary.xml b/modules/hadoop/src/test/config/igfs-cli-config-primary.xml
new file mode 100644
index 0000000..cc066b2
--- /dev/null
+++ b/modules/hadoop/src/test/config/igfs-cli-config-primary.xml
@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="clientMode" value="true"/>
+ <property name="gridName" value="test-IGFS-cli"/>
+
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11211"/>
+ </bean>
+ </property>
+
+ <!--
+ Configure one IGFS file system instance named "igfs" on this node.
+ -->
+ <property name="fileSystemConfiguration">
+ <list>
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="PRIMARY"/>
+ <property name="ipcEndpointEnabled" value="false"/>
+
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml b/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml
new file mode 100644
index 0000000..f25fd50
--- /dev/null
+++ b/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file.
+
+ When starting a standalone Ignite node, you need to execute the following command:
+ {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
+
+ When starting Ignite from Java IDE, pass path to this file into Ignition:
+ Ignition.start("path-to-this-file/default-config.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="clientMode" value="true"/>
+ <property name="marshaller">
+ <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
+ </bean>
+ </property>
+
+ <!--
+ Configure caches where IGFS will store data.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!--
+ Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL,
+ and must have backups to maintain file system consistency in case of node crash.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="replicated"/>
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ </bean>
+
+ <!--
+ Configure data cache where file's data will be stored.
+ -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="partitioned"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <property name="groupSize" value="128"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11211"/>
+ </bean>
+ </property>
+
+ <property name="fileSystemConfiguration">
+ <list>
+ <!--
+ Configure PRIMARY IGFS file system instance named "igfs"
+ -->
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
+ <!-- IGFS name you will use to access IGFS through Hadoop API. -->
+ <property name="name" value="igfs"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="replicated"/>
+ <property name="dataCacheName" value="partitioned"/>
+ <property name="blockSize" value="524288"/>
+ <property name="defaultMode" value="PROXY"/>
+
+ <property name="secondaryFileSystem">
+ <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">
+ <property name="defaultUserName" value="secondary-default"/>
+
+ <property name="fileSystemFactory">
+ <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory">
+ <property name="uri" value="igfs://igfs_secondary@127.0.0.1:11500/"/>
+ <property name="configPaths" value="/work/core-site-test.xml"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 1bcbdaa..4d4f68b 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -17,6 +17,30 @@
package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
@@ -53,6 +77,7 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -60,30 +85,6 @@ import org.apache.ignite.testframework.GridTestUtils;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ThreadLocalRandom8;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.Field;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
@@ -102,7 +103,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
private static final String PRIMARY_AUTHORITY = "igfs@";
/** Primary file systme URI. */
- private static final String PRIMARY_URI = "igfs://" + PRIMARY_AUTHORITY + "/";
+ protected static final String PRIMARY_URI = "igfs://" + PRIMARY_AUTHORITY + "/";
/** Secondary file system authority. */
private static final String SECONDARY_AUTHORITY = "igfs_secondary@127.0.0.1:11500";
@@ -122,6 +123,9 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
/** Group size. */
public static final int GRP_SIZE = 128;
+ /** Group size. */
+ public static final int GRID_COUNT = 4;
+
/** Path to the default hadoop configuration. */
public static final String HADOOP_FS_CFG = "examples/config/filesystem/core-site.xml";
@@ -135,7 +139,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
private static CyclicBarrier barrier;
/** File system. */
- private static FileSystem fs;
+ protected static FileSystem fs;
/** Default IGFS mode. */
protected final IgfsMode mode;
@@ -187,6 +191,8 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
this.skipLocShmem = skipLocShmem;
endpoint = skipLocShmem ? "127.0.0.1:10500" : "shmem:10500";
+
+ ((TcpDiscoveryVmIpFinder)IP_FINDER).setAddresses(Collections.singleton("127.0.0.1:47500..47509"));
}
/**
@@ -199,7 +205,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
- Configuration secondaryConf = configuration(SECONDARY_AUTHORITY, true, true);
+ Configuration secondaryConf = configurationSecondary(SECONDARY_AUTHORITY);
secondaryConf.setInt("fs.igfs.block.size", 1024);
@@ -220,11 +226,22 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
}
/**
+ * @throws Exception
+ */
+ protected void stopNodes() throws Exception {
+ for (int i = 0; i < GRID_COUNT; ++i)
+ stopGrid(i);
+
+ G.stop("grid_secondary", false);
+
+ Thread.sleep(500);
+ }
+ /**
* Starts the nodes for this test.
*
* @throws Exception If failed.
*/
- private void startNodes() throws Exception {
+ protected void startNodes() throws Exception {
if (mode != PRIMARY) {
// Start secondary IGFS.
FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
@@ -258,8 +275,10 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
cfg.setGridName("grid_secondary");
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+ discoSpi.setLocalPort(47510);
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+ TcpDiscoveryVmIpFinder finder = new TcpDiscoveryVmIpFinder(true);
+ discoSpi.setIpFinder(finder);
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
@@ -269,7 +288,10 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
G.start(cfg);
}
- startGrids(4);
+ for (int i = 1; i < GRID_COUNT; ++i)
+ startGrid(i);
+
+ startGrid(0);
}
/** {@inheritDoc} */
@@ -329,12 +351,14 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
+ IgniteConfiguration cfg = new IgniteConfiguration();
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
discoSpi.setIpFinder(IP_FINDER);
+ cfg.setGridName(gridName);
+ cfg.setMarshaller(new OptimizedMarshaller());
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(cacheConfiguration(gridName));
cfg.setFileSystemConfiguration(igfsConfiguration(gridName));
@@ -2049,7 +2073,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
final FSDataOutputStream s = fs.create(filePath); // Open the stream before stopping IGFS.
try {
- G.stopAll(true); // Stop the server.
+ stopNodes();
startNodes(); // Start server again.
@@ -2095,7 +2119,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
for (int i = 0; i < nClients; i++)
q.add(FileSystem.get(primaryFsUri, cfg));
- G.stopAll(true); // Stop the server.
+ stopNodes();
startNodes(); // Start server again.
@@ -2414,7 +2438,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
* @param skipLocShmem Whether to skip local shmem mode.
* @return Configuration.
*/
- private static Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
+ protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
Configuration cfg = new Configuration();
cfg.set("fs.defaultFS", "igfs://" + authority + "/");
@@ -2432,4 +2456,29 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
return cfg;
}
+
+ /**
+ * Create configuration for test.
+ *
+ * @param authority Authority.
+ * @param skipEmbed Whether to skip embedded mode.
+ * @param skipLocShmem Whether to skip local shmem mode.
+ * @return Configuration.
+ */
+ protected Configuration configurationSecondary(String authority) {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://" + authority + "/");
+ cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
+
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
+
+ return cfg;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java
new file mode 100644
index 0000000..8198cd3
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.FileSystemConfiguration;
+import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
+import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
+import org.apache.ignite.igfs.IgfsMode;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * IGFS Hadoop file system Ignite client -based self test.
+ */
+public abstract class IgniteHadoopFileSystemClientBasedAbstractSelfTest extends IgniteHadoopFileSystemAbstractSelfTest {
+ /** Alive node index. */
+ private static final int ALIVE_NODE_IDX = GRID_COUNT - 1;
+
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ */
+ IgniteHadoopFileSystemClientBasedAbstractSelfTest(IgfsMode mode) {
+ super(mode, true, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgfsIpcEndpointConfiguration primaryIpcEndpointConfiguration(final String gridName) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected FileSystemConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException {
+ FileSystemConfiguration cfg = super.igfsConfiguration(gridName);
+
+ cfg.setIpcEndpointEnabled(false);
+
+ return cfg;
+ }
+
+ /**
+ * @return Path to Ignite client node configuration.
+ */
+ protected abstract String getClientConfig();
+
+ /** {@inheritDoc} */
+ @Override protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://" + authority + "/");
+ cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP, authority), true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_REMOTE_TCP, authority), true);
+ cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority),
+ getClientConfig());
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testClientReconnect() throws Exception {
+ Path filePath = new Path(PRIMARY_URI, "file1");
+
+ final FSDataOutputStream s = fs.create(filePath); // Open the stream before stopping IGFS.
+
+ try {
+ restartServerNodesExceptOne();
+
+ // Check that client is again operational.
+ assertTrue(fs.mkdirs(new Path(PRIMARY_URI, "dir1/dir2")));
+
+ s.write("test".getBytes());
+
+ s.flush(); // Flush data to the broken output stream.
+
+ assertTrue(fs.exists(filePath));
+ }
+ finally {
+ U.closeQuiet(s); // Safety.
+ }
+ }
+
+ /**
+ * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
+ *
+ * @throws Exception If error occurs.
+ */
+ @Override public void testClientReconnectMultithreaded() throws Exception {
+ final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();
+
+ Configuration cfg = new Configuration();
+
+ for (Map.Entry<String, String> entry : primaryFsCfg)
+ cfg.set(entry.getKey(), entry.getValue());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ final int nClients = 1;
+
+ // Initialize clients.
+ for (int i = 0; i < nClients; i++)
+ q.add(FileSystem.get(primaryFsUri, cfg));
+
+ restartServerNodesExceptOne();
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ FileSystem fs = q.poll();
+
+ try {
+ // Check that client is again operational.
+ assertTrue(fs.mkdirs(new Path("/" + Thread.currentThread().getName())));
+
+ return true;
+ }
+ finally {
+ U.closeQuiet(fs);
+ }
+ }
+ }, nClients, "test-client");
+ }
+
+ /**
+ *
+ * @throws Exception If failed.
+ */
+ private void restartServerNodesExceptOne() throws Exception {
+ stopAllNodesExcept(ALIVE_NODE_IDX);
+
+ Thread.sleep(500);
+
+ startAllNodesExcept(ALIVE_NODE_IDX); // Start server again.
+
+ Thread.sleep(500);
+
+ stopGrid(ALIVE_NODE_IDX);
+
+ Thread.sleep(500);
+
+ startGrid(ALIVE_NODE_IDX);
+
+ Thread.sleep(500);
+ }
+
+ /**
+ * @param nodeIdx Node index to not stop
+ */
+ private void stopAllNodesExcept(int nodeIdx) {
+ for (int i = 0; i < GRID_COUNT; ++i)
+ if (i != nodeIdx)
+ stopGrid(i);
+ }
+
+ /**
+ * @param nodeIdx Node index to not stop
+ * @throws Exception If failed.
+ */
+ private void startAllNodesExcept(int nodeIdx) throws Exception {
+ for (int i = 0; i < GRID_COUNT; ++i)
+ if (i != nodeIdx)
+ startGrid(i);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java
new file mode 100644
index 0000000..3bd61b5
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import org.apache.ignite.igfs.IgfsMode;
+
+/**
+ * IGFS Hadoop file system Ignite client -based self test for DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemClientBasedDualAsyncSelfTest
+ extends IgniteHadoopFileSystemClientBasedAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemClientBasedDualAsyncSelfTest() {
+ super(IgfsMode.DUAL_ASYNC);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String getClientConfig() {
+ return "modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java
new file mode 100644
index 0000000..c7e97d7
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import org.apache.ignite.igfs.IgfsMode;
+
+/**
+ * IGFS Hadoop file system Ignite client -based self test for DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemClientBasedDualSyncSelfTest
+ extends IgniteHadoopFileSystemClientBasedAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemClientBasedDualSyncSelfTest() {
+ super(IgfsMode.DUAL_SYNC);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String getClientConfig() {
+ return "modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
new file mode 100644
index 0000000..6091dca
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import java.net.URI;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * IGFS Hadoop file system Ignite client -based self test for PROXY mode.
+ */
+public class IgniteHadoopFileSystemClientBasedOpenTest extends GridCommonAbstractTest {
+ /** Config root path. */
+ private static final String [] CONFIGS = {
+ "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml",
+ "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml",
+ "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml"
+ };
+
+ /** Nodes types. */
+ private static NodeType[][] nodesTypes = new NodeType[][] {
+ {NodeType.REMOTE, NodeType.REMOTE, NodeType.REMOTE},
+ {NodeType.LOCAL, NodeType.REMOTE, NodeType.REMOTE},
+ {NodeType.REMOTE, NodeType.LOCAL, NodeType.REMOTE},
+ {NodeType.LOCAL, NodeType.LOCAL, NodeType.REMOTE},
+ {NodeType.REMOTE, NodeType.REMOTE, NodeType.LOCAL},
+ {NodeType.LOCAL, NodeType.REMOTE, NodeType.LOCAL},
+ {NodeType.REMOTE, NodeType.LOCAL, NodeType.LOCAL},
+ {NodeType.LOCAL, NodeType.LOCAL, NodeType.LOCAL},
+
+ {NodeType.LOCAL_SAME_NAME, NodeType.REMOTE, NodeType.REMOTE},
+ {NodeType.REMOTE, NodeType.LOCAL_SAME_NAME, NodeType.REMOTE},
+ {NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME, NodeType.REMOTE},
+ {NodeType.REMOTE, NodeType.REMOTE, NodeType.LOCAL_SAME_NAME},
+ {NodeType.LOCAL_SAME_NAME, NodeType.REMOTE, NodeType.LOCAL_SAME_NAME},
+ {NodeType.REMOTE, NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME},
+ {NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME},
+ };
+
+ /** Test threads count. */
+ private static final int THREADS_COUNT = 20;
+
+ /** Iterations count. */
+ private static final int ITERATIONS = 10;
+
+ /** Iterations count. */
+ private static boolean skipInProc;
+
+ /**
+ * @param idx Grid index.
+ * @return Path to Ignite config file.
+ */
+ private static String cfgPath(int idx) {
+ return CONFIGS[idx];
+ }
+
+ /**
+ * @param idx Grid index.
+ * @return Path to Ignite config file.
+ */
+ private static String uri(int idx) {
+ return "igfs://" + authority(idx) + '/';
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteBiTuple<IgniteConfiguration, GridSpringResourceContext> cfgPair =
+ IgnitionEx.loadConfiguration(cfgPath(getTestGridIndex(gridName)));
+
+ IgniteConfiguration cfg = cfgPair.get1();
+
+ cfg.setGridName(gridName);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 15 * 60_000;
+ }
+
+ /**
+ * Create configuration for test.
+ *
+ * @param idx Grid index.
+ * @return Configuration.
+ */
+ protected Configuration configuration(int idx) {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://" + authority(idx) + '/');
+ cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority(idx)),
+ cfgPath(idx));
+
+ if (skipInProc)
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority(idx)), true);
+
+ return cfg;
+ }
+
+ /**
+ * Create configuration for test.
+ *
+ * @return Configuration.
+ */
+ protected Configuration configurationWrongIgfs() {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://igfs-wrong-name@/");
+ cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, "igfs-wrong-name@"),
+ cfgPath(0));
+
+ return cfg;
+ }
+
+ /**
+ * @param idx Grid index.
+ * @return IGFS authority.
+ */
+ private static String authority(int idx) {
+ return "igfs-" + idx + "@";
+ }
+
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFsOpenMultithreaded() throws Exception {
+ skipInProc = false;
+
+ checkFsOpenWithAllNodesTypes();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void checkFsOpenWithAllNodesTypes() throws Exception {
+ for (int i = 0; i < nodesTypes.length; ++i) {
+ log.info("Begin test case for nodes: " + S.arrayToString(NodeType.class, nodesTypes[i]));
+
+ startNodes(nodesTypes[i]);
+
+ // Await clusters start
+ Thread.sleep(10_000);
+
+ try {
+ checkFsOpenMultithreaded();
+ } finally {
+ stopAllGrids();
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFsOpenMultithreadedSkipInProc() throws Exception {
+ skipInProc = true;
+
+ checkFsOpenWithAllNodesTypes();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIgniteClientWithIgfsMisconfigure() throws Exception {
+ startNodes(new NodeType[] {NodeType.REMOTE, NodeType.REMOTE, NodeType.REMOTE});
+
+ // Await clusters start
+ Thread.sleep(10_000);
+
+ try {
+ try (FileSystem fs = FileSystem.get(new URI("igfs://igfs-wrong-name@"), configurationWrongIgfs())) {
+ FSDataOutputStream out = fs.create(new Path("igfs://igfs-wrong-name@/file"), true);
+
+ assert false : "Exception must be thrown";
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage().contains("Cannot find IGFS 'igfs-wrong-name' at the node"));
+ }
+ } finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param nodeTypes Types of server nodes.
+ * @throws Exception If failed.
+ */
+ public void startNodes(NodeType[] nodeTypes) throws Exception {
+ assert nodeTypes.length == CONFIGS.length;
+
+ for (int i = 0; i < CONFIGS.length; i++) {
+ String name = getTestGridName(i);
+
+ switch (nodeTypes[i]) {
+ case REMOTE:
+ startRemoteGrid(name, getConfiguration(name),
+ null, null, false);
+ break;
+
+ case LOCAL:
+ startGrid(name, getConfiguration(name));
+ break;
+
+ case LOCAL_SAME_NAME:
+ startGrid("IGFS-cli-" + i, getConfiguration(name));
+ break;
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void checkFsOpenMultithreaded() throws Exception {
+ X.println("Start hadoop client file system open test");
+
+ final AtomicInteger fileIdx = new AtomicInteger();
+
+ IgniteInternalFuture fut = multithreadedAsync(new Runnable() {
+ @Override public void run() {
+ for (int iter = 0; iter < ITERATIONS; iter++) {
+ for (int i = 0; i < CONFIGS.length; i++) {
+ try (FileSystem fs = FileSystem.get(new URI(uri(i)), configuration(i))) {
+ FSDataOutputStream out = fs.create(new Path(uri(i) + "file" +
+ fileIdx.getAndIncrement()), true);
+
+ out.close();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+
+ assert false : "Unexpected exception";
+ }
+ }
+ }
+ }
+ }, THREADS_COUNT);
+
+ fut.get();
+ }
+
+ /**
+ * Node type.
+ */
+ enum NodeType {
+ /** Remote node. */
+ REMOTE,
+
+ /** Node in the test VM. */
+ LOCAL,
+
+ /** Node in the test VM with the name equals to client node config. */
+ LOCAL_SAME_NAME
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java
new file mode 100644
index 0000000..e2995fa
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import org.apache.ignite.igfs.IgfsMode;
+
+/**
+ * IGFS Hadoop file system Ignite client -based self test for PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemClientBasedPrimarySelfTest
+ extends IgniteHadoopFileSystemClientBasedAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemClientBasedPrimarySelfTest() {
+ super(IgfsMode.PRIMARY);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String getClientConfig() {
+ return "modules/hadoop/src/test/config/igfs-cli-config-primary.xml";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java
new file mode 100644
index 0000000..4293b3b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import org.apache.ignite.igfs.IgfsMode;
+
+/**
+ * IGFS Hadoop file system Ignite client -based self test for PROXY mode.
+ */
+public class IgniteHadoopFileSystemClientBasedProxySelfTest extends IgniteHadoopFileSystemClientBasedAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemClientBasedProxySelfTest() {
+ super(IgfsMode.PROXY);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String getClientConfig() {
+ return "modules/hadoop/src/test/config/igfs-cli-config-proxy.xml";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java
new file mode 100644
index 0000000..b3c1681
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.igfs;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
+import org.apache.ignite.igfs.IgfsIpcEndpointType;
+import org.apache.ignite.igfs.IgfsMode;
+
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test.
+ */
+public abstract class IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest extends
+ IgniteHadoopFileSystemAbstractSelfTest {
+
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ */
+ protected IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest(IgfsMode mode) {
+ super(mode, true, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgfsIpcEndpointConfiguration primaryIpcEndpointConfiguration(final String gridName) {
+ IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();
+
+ endpointCfg.setType(IgfsIpcEndpointType.TCP);
+ endpointCfg.setPort(DFLT_IPC_PORT + getTestGridIndex(gridName));
+
+ return endpointCfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ if (getTestGridIndex(gridName) == 0)
+ cfg.setClientMode(true);
+
+ return cfg;
+ }
+}
\ No newline at end of file