You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2015/09/22 19:48:08 UTC
bigtop git commit: BIGTOP-1970. Ignite IGFS now fully supports
mutiltenancy: deployment should configure it with HDFS backing
Repository: bigtop
Updated Branches:
refs/heads/master 6780b1caf -> c922418d7
BIGTOP-1970. Ignite IGFS now fully supports mutiltenancy: deployment should configure it with HDFS backing
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c922418d
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c922418d
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c922418d
Branch: refs/heads/master
Commit: c922418d7a9d59c36adb75fd3f147c98b0768165
Parents: 6780b1c
Author: Konstantin Boudnik <co...@apache.org>
Authored: Mon Sep 21 16:03:25 2015 -0700
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Tue Sep 22 10:45:46 2015 -0700
----------------------------------------------------------------------
.../modules/ignite-hadoop/manifests/init.pp | 7 +
.../ignite-hadoop/templates/default-config.xml | 187 +++++++++++++++++++
2 files changed, 194 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c922418d/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
index de658a5..81832af 100644
--- a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
@@ -21,6 +21,9 @@ class ignite-hadoop {
}
define server() {
+ $hadoop_head_node = hiera("bigtop::hadoop_head_node")
+ $hadoop_namenode_port = hiera("hadoop::common_hdfs::hadoop_namenode_port", "8020")
+
package { "ignite-hadoop":
ensure => latest,
}
@@ -41,6 +44,10 @@ class ignite-hadoop {
mode => '0755',
require => Package["ignite-hadoop-service"],
}
+ file { "/etc/ignite-hadoop/conf/default-config.xml":
+ content => template('ignite-hadoop/default-config.xml'),
+ require => [Package["ignite-hadoop"]],
+ }
file { "/etc/hadoop/ignite.client.conf/core-site.xml":
content => template('ignite-hadoop/core-site.xml'),
require => [File["/etc/hadoop/ignite.client.conf"]],
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c922418d/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml
new file mode 100644
index 0000000..bfaee60
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml
@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/util
+ http://www.springframework.org/schema/util/spring-util.xsd">
+
+ <!--
+ Optional description.
+ -->
+ <description>
+ Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
+ Ignite node will start with this configuration by default.
+ </description>
+
+ <!--
+ Initialize property configurer so we can reference environment variables.
+ -->
+ <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
+ <property name="searchSystemEnvironment" value="true"/>
+ </bean>
+
+ <!--
+ Abstract IGFS file system configuration to be used as a template.
+ -->
+ <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true">
+ <!-- Must correlate with cache affinity mapper. -->
+ <property name="blockSize" value="#{128 * 1024}"/>
+ <property name="perNodeBatchSize" value="512"/>
+ <property name="perNodeParallelBatchCount" value="16"/>
+
+ <property name="prefetchBlocks" value="32"/>
+ </bean>
+
+ <!--
+ Abstract cache configuration for IGFS file data to be used as a template.
+ -->
+ <bean id="dataCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true">
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+ <property name="backups" value="0"/>
+ <property name="affinityMapper">
+ <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">
+ <!-- How many sequential blocks will be stored on the same node. -->
+ <constructor-arg value="512"/>
+ </bean>
+ </property>
+ </bean>
+
+ <!--
+ Abstract cache configuration for IGFS metadata to be used as a template.
+ -->
+ <bean id="metaCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true">
+ <property name="cacheMode" value="REPLICATED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+ </bean>
+
+ <!--
+ Configuration of Ignite node.
+ -->
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ <!--
+ Apache Hadoop Accelerator configuration.
+ -->
+ <property name="hadoopConfiguration">
+ <bean class="org.apache.ignite.configuration.HadoopConfiguration">
+ <!-- Information about finished jobs will be kept for 30 seconds. -->
+ <property name="finishedJobInfoTtl" value="30000"/>
+ </bean>
+ </property>
+
+ <!--
+ This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker.
+ -->
+ <property name="connectorConfiguration">
+ <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+ <property name="port" value="11211"/>
+ </bean>
+ </property>
+
+ <!--
+ Configure one IGFS file system instance named "igfs" on this node.
+ -->
+ <property name="fileSystemConfiguration">
+ <list>
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase">
+ <property name="name" value="igfs"/>
+
+ <!-- Caches with these names must be configured. -->
+ <property name="metaCacheName" value="igfs-meta"/>
+ <property name="dataCacheName" value="igfs-data"/>
+
+ <!-- Configure TCP endpoint for communication with the file system instance. -->
+ <property name="ipcEndpointConfiguration">
+ <bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration">
+ <property name="type" value="TCP" />
+ <property name="host" value="0.0.0.0" />
+ <property name="port" value="10500" />
+ </bean>
+ </property>
+
+ <!-- Sample secondary file system configuration.
+ 'uri' - the URI of the secondary file system.
+ 'cfgPath' - optional configuration path of the secondary file system,
+ e.g. /opt/foo/core-site.xml. Typically left to be null.
+ 'userName' - optional user name to access the secondary file system on behalf of. Use it
+ if Hadoop client and the Ignite node are running on behalf of different users.
+ -->
+ <property name="secondaryFileSystem">
+ <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">
+ <constructor-arg name="uri" value='hdfs://<%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>'/>
+ <constructor-arg name="cfgPath"><null/></constructor-arg>
+ <constructor-arg name="userName" value="ignite"/>
+ </bean>
+ </property>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ Caches needed by IGFS.
+ -->
+ <property name="cacheConfiguration">
+ <list>
+ <!-- File system metadata cache. -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="metaCacheCfgBase">
+ <property name="name" value="igfs-meta"/>
+ </bean>
+
+ <!-- File system files data cache. -->
+ <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="dataCacheCfgBase">
+ <property name="name" value="igfs-data"/>
+ </bean>
+ </list>
+ </property>
+
+ <!--
+ Disable events.
+ -->
+ <property name="includeEventTypes">
+ <list>
+ <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/>
+ <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/>
+ <util:constant static-field="org.apache.ignite.events.EventType.EVT_JOB_MAPPED"/>
+ </list>
+ </property>
+
+ <!--
+ TCP discovery SPI can be configured with list of addresses if multicast is not available.
+ -->
+ <!--
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ -->
+ </bean>
+</beans>