You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ki...@apache.org on 2013/09/20 20:30:11 UTC
[02/15] Adding Helix-task-framework and Yarn integration modules
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/Boot2By2Shell.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/Boot2By2Shell.properties b/recipes/meta-cluster-manager/src/main/resources/Boot2By2Shell.properties
new file mode 100644
index 0000000..830a586
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/Boot2By2Shell.properties
@@ -0,0 +1,87 @@
+#
+# Zookeeper (optional)
+#
+zookeeper.datadir=/tmp/meta/zk/data
+zookeeper.logdir=/tmp/meta/zk/log
+zookeeper.port=2199
+
+#
+# Cluster
+#
+cluster.name=cluster
+cluster.address=localhost:2199
+
+#
+# Resource
+#
+resource.0.name=wsprod
+resource.0.cluster=cluster
+resource.0.address=localhost:2199
+resource.0.container=webserver
+resource.0.model=MasterSlave
+resource.0.partitions=15
+resource.0.replica=1
+
+resource.1.name=dbprod
+resource.1.cluster=cluster
+resource.1.address=localhost:2199
+resource.1.container=database
+resource.1.model=MasterSlave
+resource.1.partitions=8
+resource.1.replica=3
+
+#
+# Controller
+#
+controller.name=controller
+controller.cluster=cluster
+controller.address=localhost:2199
+controller.autorefresh=5000
+
+#
+# Metacluster
+#
+metacluster.name=meta
+metacluster.address=localhost:2199
+metacluster.managedcluster=cluster
+metacluster.managedaddress=localhost:2199
+
+#
+# Metaresource
+#
+metaresource.0.name=webserver
+metaresource.0.metacluster=meta
+metaresource.0.metaaddress=localhost:2199
+metaresource.0.class=org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess
+
+metaresource.1.name=database
+metaresource.1.metacluster=meta
+metaresource.1.metaaddress=localhost:2199
+metaresource.1.class=org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess
+
+#
+# Metaprovider
+#
+metaprovider.0.name=provider0
+metaprovider.0.metacluster=meta
+metaprovider.0.metaaddress=localhost:2199
+metaprovider.0.class=org.apache.helix.metamanager.impl.shell.ShellContainerProviderProcess
+
+metaprovider.1.name=provider1
+metaprovider.1.metacluster=meta
+metaprovider.1.metaaddress=localhost:2199
+metaprovider.1.class=org.apache.helix.metamanager.impl.shell.ShellContainerProviderProcess
+
+#
+# Metacontroller
+#
+metacontroller.name=metacontroller
+metacontroller.metacluster=meta
+metacontroller.metaaddress=localhost:2199
+metacontroller.autorefresh=5000
+
+metacontroller.status.class=org.apache.helix.metamanager.impl.shell.ShellStatusProvider
+
+metacontroller.target.class=org.apache.helix.metamanager.impl.StaticTargetProvider
+metacontroller.target.webserver=5
+metacontroller.target.database=3
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/Boot2By2Yarn.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/Boot2By2Yarn.properties b/recipes/meta-cluster-manager/src/main/resources/Boot2By2Yarn.properties
new file mode 100644
index 0000000..6d220eb
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/Boot2By2Yarn.properties
@@ -0,0 +1,98 @@
+#
+# Zookeeper (optional)
+#
+zookeeper.datadir=/tmp/meta/zk/data
+zookeeper.logdir=/tmp/meta/zk/log
+zookeeper.port=2199
+
+#
+# Cluster
+#
+cluster.name=cluster
+cluster.address=rm:2199
+
+#
+# Resource
+#
+resource.0.name=wsprod
+resource.0.cluster=cluster
+resource.0.address=rm:2199
+resource.0.container=webserver
+resource.0.model=MasterSlave
+resource.0.partitions=15
+resource.0.replica=1
+
+resource.1.name=dbprod
+resource.1.cluster=cluster
+resource.1.address=rm:2199
+resource.1.container=database
+resource.1.model=MasterSlave
+resource.1.partitions=8
+resource.1.replica=3
+
+#
+# Controller
+#
+controller.name=controller
+controller.cluster=cluster
+controller.address=rm:2199
+controller.autorefresh=5000
+
+#
+# Metacluster
+#
+metacluster.name=meta
+metacluster.address=rm:2199
+metacluster.managedcluster=cluster
+metacluster.managedaddress=rm:2199
+
+#
+# Metaresource
+#
+metaresource.0.name=webserver
+metaresource.0.metacluster=meta
+metaresource.0.metaaddress=rm:2199
+metaresource.0.class=org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess
+
+metaresource.1.name=database
+metaresource.1.metacluster=meta
+metaresource.1.metaaddress=rm:2199
+metaresource.1.class=org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess
+
+#
+# Metaprovider
+#
+metaprovider.0.name=provider0
+metaprovider.0.metacluster=meta
+metaprovider.0.metaaddress=rm:2199
+metaprovider.0.class=org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProcess
+metaprovider.0.yarndata=rm:2199
+metaprovider.0.resourcemananger=rm:8032
+metaprovider.0.scheduler=rm:8030
+metaprovider.0.user=yarn
+metaprovider.0.hdfs=hdfs://rm:9000/
+
+metaprovider.1.name=provider1
+metaprovider.1.metacluster=meta
+metaprovider.1.metaaddress=rm:2199
+metaprovider.1.class=org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProcess
+metaprovider.1.yarndata=rm:2199
+metaprovider.1.resourcemananger=rm:8032
+metaprovider.1.scheduler=rm:8030
+metaprovider.1.user=yarn
+metaprovider.1.hdfs=hdfs://rm:9000/
+
+#
+# Metacontroller
+#
+metacontroller.name=metacontroller
+metacontroller.metacluster=meta
+metacontroller.metaaddress=rm:2199
+metacontroller.autorefresh=5000
+
+metacontroller.status.class=org.apache.helix.metamanager.impl.yarn.YarnStatusProvider
+metacontroller.status.yarndata=rm:2199
+
+metacontroller.target.class=org.apache.helix.metamanager.impl.StaticTargetProvider
+metacontroller.target.webserver=5
+metacontroller.target.database=3
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/BootLocal.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/BootLocal.properties b/recipes/meta-cluster-manager/src/main/resources/BootLocal.properties
new file mode 100644
index 0000000..a86c9f0
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/BootLocal.properties
@@ -0,0 +1,68 @@
+#
+# Zookeeper (optional)
+#
+zookeeper.datadir=/tmp/meta/zk/data
+zookeeper.logdir=/tmp/meta/zk/log
+zookeeper.port=2199
+
+#
+# Cluster
+#
+cluster.name=cluster
+cluster.address=localhost:2199
+
+#
+# Resource
+#
+resource.name=resource
+resource.cluster=cluster
+resource.address=localhost:2199
+resource.container=container
+resource.model=MasterSlave
+resource.partitions=10
+resource.replica=3
+
+#
+# Controller
+#
+controller.name=controller
+controller.cluster=cluster
+controller.address=localhost:2199
+controller.autorefresh=5000
+
+#
+# Metacluster
+#
+metacluster.name=meta
+metacluster.address=localhost:2199
+metacluster.managedcluster=cluster
+metacluster.managedaddress=localhost:2199
+
+#
+# Metaresource
+#
+metaresource.name=container
+metaresource.metacluster=meta
+metaresource.metaaddress=localhost:2199
+metaresource.class=org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess
+
+#
+# Metaprovider
+#
+metaprovider.name=provider
+metaprovider.metacluster=meta
+metaprovider.metaaddress=localhost:2199
+metaprovider.class=org.apache.helix.metamanager.impl.local.LocalContainerProviderProcess
+
+#
+# Metacontroller
+#
+metacontroller.name=metacontroller
+metacontroller.metacluster=meta
+metacontroller.metaaddress=localhost:2199
+metacontroller.autorefresh=5000
+
+metacontroller.status.class=org.apache.helix.metamanager.impl.local.LocalStatusProvider
+
+metacontroller.target.class=org.apache.helix.metamanager.impl.StaticTargetProvider
+metacontroller.target.container=7
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/cluster.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/cluster.properties b/recipes/meta-cluster-manager/src/main/resources/boot/cluster.properties
new file mode 100644
index 0000000..17d9406
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/cluster.properties
@@ -0,0 +1,2 @@
+cluster.cluster=managed
+cluster.address=localhost:2199
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/controller.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/controller.properties b/recipes/meta-cluster-manager/src/main/resources/boot/controller.properties
new file mode 100644
index 0000000..1d96260
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/controller.properties
@@ -0,0 +1,4 @@
+controller.name=controller
+controller.cluster=managed
+controller.address=localhost:2199
+controller.autorefresh=-1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/metacluster.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/metacluster.properties b/recipes/meta-cluster-manager/src/main/resources/boot/metacluster.properties
new file mode 100644
index 0000000..f1e6062
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/metacluster.properties
@@ -0,0 +1,4 @@
+metacluster.cluster=managed
+metacluster.address=localhost:2199
+metacluster.metacluster=meta
+metacluster.metaaddress=localhost:2199
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/metacontroller.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/metacontroller.properties b/recipes/meta-cluster-manager/src/main/resources/boot/metacontroller.properties
new file mode 100644
index 0000000..133ac69
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/metacontroller.properties
@@ -0,0 +1,4 @@
+controller.name=metacontroller
+controller.cluster=meta
+controller.address=localhost:2199
+controller.autorefresh=5000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/resdb.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/resdb.properties b/recipes/meta-cluster-manager/src/main/resources/boot/resdb.properties
new file mode 100644
index 0000000..0830e50
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/resdb.properties
@@ -0,0 +1,4 @@
+name=zookeeper
+datadir=/tmp/zk/data
+logdir=/tmp/zk/log
+port=2199
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/resws.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/resws.properties b/recipes/meta-cluster-manager/src/main/resources/boot/resws.properties
new file mode 100644
index 0000000..0830e50
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/resws.properties
@@ -0,0 +1,4 @@
+name=zookeeper
+datadir=/tmp/zk/data
+logdir=/tmp/zk/log
+port=2199
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/boot/zookeeper.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/boot/zookeeper.properties b/recipes/meta-cluster-manager/src/main/resources/boot/zookeeper.properties
new file mode 100644
index 0000000..04587c8
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/boot/zookeeper.properties
@@ -0,0 +1,4 @@
+zookeeper.name=zookeeper
+zookeeper.datadir=/tmp/zk/data
+zookeeper.logdir=/tmp/zk/log
+zookeeper.port=2199
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/container.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/container.properties b/recipes/meta-cluster-manager/src/main/resources/container.properties
new file mode 100644
index 0000000..8817165
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/container.properties
@@ -0,0 +1 @@
+class=org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/log4j.properties b/recipes/meta-cluster-manager/src/main/resources/log4j.properties
new file mode 100644
index 0000000..af33e21
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/log4j.properties
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Set root logger level to DEBUG and its only appender to R.
+log4j.rootLogger=ERROR, C
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.C=org.apache.log4j.ConsoleAppender
+log4j.appender.C.layout=org.apache.log4j.PatternLayout
+log4j.appender.C.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+log4j.logger.org.I0Itec=ERROR
+log4j.logger.org.apache=ERROR
+
+log4j.logger.org.apache.helix.metamanager=INFO
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/redisLocal.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/redisLocal.properties b/recipes/meta-cluster-manager/src/main/resources/redisLocal.properties
new file mode 100644
index 0000000..47c0800
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/redisLocal.properties
@@ -0,0 +1,50 @@
+#
+# Zookeeper (optional)
+#
+zookeeper.datadir=/tmp/meta/zk/data
+zookeeper.logdir=/tmp/meta/zk/log
+zookeeper.port=2199
+
+#
+# Meta Cluster
+#
+meta.cluster=meta
+meta.managed=managed
+meta.address=rm:2199
+
+meta.target.type=redis
+meta.target.address=rm:2199
+meta.target.root=redis
+meta.target.records=100000
+meta.target.clients=20
+meta.target.requests=100000
+meta.target.target.get=100000
+meta.target.target.set=100000
+
+meta.status.type=local
+meta.status.metadata=rm:2199
+
+meta.provider.type=local
+meta.provider.name=provider_0
+meta.provider.address=rm:2199
+meta.provider.cluster=managed
+
+meta.provider.containers=cache
+
+meta.provider.container.cache.class=org.apache.helix.metamanager.impl.container.RedisServerProcess
+meta.provider.container.cache.address=rm:2199
+meta.provider.container.cache.root=redis
+meta.provider.container.cache.baseport=17000
+
+#
+# Managed Cluster
+#
+managed.cluster=managed
+managed.address=rm:2199
+
+managed.resources=devcache
+
+managed.resource.devcache.container=cache
+managed.resource.devcache.model=OnlineOffline
+managed.resource.devcache.partitions=10
+managed.resource.devcache.replica=1
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/resources/redisYarn.properties
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/resources/redisYarn.properties b/recipes/meta-cluster-manager/src/main/resources/redisYarn.properties
new file mode 100644
index 0000000..1862781
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/resources/redisYarn.properties
@@ -0,0 +1,52 @@
+#
+# Zookeeper (optional)
+#
+zookeeper.datadir=/export/home/eng/apucher/zk/data
+zookeeper.logdir=/export/home/eng/apucher/zk/log
+zookeeper.port=2199
+
+#
+# Meta Cluster
+#
+meta.address=rm:2199
+meta.interval=5000
+
+meta.target.type=redis
+meta.target.address=rm:2199
+meta.target.root=redis
+meta.target.interval=20000
+meta.target.timeout=10000
+meta.target.get=250000
+meta.target.min=3
+meta.target.max=23
+
+meta.status.type=yarn
+meta.status.metadata=rm:2199
+
+meta.provider.type=yarn
+meta.provider.name=provider_0
+meta.provider.address=rm:2199
+meta.provider.cluster=managed
+meta.provider.metadata=rm:2199
+meta.provider.resourcemananger=rm:8032
+meta.provider.scheduler=rm:8030
+meta.provider.user=yarn
+meta.provider.hdfs=hdfs://rm:9000/
+meta.provider.containers=cache
+
+meta.provider.container.cache.class=org.apache.helix.metamanager.impl.container.RedisServerProcess
+meta.provider.container.cache.address=rm:2199
+meta.provider.container.cache.root=redis
+meta.provider.container.cache.baseport=17000
+
+#
+# Managed Cluster
+#
+managed.address=rm:2199
+
+managed.resources=devcache
+
+managed.resource.devcache.container=cache
+managed.resource.devcache.model=OnlineOffline
+managed.resource.devcache.partitions=10
+managed.resource.devcache.replica=1
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/conf/testng-integration.xml
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/conf/testng-integration.xml b/recipes/meta-cluster-manager/src/test/conf/testng-integration.xml
new file mode 100644
index 0000000..ed7d1c9
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/conf/testng-integration.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="false">
+ <packages>
+ <package name="org.apache.helix.metamanager.integration.*"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/conf/testng-unit.xml
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/conf/testng-unit.xml b/recipes/meta-cluster-manager/src/test/conf/testng-unit.xml
new file mode 100644
index 0000000..e178e4a
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/conf/testng-unit.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="false">
+ <packages>
+ <package name="org.apache.helix.metamanager.unit.*"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/conf/testng.xml
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/conf/testng.xml b/recipes/meta-cluster-manager/src/test/conf/testng.xml
new file mode 100644
index 0000000..8c3517f
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/conf/testng.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="false">
+ <packages>
+ <package name="org.apache.helix.*"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/config/testng-integration.xml
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/config/testng-integration.xml b/recipes/meta-cluster-manager/src/test/config/testng-integration.xml
new file mode 100644
index 0000000..ed7d1c9
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/config/testng-integration.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="false">
+ <packages>
+ <package name="org.apache.helix.metamanager.integration.*"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/config/testng-unit.xml
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/config/testng-unit.xml b/recipes/meta-cluster-manager/src/test/config/testng-unit.xml
new file mode 100644
index 0000000..e178e4a
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/config/testng-unit.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="false">
+ <packages>
+ <package name="org.apache.helix.metamanager.unit.*"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/config/testng.xml
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/config/testng.xml b/recipes/meta-cluster-manager/src/test/config/testng.xml
new file mode 100644
index 0000000..aa8f190
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/config/testng.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="true">
+ <packages>
+ <package name="org.apache.helix.metamanager.*"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/BootstrapperIT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/BootstrapperIT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/BootstrapperIT.java
new file mode 100644
index 0000000..7809bab
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/BootstrapperIT.java
@@ -0,0 +1,134 @@
+package org.apache.helix.metamanager;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.bootstrapper.Boot;
+import org.apache.helix.metamanager.bootstrapper.ClusterService;
+import org.apache.helix.metamanager.bootstrapper.ControllerService;
+import org.apache.helix.metamanager.bootstrapper.MetaClusterService;
+import org.apache.helix.metamanager.bootstrapper.MetaControllerService;
+import org.apache.helix.metamanager.bootstrapper.MetaProviderService;
+import org.apache.helix.metamanager.bootstrapper.MetaResourceService;
+import org.apache.helix.metamanager.bootstrapper.ResourceService;
+import org.apache.helix.metamanager.bootstrapper.ZookeeperService;
+import org.apache.log4j.Logger;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Bootstrapping tool test. Reads cluster configuration from *.properties files,
+ * spawns services and verifies number of active partitions and containers
+ *
+ * @see Boot
+ */
+@Test(groups = { "integration", "boot" })
+public class BootstrapperIT {
+
+ static final Logger log = Logger.getLogger(BootstrapperIT.class);
+
+ Boot boot;
+ HelixAdmin admin;
+
+ @AfterMethod(alwaysRun = true)
+ public void teardown() throws Exception {
+ log.debug("tearing down bootstrap test");
+ if (admin != null) {
+ admin.close();
+ admin = null;
+ }
+ if (boot != null) {
+ boot.stop();
+ boot = null;
+ }
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "local" })
+ public void bootstrapLocalTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("BootLocal.properties"));
+ boot.start();
+
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ZookeeperService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ClusterService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ResourceService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ControllerService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaClusterService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaResourceService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaProviderService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaControllerService.class));
+
+ final long limit = System.currentTimeMillis() + TestUtils.REBALANCE_TIMEOUT;
+
+ admin = new ZKHelixAdmin("localhost:2199");
+ waitUntil(admin, "meta", "container", 1, 7, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "cluster", "resource", 7, 10, (limit - System.currentTimeMillis()));
+
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "local" })
+ public void bootstrap2By2LocalTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("Boot2By2Local.properties"));
+ boot.start();
+
+ verify2By2Setup();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "shell" })
+ public void bootstrap2By2ShellTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("Boot2By2Shell.properties"));
+ boot.start();
+
+ verify2By2Setup();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "yarn" })
+ public void bootstrap2By2YarnTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("Boot2By2Yarn.properties"));
+ boot.start();
+
+ verify2By2Setup();
+ }
+
+ void verify2By2Setup() throws Exception {
+ final long limit = System.currentTimeMillis() + TestUtils.REBALANCE_TIMEOUT;
+ final String address = "localhost:2199";
+
+ log.debug(String.format("connecting to zookeeper at '%s'", address));
+
+ admin = new ZKHelixAdmin(address);
+ waitUntil(admin, "meta", "database", 2, 3, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "meta", "webserver", 2, 5, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "cluster", "dbprod", 3, 8, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "cluster", "wsprod", 5, 15, (limit - System.currentTimeMillis()));
+ }
+
+ static void waitUntil(HelixAdmin admin, String cluster, String resource, int instanceCount, int partitionCount, long timeout) throws Exception {
+ final long limit = System.currentTimeMillis() + timeout;
+ TestUtils.waitUntilInstanceCount(admin, cluster, resource, instanceCount, (limit - System.currentTimeMillis()));
+ TestUtils.waitUntilPartitionCount(admin, cluster, resource, partitionCount, (limit - System.currentTimeMillis()));
+ }
+
+ static Properties getProperties(String resourcePath) throws IOException {
+ Properties properties = new Properties();
+ properties.load(ClassLoader.getSystemResourceAsStream(resourcePath));
+ return properties;
+ }
+
+ static boolean containsInstanceOf(Collection<Service> services, Class<?> clazz) {
+ for (Service service : services) {
+ if (clazz.isAssignableFrom(service.getClass()))
+ return true;
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/FailoverIT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/FailoverIT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/FailoverIT.java
new file mode 100644
index 0000000..90e8be0
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/FailoverIT.java
@@ -0,0 +1,195 @@
+package org.apache.helix.metamanager;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.impl.StaticTargetProvider;
+import org.apache.helix.metamanager.impl.local.LocalContainerProviderProcess;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton;
+import org.apache.helix.metamanager.impl.local.LocalStatusProvider;
+import org.apache.helix.metamanager.impl.shell.ShellContainerProviderProcess;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton;
+import org.apache.helix.metamanager.impl.shell.ShellStatusProvider;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProcess;
+import org.apache.helix.metamanager.impl.yarn.YarnStatusProvider;
+import org.apache.helix.metamanager.impl.yarn.ZookeeperYarnDataProvider;
+import org.apache.helix.metamanager.provider.ProviderRebalancer;
+import org.apache.log4j.Logger;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Fault-recovery test for individual containers and whole providers. Missing
+ * containers should be replaced by the meta cluster Rebalancer using remaining
+ * active providers.
+ *
+ * @see ProviderRebalancer
+ */
+@Test(groups = { "integration", "failure" })
+public class FailoverIT {
+
+ static final Logger log = Logger.getLogger(FailoverIT.class);
+
+ static final int CONTAINER_COUNT = 7;
+
+ StaticTargetProvider targetProvider;
+ YarnStatusProvider yarnStatusProvider;
+
+ @BeforeClass(alwaysRun = true)
+ public void setupClass() {
+ log.info("installing shutdown hook");
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ teardownTest();
+ } catch (Exception ignore) {
+ }
+ ;
+ }
+ }));
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setupTest() throws Exception {
+ teardownTest();
+ targetProvider = new StaticTargetProvider(Collections.singletonMap(TestUtils.metaResourceName, CONTAINER_COUNT));
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void teardownTest() throws Exception {
+ TestUtils.stopTestCluster();
+
+ if (yarnStatusProvider != null) {
+ yarnStatusProvider.stop();
+ yarnStatusProvider = null;
+ }
+
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "local" })
+ public void testLocalContainerFailover() throws Exception {
+ log.info("testing local container failover");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ TestUtils.startTestCluster(targetProvider, new LocalStatusProvider(), makeLocalProviders(3));
+ killLocalContainers();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "local" })
+ public void testLocalProviderFailover() throws Exception {
+ log.info("testing local provider failover");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ TestUtils.startTestCluster(targetProvider, new LocalStatusProvider(), makeLocalProviders(3));
+ killProvider();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "shell" })
+ public void testShellContainerFailover() throws Exception {
+ log.info("testing shell container failover");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ TestUtils.startTestCluster(targetProvider, new ShellStatusProvider(), makeShellProviders(3));
+ killShellContainers();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "shell" })
+ public void testShellProviderFailover() throws Exception {
+ log.info("testing shell provider failover");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ TestUtils.startTestCluster(targetProvider, new ShellStatusProvider(), makeShellProviders(3));
+ killProvider();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "yarn" })
+ public void testYarnContainerFailover() throws Exception {
+ log.info("testing yarn container failover");
+ TestUtils.configure("distributed.properties");
+ TestUtils.startZookeeper();
+ yarnStatusProvider = new YarnStatusProvider(TestUtils.zkAddress);
+ yarnStatusProvider.start();
+ TestUtils.startTestCluster(targetProvider, yarnStatusProvider, makeYarnProviders(3));
+ killYarnContainers();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT, groups = { "yarn" })
+ public void testYarnProviderFailover() throws Exception {
+ log.info("testing yarn provider failover");
+ TestUtils.configure("distributed.properties");
+ TestUtils.startZookeeper();
+ yarnStatusProvider = new YarnStatusProvider(TestUtils.zkAddress);
+ yarnStatusProvider.start();
+ TestUtils.startTestCluster(targetProvider, yarnStatusProvider, makeYarnProviders(3));
+ killProvider();
+ }
+
+ void killLocalContainers() throws Exception {
+ LocalContainerSingleton.killProcess("container_2");
+ LocalContainerSingleton.killProcess("container_4");
+ LocalContainerSingleton.killProcess("container_6");
+ Thread.sleep(3000);
+ TestUtils.rebalanceTestCluster();
+ TestUtils.waitUntilRebalancedCount(CONTAINER_COUNT);
+ }
+
+ void killShellContainers() throws Exception {
+ ShellContainerSingleton.killProcess("container_2");
+ ShellContainerSingleton.killProcess("container_4");
+ ShellContainerSingleton.killProcess("container_6");
+ Thread.sleep(3000);
+ TestUtils.rebalanceTestCluster();
+ TestUtils.waitUntilRebalancedCount(CONTAINER_COUNT);
+ }
+
+ void killYarnContainers() throws Exception {
+ ZookeeperYarnDataProvider yarnDataService = new ZookeeperYarnDataProvider(TestUtils.zkAddress);
+ yarnDataService.start();
+ yarnDataService.delete("container_2");
+ yarnDataService.delete("container_4");
+ yarnDataService.delete("container_6");
+ yarnDataService.stop();
+ Thread.sleep(3000);
+ TestUtils.rebalanceTestCluster();
+ TestUtils.waitUntilRebalancedCount(CONTAINER_COUNT);
+ }
+
+ static void killProvider() throws Exception {
+ Iterator<Service> itService = TestUtils.providerServices.iterator();
+ itService.next().stop();
+ itService.remove();
+
+ TestUtils.rebalanceTestCluster();
+ TestUtils.waitUntilRebalancedCount(CONTAINER_COUNT);
+ }
+
+ LocalContainerProviderProcess[] makeLocalProviders(int count) throws Exception {
+ LocalContainerProviderProcess[] localProviders = new LocalContainerProviderProcess[count];
+ for (int i = 0; i < count; i++) {
+ localProviders[i] = TestUtils.makeLocalProvider("provider_" + i);
+ }
+ return localProviders;
+ }
+
+ ShellContainerProviderProcess[] makeShellProviders(int count) throws Exception {
+ ShellContainerProviderProcess[] shellProviders = new ShellContainerProviderProcess[count];
+ for (int i = 0; i < count; i++) {
+ shellProviders[i] = TestUtils.makeShellProvider("provider_" + i);
+ }
+ return shellProviders;
+ }
+
+ YarnContainerProviderProcess[] makeYarnProviders(int count) throws Exception {
+ YarnContainerProviderProcess[] yarnProviders = new YarnContainerProviderProcess[count];
+ for (int i = 0; i < count; i++) {
+ yarnProviders[i] = TestUtils.makeYarnProvider("provider_" + i);
+ }
+ return yarnProviders;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/LocalContainerProviderIT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/LocalContainerProviderIT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/LocalContainerProviderIT.java
new file mode 100644
index 0000000..e1faddd
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/LocalContainerProviderIT.java
@@ -0,0 +1,80 @@
+package org.apache.helix.metamanager;
+
+import java.util.Collections;
+
+import org.apache.helix.metamanager.impl.StaticTargetProvider;
+import org.apache.helix.metamanager.impl.local.LocalContainerProviderProcess;
+import org.apache.helix.metamanager.impl.local.LocalStatusProvider;
+import org.apache.log4j.Logger;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Local container provider and local status provider test. Scale-up and -down
+ * only, no failures.
+ *
+ * @see LocalContainerProvider
+ * @see LocalStatusProvider
+ */
+@Test(groups = { "integration", "local" })
+public class LocalContainerProviderIT {
+
+ static final Logger log = Logger.getLogger(LocalContainerProviderIT.class);
+
+ static final int CONTAINER_COUNT = 4;
+
+ StaticTargetProvider clusterStatusProvider;
+ LocalContainerProviderProcess containerProvider;
+ LocalStatusProvider containerStatusProvider;
+
+ @BeforeMethod(alwaysRun = true)
+ public void setupTest() throws Exception {
+ teardownTest();
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ containerProvider = TestUtils.makeLocalProvider("provider_0");
+ clusterStatusProvider = new StaticTargetProvider(Collections.singletonMap(TestUtils.metaResourceName, CONTAINER_COUNT));
+ containerStatusProvider = new LocalStatusProvider();
+ TestUtils.startTestCluster(clusterStatusProvider, containerStatusProvider, containerProvider);
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void teardownTest() throws Exception {
+ TestUtils.stopTestCluster();
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testStatic() throws Exception {
+ log.info("testing static");
+ setContainerCount(CONTAINER_COUNT);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleUp() throws Exception {
+ log.info("testing scale up");
+ setContainerCount(CONTAINER_COUNT + 2);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleDown() throws Exception {
+ log.info("testing scale down");
+ setContainerCount(CONTAINER_COUNT - 2);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleCycle() throws Exception {
+ log.info("testing scale cycle");
+ setContainerCount(CONTAINER_COUNT + 2);
+ setContainerCount(CONTAINER_COUNT);
+ setContainerCount(CONTAINER_COUNT - 2);
+ setContainerCount(CONTAINER_COUNT);
+ }
+
+ void setContainerCount(int newContainerCount) throws Exception {
+ log.debug(String.format("Setting container count to %d", newContainerCount));
+ clusterStatusProvider.setTargetContainerCount(TestUtils.metaResourceName, newContainerCount);
+ TestUtils.rebalanceTestCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/ShellContainerProviderIT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/ShellContainerProviderIT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/ShellContainerProviderIT.java
new file mode 100644
index 0000000..45b3023
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/ShellContainerProviderIT.java
@@ -0,0 +1,95 @@
+package org.apache.helix.metamanager;
+
+import java.util.Collections;
+
+import org.apache.helix.metamanager.impl.StaticTargetProvider;
+import org.apache.helix.metamanager.impl.shell.ShellContainerProviderProcess;
+import org.apache.helix.metamanager.impl.shell.ShellStatusProvider;
+import org.apache.log4j.Logger;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Shell container provider and shell status provider test. Scale-up and -down
+ * only, no failures.
+ *
+ * @see ShellContainerProvider
+ * @see ShellStatusProvider
+ */
+@Test(groups = { "integration", "shell" })
+public class ShellContainerProviderIT {
+
+ static final Logger log = Logger.getLogger(ShellContainerProviderIT.class);
+
+ static final long TEST_TIMEOUT = 20000;
+ static final long REBALANCE_TIMEOUT = 10000;
+
+ static final int CONTAINER_COUNT = 4;
+
+ StaticTargetProvider clusterStatusProvider;
+ ShellContainerProviderProcess containerProvider;
+ ShellStatusProvider containerStatusProvider;
+
+ @BeforeClass(alwaysRun = true)
+ public void setupClass() {
+ log.info("installing shutdown hook");
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try { teardownTest(); } catch(Exception ignore) {};
+ }
+ }));
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setupTest() throws Exception {
+ teardownTest();
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ containerProvider = TestUtils.makeShellProvider("provider_0");
+ clusterStatusProvider = new StaticTargetProvider(Collections.singletonMap(TestUtils.metaResourceName, CONTAINER_COUNT));
+ containerStatusProvider = new ShellStatusProvider();
+ TestUtils.startTestCluster(clusterStatusProvider, containerStatusProvider, containerProvider);
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void teardownTest() throws Exception {
+ TestUtils.stopTestCluster();
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testStatic() throws Exception {
+ log.info("testing static");
+ setContainerCount(CONTAINER_COUNT);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleUp() throws Exception {
+ log.info("testing scale up");
+ setContainerCount(CONTAINER_COUNT + 2);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleDown() throws Exception {
+ log.info("testing scale down");
+ setContainerCount(CONTAINER_COUNT - 2);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleCycle() throws Exception {
+ log.info("testing scale cycle");
+ setContainerCount(CONTAINER_COUNT + 2);
+ setContainerCount(CONTAINER_COUNT);
+ setContainerCount(CONTAINER_COUNT - 2);
+ setContainerCount(CONTAINER_COUNT);
+ }
+
+ void setContainerCount(int newContainerCount) throws Exception {
+ log.debug(String.format("Setting container count to %d", newContainerCount));
+ clusterStatusProvider.setTargetContainerCount(TestUtils.metaResourceName, newContainerCount);
+ TestUtils.rebalanceTestCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestContainerProvider.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestContainerProvider.java
new file mode 100644
index 0000000..d42a7b3
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestContainerProvider.java
@@ -0,0 +1,17 @@
+package org.apache.helix.metamanager;
+
+import org.apache.helix.metamanager.provider.local.LocalContainerProvider;
+import org.apache.helix.metamanager.provider.local.LocalContainerSingleton;
+
+public class TestContainerProvider extends LocalContainerProvider {
+
+ public TestContainerProvider(String providerName) {
+ super(TestUtils.zkAddress, TestUtils.managedClusterName, providerName);
+ }
+
+ public void destroyAll() {
+ super.destroyAll();
+ LocalContainerSingleton.reset();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestStatusProvider.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestStatusProvider.java
new file mode 100644
index 0000000..7e7f401
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestStatusProvider.java
@@ -0,0 +1,20 @@
+package org.apache.helix.metamanager;
+
+public class TestStatusProvider implements ClusterStatusProvider {
+
+ int targetContainerCount;
+
+ public TestStatusProvider(int targetContainerCount) {
+ this.targetContainerCount = targetContainerCount;
+ }
+
+ @Override
+ public int getTargetContainerCount(String type) {
+ return targetContainerCount;
+ }
+
+ public void setTargetContainerCount(int targetContainerCount) {
+ this.targetContainerCount = targetContainerCount;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtils.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtils.java
new file mode 100644
index 0000000..04587f0
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtils.java
@@ -0,0 +1,438 @@
+package org.apache.helix.metamanager;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import org.I0Itec.zkclient.IDefaultNameSpace;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.ZkServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixManager;
+import org.apache.helix.controller.HelixControllerMain;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.impl.local.LocalContainerProviderProcess;
+import org.apache.helix.metamanager.impl.shell.ShellContainerProviderProcess;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProcess;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProperties;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+import org.apache.helix.metamanager.provider.ProviderRebalancer;
+import org.apache.helix.metamanager.provider.ProviderRebalancerSingleton;
+import org.apache.helix.model.ExternalView;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.StateModelDefinition;
+import org.apache.helix.tools.StateModelConfigGenerator;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+/**
+ * Utility for creating a test cluster without the bootstrapping tool. Methods
+ * for verifying the number of active instances and partitions in a cluster.
+ *
+ */
+public class TestUtils {
+
+ static final Logger log = Logger.getLogger(TestUtils.class);
+
+ public static int zkPort;
+ public static String zkAddress;
+ public static String resmanAddress;
+ public static String schedulerAddress;
+ public static String hdfsAddress;
+ public static String yarnUser;
+
+ public static final String metaClusterName = "meta-cluster";
+ public static final String managedClusterName = "managed-cluster";
+ public static final String metaResourceName = "container";
+ public static final String managedResourceName = "database";
+
+ public static final int numManagedPartitions = 10;
+ public static final int numManagedReplica = 2;
+
+ public static final long TEST_TIMEOUT = 120000;
+ public static final long REBALANCE_TIMEOUT = 60000;
+ public static final long POLL_INTERVAL = 1000;
+
+ public static final ProviderProperties providerProperties = new ProviderProperties();
+
+ public static ZkServer server = null;
+ public static HelixAdmin admin = null;
+ public static HelixManager metaControllerManager = null;
+ public static HelixManager managedControllerManager = null;
+
+ public static Collection<Service> providerServices = new ArrayList<Service>();
+ public static Collection<Service> auxServices = new ArrayList<Service>();
+
+ public static TargetProvider targetProvider = null;
+ public static StatusProvider statusProvider = null;
+
+ static {
+ try {
+ configure();
+ } catch(Exception e) {
+ log.error("Could not setup TestUtils", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private TestUtils() {
+ // left blank
+ }
+
+ public static void configure() throws IOException {
+ configure("standalone.properties");
+ }
+
+ public static void configure(String resourcePath) throws IOException {
+ log.info(String.format("Configuring Test cluster from %s", resourcePath));
+ Properties properties = new Properties();
+ properties.load(ClassLoader.getSystemResourceAsStream(resourcePath));
+ configure(properties);
+ }
+
+ public static void configure(Properties properties) {
+ log.info(String.format("Configuring from properties '%s'", properties));
+
+ zkPort = Integer.valueOf(properties.getProperty("zookeeper.port"));
+ zkAddress = properties.getProperty("zookeeper.address");
+ resmanAddress = properties.getProperty("yarn.resourcemanager");
+ schedulerAddress = properties.getProperty("yarn.scheduler");
+ hdfsAddress = properties.getProperty("yarn.hdfs");
+ yarnUser = properties.getProperty("yarn.user");
+
+ Preconditions.checkNotNull(zkPort);
+ Preconditions.checkNotNull(zkAddress);
+ Preconditions.checkNotNull(resmanAddress);
+ Preconditions.checkNotNull(schedulerAddress);
+ Preconditions.checkNotNull(hdfsAddress);
+ Preconditions.checkNotNull(yarnUser);
+
+ configureInternal();
+ }
+
+ static void configureInternal() {
+ providerProperties.clear();
+ providerProperties.setProperty(ProviderProperties.ADDRESS, zkAddress);
+ providerProperties.setProperty(ProviderProperties.CLUSTER, managedClusterName);
+ providerProperties.setProperty(ProviderProperties.METAADDRESS, zkAddress);
+ providerProperties.setProperty(ProviderProperties.METACLUSTER, metaClusterName);
+ providerProperties.setProperty(ProviderProperties.NAME, "<unknown>");
+
+ Properties containerProperties = new Properties();
+ containerProperties.setProperty("class", "org.apache.helix.metamanager.impl.container.DummyMasterSlaveProcess");
+
+ providerProperties.addContainer("container", containerProperties);
+
+ log.info(String.format("Using provider properties '%s'", providerProperties));
+ }
+
+ public static void startZookeeper() throws Exception {
+ log.info("Starting ZooKeeper");
+
+ if (server != null)
+ throw new IllegalStateException("Zookeeper already running");
+
+ server = createLocalZookeeper();
+ server.start();
+ }
+
+ public static void stopZookeeper() throws Exception {
+ log.info("Stopping ZooKeeper");
+
+ if (server != null) {
+ server.shutdown();
+ server = null;
+ }
+ }
+
+ public static void startTestCluster(TargetProviderService targetProvider, StatusProviderService statusProvider, Service... containerProviderProcesses)
+ throws Exception {
+ log.debug(String.format("Starting test cluster"));
+
+ if (server == null)
+ throw new IllegalStateException("Zookeeper not running yet");
+
+ if (!auxServices.isEmpty() || !providerServices.isEmpty() || admin != null || metaControllerManager != null || managedControllerManager != null)
+ throw new IllegalStateException("TestCluster already running");
+
+ log.debug("Create admin");
+ admin = new ZKHelixAdmin(zkAddress);
+
+ log.debug("Create clusters");
+ admin.addCluster(metaClusterName, true);
+ admin.addCluster(managedClusterName, true);
+
+ log.debug("Setup config tool");
+ ProviderRebalancerSingleton.setTargetProvider(targetProvider);
+ ProviderRebalancerSingleton.setStatusProvider(statusProvider);
+
+ log.debug("Starting target and status provider");
+ TestUtils.targetProvider = startAuxService(targetProvider);
+ TestUtils.statusProvider = startAuxService(statusProvider);
+
+ // Managed Cluster
+ log.debug("Setup managed cluster");
+ admin.addStateModelDef(managedClusterName, "MasterSlave", new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave()));
+ admin.addResource(managedClusterName, managedResourceName, numManagedPartitions, "MasterSlave", RebalanceMode.FULL_AUTO.toString());
+ IdealState managedIdealState = admin.getResourceIdealState(managedClusterName, managedResourceName);
+ managedIdealState.setInstanceGroupTag(metaResourceName);
+ managedIdealState.setReplicas(String.valueOf(numManagedReplica));
+ admin.setResourceIdealState(managedClusterName, managedResourceName, managedIdealState);
+
+ // Meta Cluster
+ log.debug("Setup meta cluster");
+ admin.addStateModelDef(metaClusterName, "OnlineOffline", new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));
+ admin.addResource(metaClusterName, metaResourceName, targetProvider.getTargetContainerCount(metaResourceName), "OnlineOffline",
+ RebalanceMode.USER_DEFINED.toString());
+
+ IdealState idealState = admin.getResourceIdealState(metaClusterName, metaResourceName);
+ idealState.setRebalancerClassName(ProviderRebalancer.class.getName());
+ idealState.setReplicas("1");
+
+ // BEGIN workaround
+ // FIXME workaround for HELIX-226
+ Map<String, List<String>> listFields = Maps.newHashMap();
+ Map<String, Map<String, String>> mapFields = Maps.newHashMap();
+ for (int i = 0; i < 256; i++) {
+ String partitionName = metaResourceName + "_" + i;
+ listFields.put(partitionName, new ArrayList<String>());
+ mapFields.put(partitionName, new HashMap<String, String>());
+ }
+ idealState.getRecord().setListFields(listFields);
+ idealState.getRecord().setMapFields(mapFields);
+ // END workaround
+
+ admin.setResourceIdealState(metaClusterName, metaResourceName, idealState);
+
+ log.debug("Starting container providers");
+ for (Service service : containerProviderProcesses) {
+ startProviderService(service);
+ }
+
+ log.debug("Starting managed cluster controller");
+ managedControllerManager = HelixControllerMain.startHelixController(zkAddress, managedClusterName, "managedController", HelixControllerMain.STANDALONE);
+
+ log.debug("Starting meta cluster controller");
+ metaControllerManager = HelixControllerMain.startHelixController(zkAddress, metaClusterName, "metaController", HelixControllerMain.STANDALONE);
+
+ log.debug("Waiting for stable state");
+ waitUntilRebalancedCount(targetProvider.getTargetContainerCount(metaResourceName));
+ }
+
+ public static void stopTestCluster() throws Exception {
+ log.debug(String.format("Stopping test cluster"));
+ if (managedControllerManager != null) {
+ log.info("Disconnecting managed cluster controller");
+ managedControllerManager.disconnect();
+ }
+ if (metaControllerManager != null) {
+ log.info("Disconnecting meta cluster controller");
+ metaControllerManager.disconnect();
+ }
+ log.info("Stopping provider services");
+ if (providerServices != null) {
+ for (Service service : providerServices) {
+ service.stop();
+ }
+ providerServices.clear();
+ }
+ log.debug("Stopping auxillary services");
+ if (auxServices != null) {
+ for (Service service : auxServices) {
+ service.stop();
+ }
+ auxServices.clear();
+ }
+
+ admin = null;
+ metaControllerManager = null;
+ managedControllerManager = null;
+ }
+
+ public static <T extends Service> T startAuxService(T service) throws Exception {
+ auxServices.add(service);
+ service.start();
+ return service;
+ }
+
+ public static <T extends Service> T startProviderService(T service) throws Exception {
+ providerServices.add(service);
+ service.start();
+ return service;
+ }
+
+ public static void rebalanceTestCluster() throws Exception {
+ log.debug(String.format("Triggering rebalance"));
+ IdealState poke = admin.getResourceIdealState(metaClusterName, metaResourceName);
+ admin.setResourceIdealState(metaClusterName, metaResourceName, poke);
+
+ int current = targetProvider.getTargetContainerCount(TestUtils.metaResourceName);
+ waitUntilRebalancedCount(current);
+ }
+
+ public static void waitUntilRebalancedCount(int containerCount) throws Exception {
+ log.debug(String.format("Waiting for rebalance with %d containers at '%s'", containerCount, zkAddress));
+
+ HelixAdmin admin = new ZKHelixAdmin(zkAddress);
+
+ try {
+ long limit = System.currentTimeMillis() + REBALANCE_TIMEOUT;
+ waitUntilPartitionCount(admin, metaClusterName, metaResourceName, containerCount, (limit - System.currentTimeMillis()));
+ waitUntilInstanceCount(admin, metaClusterName, metaResourceName, providerServices.size(), (limit - System.currentTimeMillis()));
+ waitUntilPartitionCount(admin, managedClusterName, managedResourceName, numManagedPartitions, (limit - System.currentTimeMillis()));
+
+ // FIXME workaround for Helix FULL_AUTO rebalancer not providing guarantees for cluster expansion
+ //waitUntilInstanceCount(admin, managedClusterName, managedResourceName, containerCount, (limit - System.currentTimeMillis()));
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ admin.close();
+ }
+ }
+
+ public static void waitUntilInstanceCount(HelixAdmin admin, String cluster, String resource, int targetCount, long timeout) throws Exception {
+ log.debug(String.format("Waiting for instance count (cluster='%s', resource='%s', instanceCount=%d, timeout=%d)", cluster, resource, targetCount,
+ timeout));
+
+ long limit = System.currentTimeMillis() + timeout;
+ while (limit > System.currentTimeMillis()) {
+ int assignedCount = getAssingedInstances(admin, cluster, resource).size();
+ log.debug(String.format("checking instance count for '%s:%s': target=%d, current=%d", cluster, resource, targetCount, assignedCount));
+
+ if (targetCount == assignedCount) {
+ return;
+ }
+ Thread.sleep(POLL_INTERVAL);
+ }
+ throw new TimeoutException();
+ }
+
+ public static void waitUntilPartitionCount(HelixAdmin admin, String cluster, String resource, int targetCount, long timeout) throws Exception {
+ log.debug(String.format("Waiting for partition count (cluster='%s', resource='%s', partitionCount=%d, timeout=%d)", cluster, resource, targetCount,
+ timeout));
+
+ long limit = System.currentTimeMillis() + timeout;
+ while (limit > System.currentTimeMillis()) {
+ int assignedCount = getAssingedPartitions(admin, cluster, resource).size();
+ log.debug(String.format("checking partition count for '%s:%s': target=%d, current=%d", cluster, resource, targetCount, assignedCount));
+
+ if (targetCount == assignedCount) {
+ return;
+ }
+ Thread.sleep(POLL_INTERVAL);
+ }
+ throw new TimeoutException();
+ }
+
+ public static Set<String> getAssingedInstances(HelixAdmin admin, String clusterName, String resourceName) {
+ Set<String> assignedInstances = new HashSet<String>();
+
+ ExternalView externalView = admin.getResourceExternalView(clusterName, resourceName);
+
+ if (externalView == null)
+ return assignedInstances;
+
+ for (String partitionName : externalView.getPartitionSet()) {
+ Map<String, String> stateMap = externalView.getStateMap(partitionName);
+ if (stateMap == null)
+ continue;
+
+ for (String instanceName : stateMap.keySet()) {
+ String state = stateMap.get(instanceName);
+ if ("MASTER".equals(state) || "SLAVE".equals(state) || "ONLINE".equals(state)) {
+ assignedInstances.add(instanceName);
+ }
+ }
+ }
+
+ return assignedInstances;
+ }
+
+ public static Set<String> getAssingedPartitions(HelixAdmin admin, String clusterName, String resourceName) {
+ Set<String> assignedPartitions = new HashSet<String>();
+
+ ExternalView externalView = admin.getResourceExternalView(clusterName, resourceName);
+
+ if (externalView == null)
+ return assignedPartitions;
+
+ for (String partitionName : externalView.getPartitionSet()) {
+ Map<String, String> stateMap = externalView.getStateMap(partitionName);
+ if (stateMap == null)
+ continue;
+
+ for (String instanceName : stateMap.keySet()) {
+ String state = stateMap.get(instanceName);
+ if ("MASTER".equals(state) || "ONLINE".equals(state)) {
+ assignedPartitions.add(partitionName);
+ }
+ }
+ }
+
+ return assignedPartitions;
+ }
+
+ public static ZkServer createLocalZookeeper() throws Exception {
+ String baseDir = "/tmp/metamanager/";
+ final String dataDir = baseDir + "zk/dataDir";
+ final String logDir = baseDir + "zk/logDir";
+ FileUtils.deleteDirectory(new File(dataDir));
+ FileUtils.deleteDirectory(new File(logDir));
+
+ IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() {
+ @Override
+ public void createDefaultNameSpace(ZkClient zkClient) {
+
+ }
+ };
+ return new ZkServer(dataDir, logDir, defaultNameSpace, zkPort);
+ }
+
+ public static LocalContainerProviderProcess makeLocalProvider(String name) throws Exception {
+ LocalContainerProviderProcess process = new LocalContainerProviderProcess();
+ process.configure(makeProviderProperties(name));
+ return process;
+ }
+
+ public static ShellContainerProviderProcess makeShellProvider(String name) throws Exception {
+ ShellContainerProviderProcess process = new ShellContainerProviderProcess();
+ process.configure(makeProviderProperties(name));
+ return process;
+ }
+
+ public static YarnContainerProviderProcess makeYarnProvider(String name) throws Exception {
+ YarnContainerProviderProperties properties = new YarnContainerProviderProperties();
+
+ properties.putAll(makeProviderProperties(name));
+ properties.put(YarnContainerProviderProperties.YARNDATA, zkAddress);
+ properties.put(YarnContainerProviderProperties.RESOURCEMANAGER, resmanAddress);
+ properties.put(YarnContainerProviderProperties.SCHEDULER, schedulerAddress);
+ properties.put(YarnContainerProviderProperties.USER, yarnUser);
+ properties.put(YarnContainerProviderProperties.HDFS, hdfsAddress);
+
+ YarnContainerProviderProcess process = new YarnContainerProviderProcess();
+ process.configure(properties);
+
+ return process;
+ }
+
+ static ProviderProperties makeProviderProperties(String name) {
+ ProviderProperties properties = new ProviderProperties();
+ properties.putAll(providerProperties);
+ properties.setProperty(ProviderProperties.NAME, name);
+ return properties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsTest.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsTest.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsTest.java
new file mode 100644
index 0000000..3f0bd3e
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsTest.java
@@ -0,0 +1,30 @@
+package org.apache.helix.metamanager;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.testng.annotations.Test;
+
+public class TestUtilsTest {
+
+ @Test
+ public void testStartStop() throws Exception {
+ TestUtils.startTestCluster(new TestStatusProvider(1),
+ Collections.<ClusterContainerProvider>singletonList(new TestContainerProvider("test")));
+ TestUtils.stopTestCluster();
+ }
+
+ @Test
+ public void testStartStopRepeated() throws Exception {
+ ClusterStatusProvider statusProvider = new TestStatusProvider(1);
+ List<ClusterContainerProvider> containerProviders = Collections.<ClusterContainerProvider>singletonList(new TestContainerProvider("test"));
+
+ TestUtils.startTestCluster(statusProvider, containerProviders);
+ TestUtils.stopTestCluster();
+
+ TestUtils.startTestCluster(statusProvider, containerProviders);
+ TestUtils.stopTestCluster();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsUT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsUT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsUT.java
new file mode 100644
index 0000000..50d7121
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/TestUtilsUT.java
@@ -0,0 +1,63 @@
+package org.apache.helix.metamanager;
+
+import java.util.Collections;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.TargetProviderService;
+import org.apache.helix.metamanager.impl.StaticTargetProvider;
+import org.apache.helix.metamanager.impl.local.LocalStatusProvider;
+import org.apache.log4j.Logger;
+import org.testng.annotations.Test;
+
+/**
+ * Self-test of test cluster. Spawning zookeeper and cluster with single provider and single instance.
+ *
+ * @see TestUtils
+ */
+@Test(groups={"unit"})
+public class TestUtilsUT {
+
+ static final Logger log = Logger.getLogger(TestUtilsUT.class);
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testZookeeper() throws Exception {
+ log.info("testing zookeeper");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testCluster() throws Exception {
+ log.info("testing cluster");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+
+ TestUtils.startTestCluster(new StaticTargetProvider(Collections.singletonMap(TestUtils.metaResourceName, 1)),
+ new LocalStatusProvider(), TestUtils.makeLocalProvider("test"));
+ TestUtils.stopTestCluster();
+
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testClusterRepeated() throws Exception {
+ log.info("testing cluster restart");
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+
+ TargetProviderService statusProvider = new StaticTargetProvider(Collections.singletonMap(TestUtils.metaResourceName, 1));
+ Service containerProvider = TestUtils.makeLocalProvider("test");
+ StatusProviderService containerStatusProvider = new LocalStatusProvider();
+
+ TestUtils.startTestCluster(statusProvider, containerStatusProvider, containerProvider);
+ TestUtils.stopTestCluster();
+
+ TestUtils.startTestCluster(statusProvider, containerStatusProvider, containerProvider);
+ TestUtils.stopTestCluster();
+
+ TestUtils.stopZookeeper();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/YarnContainerProviderIT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/YarnContainerProviderIT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/YarnContainerProviderIT.java
new file mode 100644
index 0000000..5d319ff
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/YarnContainerProviderIT.java
@@ -0,0 +1,101 @@
+package org.apache.helix.metamanager;
+
+import java.util.Collections;
+
+import org.apache.helix.metamanager.impl.StaticTargetProvider;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProcess;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProperties;
+import org.apache.helix.metamanager.impl.yarn.YarnStatusProvider;
+import org.apache.log4j.Logger;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Yarn container provider and yarn status provider test. Scale-up and -down
+ * only, no failures.
+ *
+ * @see YarnContainerProvider
+ * @see YarnStatusProvider
+ */
+@Test(groups={"integration", "yarn"})
+public class YarnContainerProviderIT {
+
+ static final Logger log = Logger.getLogger(YarnContainerProviderIT.class);
+
+ static final int CONTAINER_COUNT = 4;
+
+ StaticTargetProvider clusterStatusProvider;
+ YarnContainerProviderProcess containerProvider;
+ YarnStatusProvider containerStatusProvider;
+
+ YarnContainerProviderProperties properties;
+
+ @BeforeClass(alwaysRun = true)
+ public void setupClass() throws Exception {
+ log.info("installing shutdown hook");
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try { teardownTest(); } catch(Exception ignore) {};
+ }
+ }));
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setupTest() throws Exception {
+ log.debug("setting up yarn test case");
+
+ teardownTest();
+ TestUtils.configure("distributed.properties");
+ TestUtils.startZookeeper();
+
+ containerProvider = TestUtils.makeYarnProvider("provider_0");
+ containerStatusProvider = new YarnStatusProvider(TestUtils.zkAddress);
+ clusterStatusProvider = new StaticTargetProvider(Collections.singletonMap(TestUtils.metaResourceName, CONTAINER_COUNT));
+ TestUtils.startTestCluster(clusterStatusProvider, containerStatusProvider, containerProvider);
+
+ log.debug("running yarn test case");
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void teardownTest() throws Exception {
+ log.debug("cleaning up yarn test case");
+ TestUtils.stopTestCluster();
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testStatic() throws Exception {
+ log.info("testing static");
+ setContainerCount(CONTAINER_COUNT);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleUp() throws Exception {
+ log.info("testing scale up");
+ setContainerCount(CONTAINER_COUNT + 2);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleDown() throws Exception {
+ log.info("testing scale down");
+ setContainerCount(CONTAINER_COUNT - 2);
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testScaleCycle() throws Exception {
+ log.info("testing scale cycle");
+ setContainerCount(CONTAINER_COUNT + 2);
+ setContainerCount(CONTAINER_COUNT);
+ setContainerCount(CONTAINER_COUNT - 2);
+ setContainerCount(CONTAINER_COUNT);
+ }
+
+ void setContainerCount(int newContainerCount) throws Exception {
+ log.debug(String.format("Setting container count to %d", newContainerCount));
+ clusterStatusProvider.setTargetContainerCount(TestUtils.metaResourceName, newContainerCount);
+ TestUtils.rebalanceTestCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/integration/BootstrapperIT.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/integration/BootstrapperIT.java b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/integration/BootstrapperIT.java
new file mode 100644
index 0000000..a7bae00
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/test/java/org/apache/helix/metamanager/integration/BootstrapperIT.java
@@ -0,0 +1,127 @@
+package org.apache.helix.metamanager.integration;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.TestUtils;
+import org.apache.helix.metamanager.bootstrapper.Boot;
+import org.apache.helix.metamanager.bootstrapper.ClusterService;
+import org.apache.helix.metamanager.bootstrapper.ControllerService;
+import org.apache.helix.metamanager.bootstrapper.MetaClusterService;
+import org.apache.helix.metamanager.bootstrapper.MetaControllerService;
+import org.apache.helix.metamanager.bootstrapper.MetaProviderService;
+import org.apache.helix.metamanager.bootstrapper.MetaResourceService;
+import org.apache.helix.metamanager.bootstrapper.ResourceService;
+import org.apache.helix.metamanager.bootstrapper.ZookeeperService;
+import org.apache.log4j.Logger;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+public class BootstrapperIT {
+
+ static final Logger log = Logger.getLogger(BootstrapperIT.class);
+
+ Boot boot;
+ HelixAdmin admin;
+
+ @AfterMethod
+ public void teardown() throws Exception {
+ log.debug("tearing down bootstrap test");
+ if(admin != null) {
+ admin.close();
+ admin = null;
+ }
+ if (boot != null) {
+ boot.stop();
+ boot = null;
+ }
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void bootstrapLocalTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("BootLocal.properties"));
+ boot.start();
+
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ZookeeperService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ClusterService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ResourceService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), ControllerService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaClusterService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaResourceService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaProviderService.class));
+ Assert.assertTrue(containsInstanceOf(boot.getServcies(), MetaControllerService.class));
+
+ final long limit = System.currentTimeMillis() + TestUtils.REBALANCE_TIMEOUT;
+
+ admin = new ZKHelixAdmin("localhost:2199");
+ waitUntil(admin, "meta", "container", 1, 7, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "cluster", "resource", 7, 10, (limit - System.currentTimeMillis()));
+
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void bootstrap2By2LocalTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("Boot2By2Local.properties"));
+ boot.start();
+
+ verify2By2Setup();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void bootstrap2By2ShellTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("Boot2By2Shell.properties"));
+ boot.start();
+
+ verify2By2Setup();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void bootstrap2By2YarnTest() throws Exception {
+ boot = new Boot();
+ boot.configure(getProperties("Boot2By2Yarn.properties"));
+ boot.start();
+
+ verify2By2Setup();
+ }
+
+ void verify2By2Setup() throws Exception {
+ final long limit = System.currentTimeMillis() + TestUtils.REBALANCE_TIMEOUT;
+ final String address = "localhost:2199";
+
+ log.debug(String.format("connecting to zookeeper at '%s'", address));
+
+ admin = new ZKHelixAdmin(address);
+ waitUntil(admin, "meta", "database", 2, 3, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "meta", "webserver", 2, 5, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "cluster", "dbprod", 3, 8, (limit - System.currentTimeMillis()));
+ waitUntil(admin, "cluster", "wsprod", 5, 15, (limit - System.currentTimeMillis()));
+ }
+
+ static void waitUntil(HelixAdmin admin, String cluster, String resource, int instanceCount, int partitionCount, long timeout) throws Exception {
+ final long limit = System.currentTimeMillis() + timeout;
+ TestUtils.waitUntilInstanceCount(admin, cluster, resource, instanceCount, (limit - System.currentTimeMillis()));
+ TestUtils.waitUntilPartitionCount(admin, cluster, resource, partitionCount, (limit - System.currentTimeMillis()));
+ }
+
+ static Properties getProperties(String resourcePath) throws IOException {
+ Properties properties = new Properties();
+ properties.load(ClassLoader.getSystemResourceAsStream(resourcePath));
+ return properties;
+ }
+
+ static boolean containsInstanceOf(Collection<Service> services, Class<?> clazz) {
+ for(Service service : services) {
+ if(clazz.isAssignableFrom(service.getClass())) return true;
+ }
+ return false;
+ }
+
+}