You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/08/09 00:50:46 UTC
[11/13] git commit: SLIDER-288 Add role for HBase thrift2 gateway
SLIDER-288 Add role for HBase thrift2 gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6f912df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6f912df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6f912df2
Branch: refs/heads/feature/SLIDER-285_Restart_AM
Commit: 6f912df214136a8cd903b0c5d17ee9d680e6cffc
Parents: a1bbf4d
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:24:28 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:24:28 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/links.py | 20 +++++++
app-packages/hbase/metainfo.xml | 15 +++++
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift2.py | 62 ++++++++++++++++++++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 9 ++-
.../providers/hbase/HBaseClientProvider.java | 4 ++
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 2 +
11 files changed, 124 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index f0ab144..24d9a84 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -24,6 +24,7 @@
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/links.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/links.py b/app-packages/hbase/links.py
index c77e620..19b0d91 100644
--- a/app-packages/hbase/links.py
+++ b/app-packages/hbase/links.py
@@ -1,3 +1,23 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
from __future__ import print_function
import logging
import json
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index ee6ddea..a5dab0c 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
<value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
</export>
<export>
+ <name>org.apache.slider.hbase.thrift2</name>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ </export>
+ <export>
<name>org.apache.slider.hbase.thrift</name>
<value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
</export>
@@ -68,6 +72,7 @@
<command>HBASE_MASTER-START</command>
<requires>HBASE_REST-INSTALLED</requires>
<requires>HBASE_THRIFT-INSTALLED</requires>
+ <requires>HBASE_THRIFT2-INSTALLED</requires>
</commandOrder>
</commandOrders>
<components>
@@ -114,6 +119,16 @@
</component>
<component>
+ <name>HBASE_THRIFT2</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_thrift2.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
<minInstanceCount>0</minInstanceCount>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index 01d38b3..96add84 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -39,6 +39,8 @@ def hbase_service(
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
elif name == 'thrift':
daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
+ elif name == 'thrift2':
+ daemon_cmd = format("{daemon_cmd} -p {thrift2_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_thrift2.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_thrift2.py b/app-packages/hbase/package/scripts/hbase_thrift2.py
new file mode 100644
index 0000000..b72196c
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift2.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift2(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift2')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift2',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift2',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift2().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index 2d05289..1f25f68 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -59,6 +59,7 @@ ganglia_server_port = default('/configurations/global/ganglia_server_port', '866
rest_port = config['configurations']['global']['hbase_rest_port']
thrift_port = config['configurations']['global']['hbase_thrift_port']
+thrift2_port = config['configurations']['global']['hbase_thrift2_port']
if security_enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
index 2d2b4f9..d2fdbd8 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -21,9 +21,14 @@
"yarn.role.priority": "3",
"yarn.component.instances": "1",
"yarn.memory": "256"
- }
+ },
"HBASE_THRIFT": {
- "yarn.role.priority": "3",
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT2": {
+ "yarn.role.priority": "5",
"yarn.component.instances": "1",
"yarn.memory": "256"
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index fde2bf7..cd104a2 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -239,8 +239,12 @@ public class HBaseClientProvider extends AbstractClientProvider implements
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER,
0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY,
+ 0, -1);
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY,
0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT2_GATEWAY,
+ 0, -1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index 9c38fb2..2a20438 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -30,6 +30,8 @@ public interface HBaseKeys {
String ROLE_THRIFT_GATEWAY = "THRIFT";
+ String ROLE_THRIFT2_GATEWAY = "THRIFT2";
+
/** {@value */
String REGION_SERVER = "regionserver";
@@ -38,6 +40,9 @@ public interface HBaseKeys {
/** {@value */
String THRIFT_GATEWAY = "thrift";
+ /** {@value */
+ String THRIFT2_GATEWAY = "thrift2";
+ /**
/**
* What is the command for hbase to print a version: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 953ee79..f72967f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -201,6 +201,11 @@ public class HBaseProviderService extends AbstractProviderService implements
logfile = "/thrift-gateway.txt";
break;
+ case ROLE_THRIFT2_GATEWAY:
+ roleCommand = THRIFT2_GATEWAY;
+
+ logfile = "/thrift2-gateway.txt";
+ break;
default:
throw new SliderInternalStateException("Cannot start role %s", role);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
index 2cc457b..b2825b8 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -41,6 +41,7 @@ public class HBaseRoles {
public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4;
+ public static final int KEY_THRIFT2_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 5;
/**
* Initialize role list
*/
@@ -49,6 +50,7 @@ public class HBaseRoles {
ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY));
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT2_GATEWAY));
}