You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/11/05 18:00:46 UTC
git commit: SLIDER-525. Storm app package for windows does not report
component status correctly HBase Win package has the ability to provide
custom ports for Thrift and Thrift2
Repository: incubator-slider
Updated Branches:
refs/heads/develop 0fe9f5711 -> 66b1aa4a4
SLIDER-525. Storm app package for windows does not report component status correctly
HBase Win package has the ability to provide custom ports for Thrift and Thrift2
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66b1aa4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66b1aa4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66b1aa4a
Branch: refs/heads/develop
Commit: 66b1aa4a403bfe60c026b63f201f8cdc539f2dfe
Parents: 0fe9f57
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Nov 5 09:00:06 2014 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Nov 5 09:00:10 2014 -0800
----------------------------------------------------------------------
app-packages/hbase-win/README.txt | 2 +-
.../hbase-win/package/scripts/hbase_service.py | 13 +++++++++
.../hbase-win/package/scripts/params.py | 10 ++++---
app-packages/hbase-win/resources-default.json | 14 ++++-----
.../storm-win/package/scripts/nimbus.py | 2 +-
.../storm-win/package/scripts/rest_api.py | 2 +-
.../storm-win/package/scripts/service.py | 30 ++++++++++++++++----
.../storm-win/package/scripts/supervisor.py | 2 +-
.../storm-win/package/scripts/ui_server.py | 2 +-
.../src/main/python/agent/Controller.py | 2 +-
10 files changed, 56 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
index a413e76..6389fb2 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -30,7 +30,7 @@ mvn clean package -Phbase-app-package-win -Dpkg.version=<version>
-Dpkg.name=<file name of app zip file> -Dpkg.src=<folder location where the pkg is available>
Example:
-mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
+mvn clean package -Phbase-app-package-win -Dpkg.version=0.98.5-hadoop2
-Dpkg.name=hbase-0.98.5-hadoop2-bin.zip
-Dpkg.src=/Users/user1/Downloads
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py
index 9a8dc56..e269531 100644
--- a/app-packages/hbase-win/package/scripts/hbase_service.py
+++ b/app-packages/hbase-win/package/scripts/hbase_service.py
@@ -28,6 +28,8 @@ def hbase_service(
import params
pid_file = format("{pid_dir}/hbase-{hbase_user}-{name}.pid")
+ custom_port = None
+ custom_info_port = None
heap_size = params.master_heapsize
main_class = "org.apache.hadoop.hbase.master.HMaster"
if name == "regionserver":
@@ -36,12 +38,17 @@ def hbase_service(
if name == "rest":
heap_size = params.restserver_heapsize
main_class = "org.apache.hadoop.hbase.rest.RESTServer"
+ custom_port = params.rest_port
if name == "thrift":
heap_size = params.thriftserver_heapsize
main_class = "org.apache.hadoop.hbase.thrift.ThriftServer"
+ custom_port = params.thrift_port
+ custom_info_port = params.thrift_info_port
if name == "thrift2":
heap_size = params.thrift2server_heapsize
main_class = "org.apache.hadoop.hbase.thrift2.ThriftServer"
+ custom_port = params.thrift2_port
+ custom_info_port = params.thrift2_info_port
role_user = format("{hbase_user}-{name}")
@@ -49,6 +56,12 @@ def hbase_service(
process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}")
+ if custom_port:
+ process_cmd = format("{process_cmd} -p {custom_port}")
+
+ if custom_info_port:
+ process_cmd = format("{process_cmd} --infoport {custom_info_port}")
+
Execute(process_cmd,
logoutput=False,
wait_for_finish=False,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py
index 4f55cd0..90d274c 100644
--- a/app-packages/hbase-win/package/scripts/params.py
+++ b/app-packages/hbase-win/package/scripts/params.py
@@ -42,11 +42,13 @@ input_conf_files_dir = config['configurations']['global']['app_input_conf_dir']
hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
"""
-Read various ports, unused as of now
+Read various ports
"""
-rest_port = config['configurations']['global']['hbase_rest_port']
-thrift_port = config['configurations']['global']['hbase_thrift_port']
-thrift2_port = config['configurations']['global']['hbase_thrift2_port']
+rest_port = default("configurations/global/hbase_rest_port", 1700)
+thrift_port = default("configurations/global/hbase_thrift_port", 9090)
+thrift2_port = default("configurations/global/hbase_thrift2_port", 9091)
+thrift_info_port = default("configurations/global/hbase_info_thrift_port", 9095)
+thrift2_info_port = default("configurations/global/hbase_info_thrift2_port", 9096)
"""
Compute or read various heap sizes
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/hbase-win/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/resources-default.json b/app-packages/hbase-win/resources-default.json
index e451c51..93dc17c 100644
--- a/app-packages/hbase-win/resources-default.json
+++ b/app-packages/hbase-win/resources-default.json
@@ -10,7 +10,7 @@
"HBASE_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"slider-appmaster": {
"yarn.memory": "1024"
@@ -18,22 +18,22 @@
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_REST": {
"yarn.role.priority": "3",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
- "yarn.component.instances": "0",
- "yarn.memory": "256"
+ "yarn.component.instances": "1",
+ "yarn.memory": "512"
},
"HBASE_THRIFT2": {
"yarn.role.priority": "5",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.component.instances": "0",
+ "yarn.memory": "512"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/nimbus.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-win/package/scripts/nimbus.py b/app-packages/storm-win/package/scripts/nimbus.py
index fafc97a..c7c3120 100644
--- a/app-packages/storm-win/package/scripts/nimbus.py
+++ b/app-packages/storm-win/package/scripts/nimbus.py
@@ -49,7 +49,7 @@ class Nimbus(Script):
def status(self, env):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_nimbus)
+ check_process_status(status_params.pid_nimbus)
if __name__ == "__main__":
Nimbus().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/rest_api.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-win/package/scripts/rest_api.py b/app-packages/storm-win/package/scripts/rest_api.py
index 6a09e4e..33d8924 100644
--- a/app-packages/storm-win/package/scripts/rest_api.py
+++ b/app-packages/storm-win/package/scripts/rest_api.py
@@ -51,7 +51,7 @@ class StormRestApi(Script):
def status(self, env):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_rest_api)
+ check_process_status(status_params.pid_rest_api)
if __name__ == "__main__":
StormRestApi().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-win/package/scripts/service.py b/app-packages/storm-win/package/scripts/service.py
index b17c636..aa7d339 100644
--- a/app-packages/storm-win/package/scripts/service.py
+++ b/app-packages/storm-win/package/scripts/service.py
@@ -20,9 +20,9 @@ limitations under the License.
from resource_management import *
-import time
import os
import sys
+import xml.etree.ElementTree as et
"""
Slider package uses jps as pgrep does not list the whole process start command
@@ -40,14 +40,32 @@ def service(
os.environ['STORM_LOG_DIR'] = params.log_dir
os.environ['STORM_HOME'] = params.app_root
os.environ['STORM_CONF_DIR'] = params.conf_dir
- cmd = format("{storm_bin} {name} > {log_dir}/{name}.out 2>&1")
- Execute(cmd,
- logoutput=False,
- wait_for_finish=False,
- pid_file = pid_file
+ generate_xml = format("{storm_bin} --service {name} > {log_dir}/{name}.cmd")
+
+ Execute(generate_xml,
+ logoutput=True,
+ wait_for_finish=True
)
+ tree = et.parse(format("{log_dir}/{name}.cmd"))
+ root = tree.getroot()
+ cmd_part = None
+ for child in root:
+ if child.tag == "arguments":
+ cmd_part = child.text
+
+ if cmd_part:
+ cmd = format("{java64_home}\\bin\\java {cmd_part}")
+
+ Execute(cmd,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=pid_file
+ )
+ else:
+ Logger.warn("Valid command file did not get generated at " + format("{log_dir}/{name}.cmd"))
+
elif action == "stop":
pid = format("`cat {pid_file}` >/dev/null 2>&1")
Execute(format("kill {pid}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/supervisor.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-win/package/scripts/supervisor.py b/app-packages/storm-win/package/scripts/supervisor.py
index eba9fa9..47c20c9 100644
--- a/app-packages/storm-win/package/scripts/supervisor.py
+++ b/app-packages/storm-win/package/scripts/supervisor.py
@@ -53,7 +53,7 @@ class Supervisor(Script):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_supervisor)
+ check_process_status(status_params.pid_supervisor)
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/app-packages/storm-win/package/scripts/ui_server.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-win/package/scripts/ui_server.py b/app-packages/storm-win/package/scripts/ui_server.py
index 0875b76..0fe7cd2 100644
--- a/app-packages/storm-win/package/scripts/ui_server.py
+++ b/app-packages/storm-win/package/scripts/ui_server.py
@@ -49,7 +49,7 @@ class UiServer(Script):
def status(self, env):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_ui)
+ check_process_status(status_params.pid_ui)
if __name__ == "__main__":
UiServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66b1aa4a/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index fe7047d..387bc7e 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -295,7 +295,7 @@ class Controller(threading.Thread):
if 'restartEnabled' in response:
restartEnabled = response['restartEnabled']
if restartEnabled:
- logger.info("Component auto-restart is enabled.")
+ logger.debug("Component auto-restart is enabled.")
if 'hasMappedComponents' in response.keys():
self.hasMappedComponents = response['hasMappedComponents'] != False