You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/07/17 08:53:54 UTC
svn commit: r1611262 [2/2] - in /incubator/slider/site:
content/docs/slider_specs/ trunk/content/docs/slider_specs/
Modified: incubator/slider/site/trunk/content/docs/slider_specs/writing_app_command_scripts.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/slider_specs/writing_app_command_scripts.md?rev=1611262&r1=1611261&r2=1611262&view=diff
==============================================================================
--- incubator/slider/site/trunk/content/docs/slider_specs/writing_app_command_scripts.md (original)
+++ incubator/slider/site/trunk/content/docs/slider_specs/writing_app_command_scripts.md Thu Jul 17 06:53:53 2014
@@ -70,109 +70,107 @@ The parameters are organized as multi-la
### Sample configuration parameters
-While a script can access all the config parameters the key property bags are
+While a script can access all the config parameters the key property bags are **hostLevelParams** and **configurations**.
+
-* hostLevelParams
-* configurations
-
-```
-{
- "roleCommand": "START",
- "commandParams": {
- "record_config": "true",
- "service_package_folder": "${AGENT_WORK_ROOT}/work/app/definition/package",
- "script": "scripts/hbase_regionserver.py",
- "schema_version": "2.0",
- "command_timeout": "300",
- "script_type": "PYTHON"
- },
- "componentName": "HBASE_REGIONSERVER",
- "hostname": "c6403.ambari.apache.org",
- "hostLevelParams": {
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "container_id": "container_1405048900371_0015_02_000003"
- },
- "commandType": "EXECUTION_COMMAND",
- "clusterName": "cl1",
- "serviceName": "cl1",
- "role": "HBASE_REGIONSERVER",
- "taskId": 4,
- "public_hostname": "c6403.ambari.apache.org",
- "commandId": "4-1",
- "configurations": {
- "hbase-site": {
- "hbase.hstore.flush.retries.number": "120",
- "hbase.client.keyvalue.maxsize": "10485760",
- "hbase.hstore.compactionThreshold": "3",
- "hbase.rootdir": "hdfs://c6403.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/database/data",
- "hbase.regionserver.handler.count": "60",
- "hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "hbase.hregion.memstore.block.multiplier": "2",
- "hbase.hregion.memstore.flush.size": "134217728",
- "hbase.superuser": "yarn",
- "hbase.zookeeper.property.clientPort": "2181",
- "hbase.regionserver.global.memstore.upperLimit": "0.4",
- "zookeeper.session.timeout": "30000",
- "hbase.tmp.dir": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/work/app/tmp",
- "hfile.block.cache.size": "0.40",
- "hbase.stagingdir": "hdfs://c6403.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/database/staging",
- "hbase.hregion.max.filesize": "10737418240",
- "hbase.regionserver.port": "0",
- "hbase.security.authentication": "simple",
- "hbase.defaults.for.version.skip": "true",
- "hbase.master.info.port": "52502",
- "hbase.zookeeper.quorum": "c6403.ambari.apache.org",
- "hbase.regionserver.info.port": "0",
- "zookeeper.znode.parent": "/services/slider/users/yarn/cl1",
- "hbase.hstore.blockingStoreFiles": "10",
- "hbase.hregion.majorcompaction": "86400000",
- "hbase.security.authorization": "false",
- "hbase.local.dir": "${hbase.tmp.dir}/local",
- "hbase.cluster.distributed": "true",
- "hbase.hregion.memstore.mslab.enabled": "true",
- "hbase.client.scanner.caching": "100",
- "hbase.zookeeper.useMulti": "true"
+ {
+ "roleCommand": "START",
+ "commandParams": {
+ "record_config": "true",
+ "service_package_folder": "${AGENT_WORK_ROOT}/work/app/definition/package",
+ "script": "scripts/hbase_regionserver.py",
+ "schema_version": "2.0",
+ "command_timeout": "300",
+ "script_type": "PYTHON"
},
- "global": {
- "security_enabled": "false",
- "ganglia_server_port": "8667",
- "hbase_master_heapsize": "1024m",
- "app_root": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/install/hbase-0.98.3-hadoop2",
- "app_log_dir": "/hadoop/yarn/log/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/log",
- "ganglia_server_host": "c6403.ambari.apache.org",
- "ganglia_server_id": "Application1",
- "hbase_regionserver_heapsize": "1024m",
- "user_group": "hadoop",
- "app_pid_dir": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/run",
- "app_install_dir": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/install",
- "app_user": "yarn"
+ "componentName": "HBASE_REGIONSERVER",
+ "hostname": "c6403.ambari.apache.org",
+ "hostLevelParams": {
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "container_id": "container_1405048900371_0015_02_000003"
},
- "hdfs-site": {},
- "core-site": {}
+ "commandType": "EXECUTION_COMMAND",
+ "clusterName": "cl1",
+ "serviceName": "cl1",
+ "role": "HBASE_REGIONSERVER",
+ "taskId": 4,
+ "public_hostname": "c6403.ambari.apache.org",
+ "commandId": "4-1",
+ "configurations": {
+ "hbase-site": {
+ "hbase.hstore.flush.retries.number": "120",
+ "hbase.client.keyvalue.maxsize": "10485760",
+ "hbase.hstore.compactionThreshold": "3",
+ "hbase.rootdir": "hdfs://c6403.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/database/data",
+ "hbase.regionserver.handler.count": "60",
+ "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "hbase.hregion.memstore.block.multiplier": "2",
+ "hbase.hregion.memstore.flush.size": "134217728",
+ "hbase.superuser": "yarn",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.regionserver.global.memstore.upperLimit": "0.4",
+ "zookeeper.session.timeout": "30000",
+ "hbase.tmp.dir": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/work/app/tmp",
+ "hfile.block.cache.size": "0.40",
+ "hbase.stagingdir": "hdfs://c6403.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/database/staging",
+ "hbase.hregion.max.filesize": "10737418240",
+ "hbase.regionserver.port": "0",
+ "hbase.security.authentication": "simple",
+ "hbase.defaults.for.version.skip": "true",
+ "hbase.master.info.port": "52502",
+ "hbase.zookeeper.quorum": "c6403.ambari.apache.org",
+ "hbase.regionserver.info.port": "0",
+ "zookeeper.znode.parent": "/services/slider/users/yarn/cl1",
+ "hbase.hstore.blockingStoreFiles": "10",
+ "hbase.hregion.majorcompaction": "86400000",
+ "hbase.security.authorization": "false",
+ "hbase.local.dir": "${hbase.tmp.dir}/local",
+ "hbase.cluster.distributed": "true",
+ "hbase.hregion.memstore.mslab.enabled": "true",
+ "hbase.client.scanner.caching": "100",
+ "hbase.zookeeper.useMulti": "true"
+ },
+ "global": {
+ "security_enabled": "false",
+ "ganglia_server_port": "8667",
+ "hbase_master_heapsize": "1024m",
+ "app_root": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/install/hbase-0.98.3-hadoop2",
+ "app_log_dir": "/hadoop/yarn/log/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/log",
+ "ganglia_server_host": "c6403.ambari.apache.org",
+ "ganglia_server_id": "Application1",
+ "hbase_regionserver_heapsize": "1024m",
+ "user_group": "hadoop",
+ "app_pid_dir": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/run",
+ "app_install_dir": "/hadoop/yarn/local/usercache/yarn/appcache/application_1405048900371_0015/container_1405048900371_0015_02_000003/app/install",
+ "app_user": "yarn"
+ },
+ "hdfs-site": {},
+ "core-site": {}
+ }
}
-}
-```
+
## Command scripts
A command script essentially implements commands. The basic structure of a script is as follows:
-```
-class HbaseMaster(Script):
- def install(self, env):
- ...
+
+ class HbaseMaster(Script):
+ def install(self, env):
+ ...
- def configure(self, env):
- ...
+ def configure(self, env):
+ ...
- def start(self, env):
- ...
+ def start(self, env):
+ ...
- def stop(self, env):
- ...
+ def stop(self, env):
+ ...
+
+ def status(self, env):
+ ...
+
- def status(self, env):
- ...
-```
At this point, you are free to implement as you wish. Slider provides a rich library for basic operations needed for application commands. So the structure we suggest is:
* Create a params.py file to read the json config file into variables. This will allow you to provide initial validation, provide default values, or create complex values
@@ -180,54 +178,53 @@ At this point, you are free to implement
A params.py file may be as follows:
-```
-from resource_management import *
-# server configurations
-config = Script.get_config()
+ from resource_management import *
-hbase_root = config['configurations']['global']['app_root']
-conf_dir = format("{hbase_root}/conf")
-daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
-...
-```
+ # server configurations
+ config = Script.get_config()
+
+ hbase_root = config['configurations']['global']['app_root']
+ conf_dir = format("{hbase_root}/conf")
+ daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
+ ...
### Sample command script
In this example, hbase master command implementation is using the above parameter file and calling into several helper functions. Some are provided by Slider (e.g. install_package, check_process_status) and some are developed for HBase (e.g. habse_service called by start)
-```
-class HbaseMaster(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
-
- hbase(name='master')
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env) # for security
-
- hbase_service( 'master',
- action = 'start'
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- hbase_service( 'master',
- action = 'stop'
- )
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
- check_process_status(pid_file)
-```
+
+ class HbaseMaster(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='master')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'master',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'master',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
+ check_process_status(pid_file)
+