You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/28 13:19:54 UTC
[06/12] git commit: SLIDER-359. Create hbase package for Windows
SLIDER-359. Create hbase package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b3777fde
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b3777fde
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b3777fde
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b3777fde109cf2e9c9cf22d674d571e7ce845918
Parents: 829821a
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 26 18:13:29 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 26 18:13:29 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/README.txt | 1 +
app-packages/hbase-win/appConfig.json | 6 +-
.../hbase-win/configuration/hbase-env.xml | 67 +---------------
app-packages/hbase-win/metainfo.xml | 10 +++
app-packages/hbase-win/package/scripts/hbase.py | 15 +---
.../hbase-win/package/scripts/hbase_master.py | 8 +-
.../package/scripts/hbase_regionserver.py | 8 +-
.../hbase-win/package/scripts/hbase_service.py | 53 ++++++-------
.../hbase-win/package/scripts/params.py | 82 +++++++-------------
app-packages/hbase/appConfig.json | 3 +-
app-packages/hbase/package/scripts/hbase.py | 2 +
.../core/providers/__init__.py | 1 +
.../core/providers/windows/system.py | 29 +++----
.../libraries/functions/check_process_status.py | 44 ++++++++---
14 files changed, 127 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
index c122d90..625bce6 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -22,6 +22,7 @@ Download the tarball for HBase:
Copy the hbase tarball to package/files
cp ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip package/files
+ rm -rf package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME
Edit appConfig.json/metainfo.xml and replace all occurances of
0.98.0.2.1.5.0-2047-hadoop2 with the correct value.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json
index 52f3aa3..8261f8c 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -21,15 +21,15 @@
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+ "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
+ "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
"site.hbase-env.hbase_master_heapsize": "1024m",
"site.hbase-env.hbase_regionserver_heapsize": "1024m",
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
- "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
"site.hbase-site.hbase.superuser": "yarn",
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/configuration/hbase-env.xml b/app-packages/hbase-win/configuration/hbase-env.xml
index 554c3c5..fa5686f 100644
--- a/app-packages/hbase-win/configuration/hbase-env.xml
+++ b/app-packages/hbase-win/configuration/hbase-env.xml
@@ -45,72 +45,9 @@
<!-- hbase-env.sh -->
<property>
<name>content</name>
- <description>This is the jinja template for hbase-env.sh file</description>
+ <description>This is the jinja template for start command</description>
<value>
-# Set environment variables here.
-
-# The java implementation to use. Java 1.6 required.
-export JAVA_HOME={{java64_home}}
-
-# HBase Configuration directory
-export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}}
-
-# Extra Java CLASSPATH elements. Optional.
-export HBASE_CLASSPATH=${HBASE_CLASSPATH}
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-# export HBASE_HEAPSIZE=1000
-
-# Extra Java runtime options.
-# Below are what we set by default. May only work with SUN JVM.
-# For more on why as well as other possible settings,
-# see http://wiki.apache.org/hadoop/PerformanceTuning
-export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
-export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`"
-# Uncomment below to enable java garbage collection logging.
-# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
-
-# Uncomment and adjust to enable JMX exporting
-# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
-# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-#
-# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
-export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
-export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
-# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
-# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
-
-# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
-export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
-
-# Extra ssh options. Empty by default.
-# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
-
-# Where log files are stored. $HBASE_HOME/logs by default.
-export HBASE_LOG_DIR={{log_dir}}
-
-# A string representing this instance of hbase. $USER by default.
-# export HBASE_IDENT_STRING=$USER
-
-# The scheduling priority for daemon processes. See 'man nice'.
-# export HBASE_NICENESS=10
-
-# The directory where pid files are stored. /tmp by default.
-export HBASE_PID_DIR={{pid_dir}}
-
-# Seconds to sleep between slave commands. Unset by default. This
-# can be useful in large clusters, where, e.g., slave rsyncs can
-# otherwise arrive faster than the master can service them.
-# export HBASE_SLAVE_SLEEP=0.1
-
-# Tell HBase whether it should manage it's own instance of Zookeeper or not.
-export HBASE_MANAGES_ZK=false
-
-{% if security_enabled %}
-export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}"
-export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
-{% endif %}
+ -Xmx{{heap_size}} "-XX:+UseConcMarkSweepGC" "-XX:CMSInitiatingOccupancyFraction=70" "-Djava.net.preferIPv4Stack=true" "-XX:+ForceTimeHighResolution" "-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" -Xloggc:"{{log_dir}}\hbase-{{role_user}}.gc" -Dhbase.log.dir="{{log_dir}}" -Dhbase.log.file="hbase-{{role_user}}.log" -Dhbase.home.dir="{{hbase_root}}" -Dhbase.id.str="{{hbase_instance_name}}" -XX:OnOutOfMemoryError="taskkill /F /PID p" -Dhbase.root.logger="INFO,DRFA" -Djava.library.path="{{java_library_path}}" -Dhbase.security.logger="INFO,DRFAS" -classpath "{{conf_dir}};{{hbase_root}}\lib\*;{{hbase_additional_cp}}"
</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml
index 4ab56ef..315d279 100644
--- a/app-packages/hbase-win/metainfo.xml
+++ b/app-packages/hbase-win/metainfo.xml
@@ -110,9 +110,19 @@
</configFile>
<configFile>
<type>env</type>
+ <fileName>hbase-env.sh</fileName>
+ <dictionaryName>hbase-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
<fileName>hbase-log4j.properties</fileName>
<dictionaryName>hbase-log4j</dictionaryName>
</configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-policy.xml</fileName>
+ <dictionaryName>hbase-policy</dictionaryName>
+ </configFile>
</configFiles>
</application>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase.py b/app-packages/hbase-win/package/scripts/hbase.py
index 70601bc..5ae64eb 100644
--- a/app-packages/hbase-win/package/scripts/hbase.py
+++ b/app-packages/hbase-win/package/scripts/hbase.py
@@ -23,20 +23,10 @@ from resource_management import *
import sys
import shutil
-def hbase(name=None # 'master' or 'regionserver' or 'client'
+def hbase(name=None # 'master' or 'regionserver'
):
import params
- if name in ["master","regionserver"]:
- params.HdfsDirectory(params.hbase_hdfs_root_dir,
- action="create_delayed"
- )
- params.HdfsDirectory(params.hbase_staging_dir,
- action="create_delayed",
- mode=0711
- )
- params.HdfsDirectory(None, action="create")
-
Directory( params.conf_dir,
owner = params.hbase_user,
group = params.user_group,
@@ -52,7 +42,6 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
Directory (os.path.join(params.local_dir, "jars"),
owner = params.hbase_user,
group = params.user_group,
- mode=0775,
recursive = True
)
@@ -66,14 +55,12 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
if (params.log4j_props != None):
File(format("{params.conf_dir}/log4j.properties"),
- mode=0644,
group=params.user_group,
owner=params.hbase_user,
content=params.log4j_props
)
elif (os.path.exists(format("{conf_dir}/log4j.properties"))):
File(format("{params.conf_dir}/log4j.properties"),
- mode=0644,
group=params.user_group,
owner=params.hbase_user
)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase_master.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_master.py b/app-packages/hbase-win/package/scripts/hbase_master.py
index f4e202f..47b2409 100644
--- a/app-packages/hbase-win/package/scripts/hbase_master.py
+++ b/app-packages/hbase-win/package/scripts/hbase_master.py
@@ -40,9 +40,9 @@ class HbaseMaster(Script):
env.set_params(params)
self.configure(env) # for security
- ##hbase_service( 'master',
- ## action = 'start'
- ##)
+ hbase_service( 'master',
+ action = 'start'
+ )
def stop(self, env):
import params
@@ -56,7 +56,7 @@ class HbaseMaster(Script):
import status_params
env.set_params(status_params)
pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
- ##check_process_status(pid_file)
+ check_process_status(pid_file)
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_regionserver.py b/app-packages/hbase-win/package/scripts/hbase_regionserver.py
index 7114dc8..daa5732 100644
--- a/app-packages/hbase-win/package/scripts/hbase_regionserver.py
+++ b/app-packages/hbase-win/package/scripts/hbase_regionserver.py
@@ -40,9 +40,9 @@ class HbaseRegionServer(Script):
env.set_params(params)
self.configure(env) # for security
- ##hbase_service( 'regionserver',
- ## action = 'start'
- ##)
+ hbase_service( 'regionserver',
+ action = 'start'
+ )
def stop(self, env):
import params
@@ -56,7 +56,7 @@ class HbaseRegionServer(Script):
import status_params
env.set_params(status_params)
pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
- ##check_process_status(pid_file)
+ check_process_status(pid_file)
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py
index 96add84..c02526c 100644
--- a/app-packages/hbase-win/package/scripts/hbase_service.py
+++ b/app-packages/hbase-win/package/scripts/hbase_service.py
@@ -20,32 +20,29 @@ limitations under the License.
from resource_management import *
+
def hbase_service(
- name,
- action = 'start'): # 'start' or 'stop' or 'status'
-
- import params
-
- role = name
- cmd = format("{daemon_script} --config {conf_dir}")
- pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid")
-
- daemon_cmd = None
- no_op_test = None
-
- if action == 'start':
- daemon_cmd = format("{cmd} start {role}")
- if name == 'rest':
- daemon_cmd = format("{daemon_cmd} -p {rest_port}")
- elif name == 'thrift':
- daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
- elif name == 'thrift2':
- daemon_cmd = format("{daemon_cmd} -p {thrift2_port}")
- no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
- elif action == 'stop':
- daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
-
- if daemon_cmd is not None:
- Execute ( daemon_cmd,
- not_if = no_op_test
- )
+ name,
+ action='start'): # 'start' or 'stop' or 'status'
+
+ import params
+
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-{name}.pid")
+ heap_size = params.master_heapsize
+ main_class = "org.apache.hadoop.hbase.master.HMaster"
+ if name == "regionserver":
+ heap_size = params.regionserver_heapsize
+ main_class = "org.apache.hadoop.hbase.regionserver.HRegionServer"
+
+ role_user = format("{hbase_user}-{name}")
+
+ rest_of_the_command = InlineTemplate(params.hbase_env_sh_template, [], heap_size=heap_size, role_user=role_user)()
+
+ process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}")
+
+ Execute(process_cmd,
+ user=params.hbase_user,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=pid_file
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py
index 33e2bbf..421b9f5 100644
--- a/app-packages/hbase-win/package/scripts/params.py
+++ b/app-packages/hbase-win/package/scripts/params.py
@@ -25,83 +25,57 @@ import status_params
# server configurations
config = Script.get_config()
+java64_home = config['hostLevelParams']['java_home']
hbase_root = config['configurations']['global']['app_root']
-conf_dir = format("{hbase_root}/conf")
-daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
-
+hbase_instance_name = config['configurations']['global']['hbase_instance_name']
hbase_user = status_params.hbase_user
-_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
user_group = config['configurations']['global']['user_group']
-# this is "hadoop-metrics.properties" for 1.x stacks
-metric_prop_file_name = "hadoop-metrics2-hbase.properties"
+pid_dir = status_params.pid_dir
+tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'],
+ config['configurations']['hbase-site'])
+conf_dir = format("{hbase_root}/conf")
+log_dir = config['configurations']['global']['app_log_dir']
+input_conf_files_dir = config['configurations']['global']['app_input_conf_dir']
-# not supporting 32 bit jdk.
-java64_home = config['hostLevelParams']['java_home']
+hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
+rest_port = config['configurations']['global']['hbase_rest_port']
+thrift_port = config['configurations']['global']['hbase_thrift_port']
+thrift2_port = config['configurations']['global']['hbase_thrift2_port']
-log_dir = config['configurations']['global']['app_log_dir']
master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
-
regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
-pid_dir = status_params.pid_dir
-tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
-local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site'])
-input_conf_files_dir = config['configurations']['global']['app_input_conf_dir']
+hbase_env_sh_template = config['configurations']['hbase-env']['content']
+java_library_path = config['configurations']['global']['java_library_path']
+hbase_additional_cp = config['configurations']['global']['hbase_additional_cp']
-client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf"))
master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
-regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf"))
-
-ganglia_server_host = default('/configurations/global/ganglia_server_host', '')
-ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663')
-
-rest_port = config['configurations']['global']['hbase_rest_port']
-thrift_port = config['configurations']['global']['hbase_thrift_port']
-thrift2_port = config['configurations']['global']['hbase_thrift2_port']
+regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file',
+ format("{conf_dir}/hbase_regionserver_jaas.conf"))
+master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
+regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
- master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
- regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+ master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST', hostname_lowercase)
+ regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST', hostname_lowercase)
-
-master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
-regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
-kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = functions.get_kinit_path(
+ [default("kinit_path_local", None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
if security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
else:
kinit_cmd = ""
-#log4j.properties
+# log4j.properties
if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])):
log4j_props = config['configurations']['hbase-log4j']['content']
else:
- log4j_props = None
-
-hbase_env_sh_template = config['configurations']['hbase-env']['content']
-
-hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
-hbase_staging_dir = config['configurations']['hbase-site']['hbase.stagingdir']
-#for create_hdfs_directory
-hostname = config["hostname"]
-hadoop_conf_dir = "/etc/hadoop/conf"
-hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab']
-hdfs_user = config['configurations']['global']['hdfs_user']
-kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local
-)
+ log4j_props = None
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index f065f90..9c37ed9 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -25,8 +25,7 @@
"site.hbase-env.hbase_master_heapsize": "1024m",
"site.hbase-env.hbase_regionserver_heapsize": "1024m",
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
- "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
"site.hbase-site.hbase.superuser": "yarn",
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase.py b/app-packages/hbase/package/scripts/hbase.py
index a979bc6..35897df 100644
--- a/app-packages/hbase/package/scripts/hbase.py
+++ b/app-packages/hbase/package/scripts/hbase.py
@@ -27,6 +27,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
):
import params
+ """
if name in ["master","regionserver"]:
params.HdfsDirectory(params.hbase_hdfs_root_dir,
action="create_delayed"
@@ -36,6 +37,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
mode=0711
)
params.HdfsDirectory(None, action="create")
+ """
Directory( params.conf_dir,
owner = params.hbase_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/slider-agent/src/main/python/resource_management/core/providers/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
index c6f5c03..8ebeb77 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/__init__.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
@@ -54,6 +54,7 @@ PROVIDERS = dict(
Service="resource_management.core.providers.windows.service.ServiceProvider",
Execute="resource_management.core.providers.windows.system.ExecuteProvider",
File="resource_management.core.providers.windows.system.FileProvider",
+ Directory="resource_management.core.providers.windows.system.DirectoryProvider",
Tarball="resource_management.core.providers.windows.tarball.TarballProvider"
),
default=dict(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index 3165b5d..9514bdb 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -29,15 +29,6 @@ import os
import subprocess
import shutil
from resource_management.libraries.script import Script
-import win32con
-from win32security import *
-from win32api import *
-from winerror import ERROR_INVALID_HANDLE
-from win32profile import CreateEnvironmentBlock
-from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser
-from win32event import WaitForSingleObject, INFINITE
-import msvcrt
-import tempfile
def _merge_env(env1, env2, merge_keys=['PYTHONPATH']):
@@ -76,25 +67,28 @@ def _merge_env(env1, env2, merge_keys=['PYTHONPATH']):
def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, user=None, pid_file_name=None):
# TODO implement user
Logger.info("Executing %s" % (command))
- cur_token = OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS)
- current_env = CreateEnvironmentBlock(cur_token, False)
- current_env = _merge_env(current_env, env)
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- cwd=cwd, env=current_env, shell=False)
+ cwd=cwd, env=env, shell=False)
+ code = None
if not wait_for_finish:
+ Logger.debug("No need to wait for the process to exit. Will leave the process running ...")
+ code = 0
if pid_file_name:
+ Logger.debug("Writing the process id %s to file %s" % (str(proc.pid), pid_file_name))
pidfile = open(pid_file_name, 'w')
pidfile.write(str(proc.pid))
pidfile.close()
- return None, None
+ Logger.info("Wrote the process id to file %s" % pid_file_name)
+ return code, None, None
if timeout:
q = Queue()
t = threading.Timer( timeout, on_timeout, [proc, q] )
t.start()
- out, err = proc.communicate()
- code = proc.returncode
+ out, err = proc.communicate()
+ code = proc.returncode
+
if logoutput and out:
Logger.info(out)
if logoutput and err:
@@ -110,7 +104,7 @@ def _set_file_acl(file, user, rights):
raise Fail("Can not remove rights for path {0} and user {1}".format(file, user))
code, out, err = _call_command(acls_modify_cmd)
if code != 0:
- raise Fail("Can not set rights {0} for path {1} and user {2}".format(file, user))
+ raise Fail("Can not set rights {0} for path {1} and user {2}".format(rights, file, user))
else:
return
@@ -194,6 +188,7 @@ class ExecuteProvider(Provider):
raise Fail("Failed to execute " + self.resource.command)
break
except Fail as ex:
+ Logger.info("Error raised: %s" % str(ex))
if i == self.resource.tries - 1: # last try
raise ex
else:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
index e8ed9f0..e17b5f7 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
@@ -25,6 +25,11 @@ from resource_management.core.logger import Logger
__all__ = ["check_process_status"]
import os
+import subprocess
+import platform
+
+IS_WINDOWS = platform.system() == "Windows"
+
def check_process_status(pid_file):
"""
@@ -43,15 +48,32 @@ def check_process_status(pid_file):
except:
Logger.debug("Pid file {0} does not exist".format(pid_file))
raise ComponentIsNotRunning()
- try:
- # Kill will not actually kill the process
- # From the doc:
- # If sig is 0, then no signal is sent, but error checking is still
- # performed; this can be used to check for the existence of a
- # process ID or process group ID.
- os.kill(pid, 0)
- except OSError:
- Logger.debug("Process with pid {0} is not running. Stale pid file"
- " at {1}".format(pid, pid_file))
- raise ComponentIsNotRunning()
+
+ if IS_WINDOWS:
+ not_running = True
+ try:
+ ps = subprocess.Popen(r'tasklist.exe /NH /FI "PID eq %d"' % (pid),
+ shell=True, stdout=subprocess.PIPE)
+ output = ps.stdout.read()
+ ps.stdout.close()
+ ps.wait()
+ not_running = str(pid) not in output
+ except OSError, e:
+ Logger.debug("Error {0}".format(str(e)))
+ Logger.info("Process with pid {0} is not running. Stale pid file"
+ " at {1}".format(pid, pid_file))
+ if not_running:
+ raise ComponentIsNotRunning()
+ else:
+ try:
+ # Kill will not actually kill the process
+ # From the doc:
+ # If sig is 0, then no signal is sent, but error checking is still
+ # performed; this can be used to check for the existence of a
+ # process ID or process group ID.
+ os.kill(pid, 0)
+ except OSError:
+ Logger.info("Process with pid {0} is not running. Stale pid file"
+ " at {1}".format(pid, pid_file))
+ raise ComponentIsNotRunning()
pass