You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/07/25 04:33:13 UTC
git commit: Agent changes for windows support
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-196-Agent-Support-For-Windows 30ee40075 -> ee00fccb7
Agent changes for windows support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ee00fccb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ee00fccb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ee00fccb
Branch: refs/heads/feature/SLIDER-196-Agent-Support-For-Windows
Commit: ee00fccb7a5a45c5a15bb3a62c6d119f8a50d54a
Parents: 30ee400
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jul 24 19:32:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jul 24 19:32:12 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/README.txt | 36 ++++
app-packages/memcached-win/appConfig.json | 22 +++
app-packages/memcached-win/metainfo.xml | 51 ++++++
.../jmemcached-cli-1.0.0.jar.REPLACEME | 16 ++
.../jmemcached-core-1.0.0.jar.REPLACEME | 16 ++
.../memcached-win/package/scripts/memcached.py | 54 ++++++
.../memcached-win/package/scripts/params.py | 32 ++++
app-packages/memcached-win/resources.json | 16 ++
app-packages/memcached/appConfig.json | 2 -
.../src/main/python/agent/AgentConfig.py | 3 +-
.../python/agent/CustomServiceOrchestrator.py | 38 +++--
.../src/main/python/agent/ProcessHelper.py | 3 +-
.../src/main/python/agent/PythonExecutor.py | 7 +-
slider-agent/src/main/python/agent/main.py | 20 ++-
slider-agent/src/main/python/agent/shell.py | 102 ++++++++++--
.../python/resource_management/core/logger.py | 5 +-
.../core/providers/__init__.py | 5 +
.../core/providers/accounts.py | 20 +--
.../core/providers/system.py | 20 +--
.../core/providers/windows/__init__.py | 20 +++
.../core/providers/windows/service.py | 65 ++++++++
.../core/providers/windows/system.py | 142 ++++++++++++++++
.../libraries/functions/__init__.py | 2 +-
.../libraries/functions/os_check.py | 166 +++++++++++++++----
.../libraries/providers/__init__.py | 2 +
.../libraries/providers/monitor_webserver.py | 2 +-
.../libraries/script/script.py | 15 +-
slider-agent/src/test/python/agent/TestShell.py | 6 -
.../providers/agent/AgentProviderService.java | 31 +++-
.../server/services/security/SecurityUtils.java | 8 +-
30 files changed, 817 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/README.txt b/app-packages/memcached-win/README.txt
new file mode 100644
index 0000000..4d93b91
--- /dev/null
+++ b/app-packages/memcached-win/README.txt
@@ -0,0 +1,36 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+How to create a Slider app package for Memcached for Windows?
+
+To create the app package you will need the Memcached tarball copied to a specific location.
+
+Replace the placeholder jar files for JMemcached.
+ cp ~/Downloads/jmemcached-cli-1.0.0.jar package/files/jmemcached-1.0.0/
+ cp ~/Downloads/jmemcached-core-1.0.0.jar package/files/jmemcached-1.0.0/
+ rm package/files/jmemcached-1.0.0/*.REPLACEME
+
+Create a zip package at the root of the package (<slider enlistment>/app-packages/memcached/)
+ zip -r jmemcached-1.0.0.zip .
+
+Verify the content using
+ unzip -l "$@" jmemcached-1.0.0.zip
+
+While appConfig.json and resources.json are not required for the package they work
+well as the default configuration for Slider apps. So its advisable that when you
+create an application package for Slider, include sample/default resources.json and
+appConfig.json for a minimal Yarn cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
new file mode 100644
index 0000000..8c00660
--- /dev/null
+++ b/app-packages/memcached-win/appConfig.json
@@ -0,0 +1,22 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": "/slider/jmemcached-1.0.0.zip",
+ "java_home": "C:\\java",
+ "site.global.app_user": "hadoop",
+ "site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
+ "site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
+ "site.global.xmx_val": "256m",
+ "site.global.xms_val": "128m",
+ "site.global.memory_val": "200M"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ },
+ "MEMCACHED": {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/metainfo.xml b/app-packages/memcached-win/metainfo.xml
new file mode 100644
index 0000000..a0de8c2
--- /dev/null
+++ b/app-packages/memcached-win/metainfo.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>MEMCACHED</name>
+ <comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
+ <version>1.0.0</version>
+
+ <components>
+ <component>
+ <name>MEMCACHED</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/memcached.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>folder</type>
+ <name>files\\jmemcached-1.0.0</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </application>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-cli-1.0.0.jar.REPLACEME
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-cli-1.0.0.jar.REPLACEME b/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-cli-1.0.0.jar.REPLACEME
new file mode 100644
index 0000000..6855ef9
--- /dev/null
+++ b/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-cli-1.0.0.jar.REPLACEME
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace with the real jar.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-core-1.0.0.jar.REPLACEME
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-core-1.0.0.jar.REPLACEME b/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-core-1.0.0.jar.REPLACEME
new file mode 100644
index 0000000..6855ef9
--- /dev/null
+++ b/app-packages/memcached-win/package/files/jmemcached-1.0.0/jmemcached-core-1.0.0.jar.REPLACEME
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace with the real jar.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/memcached.py b/app-packages/memcached-win/package/scripts/memcached.py
new file mode 100644
index 0000000..222a62c
--- /dev/null
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+class Memcached(Script):
+ def install(self, env):
+ self.install_packages(env)
+ pass
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val}")
+
+ Execute(process_cmd,
+ user=params.app_user,
+ logoutput=False,
+ wait_for_finish=False
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+
+if __name__ == "__main__":
+ Memcached().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/params.py b/app-packages/memcached-win/package/scripts/params.py
new file mode 100644
index 0000000..a2fba8b
--- /dev/null
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+app_root = config['configurations']['global']['app_root']
+java64_home = config['hostLevelParams']['java_home']
+app_user = config['configurations']['global']['app_user']
+additional_cp = config['configurations']['global']['additional_cp']
+xmx_val = config['configurations']['global']['xmx_val']
+xms_val = config['configurations']['global']['xms_val']
+memory_val = config['configurations']['global']['memory_val']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached-win/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/resources.json b/app-packages/memcached-win/resources.json
new file mode 100644
index 0000000..f0e02ac
--- /dev/null
+++ b/app-packages/memcached-win/resources.json
@@ -0,0 +1,16 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "MEMCACHED": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index e33179c..d205591 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -5,8 +5,6 @@
"global": {
"application.def": "/slider/jmemcached-1.0.0.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/jmemcached-1.0.0.tar",
- "config_types": "",
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/agent/AgentConfig.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py
index 16b924c..7f8a350 100644
--- a/slider-agent/src/main/python/agent/AgentConfig.py
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -22,6 +22,7 @@ import ConfigParser
import StringIO
import os
import logging
+import posixpath
logger = logging.getLogger()
@@ -141,7 +142,7 @@ class AgentConfig:
if name in AgentConfig.FOLDER_MAPPING and AgentConfig.FOLDER_MAPPING[
name] == "LOG":
root_folder_to_use = self.logroot
- return os.path.join(root_folder_to_use, relativePath)
+ return posixpath.join(root_folder_to_use, relativePath)
else:
return relativePath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 6296033..4eb22bf 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -24,6 +24,8 @@ import json
import pprint
import sys
import socket
+import posixpath
+import platform
from AgentConfig import AgentConfig
from AgentException import AgentException
from PythonExecutor import PythonExecutor
@@ -49,10 +51,10 @@ class CustomServiceOrchestrator():
self.config = config
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
- self.status_commands_stdout = os.path.join(self.tmp_dir,
- 'status_command_stdout.txt')
- self.status_commands_stderr = os.path.join(self.tmp_dir,
- 'status_command_stderr.txt')
+ self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
+ 'status_command_stdout.txt'))
+ self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
+ 'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
@@ -61,8 +63,8 @@ class CustomServiceOrchestrator():
os.unlink(self.status_commands_stderr)
except OSError:
pass # Ignore fail
- self.base_dir = os.path.join(
- config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package")
+ self.base_dir = os.path.realpath(posixpath.join(
+ config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package"))
def runCommand(self, command, tmpoutfile, tmperrfile,
@@ -78,8 +80,8 @@ class CustomServiceOrchestrator():
script_path = self.resolve_script_path(self.base_dir, script, script_type)
script_tuple = (script_path, self.base_dir)
- tmpstrucoutfile = os.path.join(self.tmp_dir,
- "structured-out-{0}.json".format(task_id))
+ tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
+ "structured-out-{0}.json".format(task_id)))
if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
@@ -94,11 +96,15 @@ class CustomServiceOrchestrator():
ret = None
for py_file, current_base_dir in filtered_py_file_list:
script_params = [command_name, json_path, current_base_dir]
- python_paths = [os.path.join(self.config.getWorkRootPath(),
- "infra/agent/slider-agent/jinja2"),
- os.path.join(self.config.getWorkRootPath(),
- "infra/agent/slider-agent")]
- environment_vars = [("PYTHONPATH", ":".join(python_paths))]
+ python_paths = [os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
+ "infra", "agent", "slider-agent", "jinja2")),
+ os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
+ "infra", "agent", "slider-agent"))]
+ if platform.system() != "Windows":
+ environment_vars = [("PYTHONPATH", ":".join(python_paths))]
+ else:
+ environment_vars = [("PYTHONPATH", ";".join(python_paths))]
+
ret = self.python_executor.run_file(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
@@ -141,7 +147,7 @@ class CustomServiceOrchestrator():
"""
Encapsulates logic of script location determination.
"""
- path = os.path.join(base_dir, script)
+ path = os.path.realpath(posixpath.join(base_dir, script))
if not os.path.exists(path):
message = "Script {0} does not exist".format(path)
raise AgentException(message)
@@ -207,10 +213,10 @@ class CustomServiceOrchestrator():
if command_type == ActionQueue.STATUS_COMMAND:
# These files are frequently created, thats why we don't
# store them all, but only the latest one
- file_path = os.path.join(self.tmp_dir, "status_command.json")
+ file_path = os.path.realpath(posixpath.join(self.tmp_dir, "status_command.json"))
else:
task_id = command['taskId']
- file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id))
+ file_path = os.path.realpath(posixpath.join(self.tmp_dir, "command-{0}.json".format(task_id)))
# Json may contain passwords, that's why we need proper permissions
if os.path.isfile(file_path):
os.unlink(file_path)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/agent/ProcessHelper.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ProcessHelper.py b/slider-agent/src/main/python/agent/ProcessHelper.py
index b6283b0..467c4d8 100644
--- a/slider-agent/src/main/python/agent/ProcessHelper.py
+++ b/slider-agent/src/main/python/agent/ProcessHelper.py
@@ -22,12 +22,13 @@ import os
import logging
import traceback
import sys
+import posixpath
from shell import getTempFiles
logger = logging.getLogger()
if 'AGENT_WORK_ROOT' in os.environ:
- pidfile = os.path.join(os.environ['AGENT_WORK_ROOT'], "infra/run/agent.pid")
+ pidfile = os.path.realpath(posixpath.join(os.environ['AGENT_WORK_ROOT'], "infra", "run", "agent.pid"))
else:
pidfile = None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/agent/PythonExecutor.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/PythonExecutor.py b/slider-agent/src/main/python/agent/PythonExecutor.py
index 5f29e5e..54ce247 100644
--- a/slider-agent/src/main/python/agent/PythonExecutor.py
+++ b/slider-agent/src/main/python/agent/PythonExecutor.py
@@ -28,6 +28,7 @@ from threading import Thread
from Grep import Grep
import shell
import sys
+import platform
import Constants
@@ -125,6 +126,7 @@ class PythonExecutor:
Creates subprocess with given parameters. This functionality was moved to separate method
to make possible unit testing
"""
+ close_fds = None if platform.system() == "Windows" else True
env = os.environ.copy()
if environment_vars:
for k, v in environment_vars:
@@ -132,13 +134,14 @@ class PythonExecutor:
env[k] = v
return subprocess.Popen(command,
stdout=tmpout,
- stderr=tmperr, close_fds=True, env=env)
+ stderr=tmperr, close_fds=close_fds, env=env)
def isSuccessfull(self, returncode):
return not self.python_process_has_been_killed and returncode == 0
def python_command(self, script, script_params):
- python_binary = sys.executable
+ #we need manually pass python executable on windows because sys.executable will return service wrapper
+ python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
python_command = [python_binary, "-S", script] + script_params
return python_command
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index 12e07ba..a6a661c 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -26,13 +26,17 @@ import sys
import traceback
import os
import time
-import errno
+import platform
+import ConfigParser
import ProcessHelper
+import errno
+import posixpath
from Controller import Controller
from AgentConfig import AgentConfig
from NetUtil import NetUtil
logger = logging.getLogger()
+IS_WINDOWS = platform.system() == "Windows"
formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()
@@ -99,12 +103,13 @@ def update_log_level(config, logfile):
def bind_signal_handlers():
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
- signal.signal(signal.SIGUSR1, debug)
+ if platform.system() != "Windows":
+ signal.signal(signal.SIGUSR1, debug)
def update_config_from_file(agentConfig):
try:
- configFile = os.path.join(agentConfig.getWorkRootPath(), configFileRelPath)
+ configFile = posixpath.join(agentConfig.getWorkRootPath(), configFileRelPath)
if os.path.exists(configFile):
agentConfig.setConfig(configFile)
else:
@@ -136,7 +141,7 @@ def ensure_folder_layout(config):
def ensure_path_exists(path):
try:
- os.makedirs(path)
+ os.makedirs(os.path.realpath(path))
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
@@ -188,7 +193,8 @@ def main():
if not options.label:
parser.error("label is required.");
- bind_signal_handlers()
+ if not IS_WINDOWS:
+ bind_signal_handlers()
# Check for configuration file.
agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label)
@@ -208,11 +214,11 @@ def main():
agentConfig.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, options.debug)
# set the security directory to a subdirectory of the run dir
- secDir = os.path.join(agentConfig.getResolvedPath(AgentConfig.RUN_DIR), "security")
+ secDir = posixpath.join(agentConfig.getResolvedPath(AgentConfig.RUN_DIR), "security")
logger.info("Security/Keys directory: " + secDir)
agentConfig.set(AgentConfig.SECURITY_SECTION, "keysdir", secDir)
- logFile = os.path.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
+ logFile = posixpath.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
perform_prestart_checks(agentConfig)
ensure_folder_layout(agentConfig)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/agent/shell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/shell.py b/slider-agent/src/main/python/agent/shell.py
index d339764..446dde9 100644
--- a/slider-agent/src/main/python/agent/shell.py
+++ b/slider-agent/src/main/python/agent/shell.py
@@ -28,16 +28,19 @@ import threading
import time
import traceback
import pprint
+import platform
-try:
+if platform.system() != "Windows":
+ try:
import pwd
-except ImportError:
+ except ImportError:
import winpwd as pwd
global serverTracker
serverTracker = {}
logger = logging.getLogger()
+shellRunner = None
threadLocal = threading.local()
gracefull_kill_delay = 5 # seconds between SIGTERM and SIGKILL
tempFiles = []
@@ -47,7 +50,51 @@ def noteTempFile(filename):
def getTempFiles():
return tempFiles
-def kill_process_with_children(parent_pid):
+class _dict_to_object:
+ def __init__(self, entries):
+ self.__dict__.update(entries)
+ def __getitem__(self, item):
+ return self.__dict__[item]
+# windows specific code
+def _kill_process_with_children_windows(parent_pid):
+ shellRunner().run(["taskkill", "/T", "/PID", "{0}".format(parent_pid)])
+
+
+class shellRunnerWindows:
+ # Run any command
+ def run(self, script, user=None):
+ logger.warn("user argument ignored on windows")
+ code = 0
+ if not isinstance(script, list):
+ cmd = " "
+ cmd = cmd.join(script)
+ else:
+ cmd = script
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=False)
+ out, err = p.communicate()
+ code = p.wait()
+ logger.debug("Exitcode for %s is %d" % (cmd, code))
+ return {'exitCode': code, 'output': out, 'error': err}
+
+ def runPowershell(self, file=None, script_block=None, args=[]):
+ logger.warn("user argument ignored on windows")
+ code = 0
+ cmd = None
+ if file:
+ cmd = ['powershell', '-WindowStyle', 'Hidden', '-File', file] + args
+ elif script_block:
+ cmd = ['powershell', '-WindowStyle', 'Hidden', '-Command', script_block] + args
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=False)
+ out, err = p.communicate()
+ code = p.wait()
+ logger.debug("Exitcode for %s is %d" % (cmd, code))
+ return _dict_to_object({'exitCode': code, 'output': out, 'error': err})
+
+
+#linux specific code
+def _kill_process_with_children_linux(parent_pid):
def kill_tree_function(pid, signal):
'''
Kills process tree starting from a given pid.
@@ -57,15 +104,15 @@ def kill_process_with_children(parent_pid):
# a given PID and then passes list of "kill -<SIGNAL> PID" commands to 'sh'
# shell.
CMD = """ps xf | awk -v PID=""" + str(pid) + \
- """ ' $1 == PID { P = $1; next } P && /_/ { P = P " " $1;""" + \
- """K=P } P && !/_/ { P="" } END { print "kill -""" \
- + str(signal) + """ "K }' | sh """
+ """ ' $1 == PID { P = $1; next } P && /_/ { P = P " " $1;""" + \
+ """K=P } P && !/_/ { P="" } END { print "kill -""" \
+ + str(signal) + """ "K }' | sh """
process = subprocess.Popen(CMD, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
process.communicate()
- run_kill_function(kill_tree_function, parent_pid)
+ _run_kill_function(kill_tree_function, parent_pid)
-def run_kill_function(kill_function, pid):
+def _run_kill_function(kill_function, pid):
try:
kill_function(pid, signal.SIGTERM)
except Exception, e:
@@ -80,8 +127,43 @@ def run_kill_function(kill_function, pid):
logger.error("Failed to send SIGKILL to PID %d. Process exited?" % (pid))
logger.error("Reported error: " + repr(e))
-def changeUid():
+def _changeUid():
try:
os.setuid(threadLocal.uid)
except Exception:
- logger.warn("can not switch user for running command.")
\ No newline at end of file
+ logger.warn("can not switch user for running command.")
+
+
+class shellRunnerLinux:
+ # Run any command
+ def run(self, script, user=None):
+ try:
+ if user != None:
+ user = pwd.getpwnam(user)[2]
+ else:
+ user = os.getuid()
+ threadLocal.uid = user
+ except Exception:
+ logger.warn("can not switch user for RUN_COMMAND.")
+ code = 0
+ cmd = " "
+ cmd = cmd.join(script)
+ p = subprocess.Popen(cmd, preexec_fn=_changeUid, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=True, close_fds=True)
+ out, err = p.communicate()
+ code = p.wait()
+ logger.debug("Exitcode for %s is %d" % (cmd, code))
+ return {'exitCode': code, 'output': out, 'error': err}
+
+
+def kill_process_with_children(parent_pid):
+ if platform.system() == "Windows":
+ _kill_process_with_children_windows(parent_pid)
+ else:
+ _kill_process_with_children_linux(parent_pid)
+
+
+if platform.system() == "Windows":
+ shellRunner = shellRunnerWindows
+else:
+ shellRunner = shellRunnerLinux
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/logger.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/logger.py b/slider-agent/src/main/python/resource_management/core/logger.py
index 7370c97..b80042a 100644
--- a/slider-agent/src/main/python/resource_management/core/logger.py
+++ b/slider-agent/src/main/python/resource_management/core/logger.py
@@ -79,7 +79,10 @@ class Logger:
val = "[EMPTY]"
# correctly output 'mode' (as they are octal values like 0755)
elif y and x == 'mode':
- val = oct(y)
+ try:
+ val = oct(y)
+ except:
+ val = repr(y)
else:
val = repr(y)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/providers/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
index 0c170e7..630183b 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/__init__.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
@@ -50,6 +50,11 @@ PROVIDERS = dict(
debian=dict(
Package="resource_management.core.providers.package.apt.AptProvider",
),
+ winsrv=dict(
+ Service="resource_management.core.providers.windows.service.ServiceProvider",
+ Execute="resource_management.core.providers.windows.system.ExecuteProvider",
+ File="resource_management.core.providers.windows.system.FileProvider"
+ ),
default=dict(
File="resource_management.core.providers.system.FileProvider",
Directory="resource_management.core.providers.system.DirectoryProvider",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/providers/accounts.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/accounts.py b/slider-agent/src/main/python/resource_management/core/providers/accounts.py
index 747f120..8711e45 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/accounts.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/accounts.py
@@ -22,8 +22,8 @@ Slider Agent
from __future__ import with_statement
-import grp
-import pwd
+#import grp
+#import pwd
from resource_management.core import shell
from resource_management.core.providers import Provider
from resource_management.core.logger import Logger
@@ -70,10 +70,10 @@ class UserProvider(Provider):
@property
def user(self):
- try:
- return pwd.getpwnam(self.resource.username)
- except KeyError:
- return None
+ #try:
+ # return pwd.getpwnam(self.resource.username)
+ #except KeyError:
+ return None
class GroupProvider(Provider):
@@ -110,7 +110,7 @@ class GroupProvider(Provider):
@property
def group(self):
- try:
- return grp.getgrnam(self.resource.group_name)
- except KeyError:
- return None
+ #try:
+ # return grp.getgrnam(self.resource.group_name)
+ #except KeyError:
+ return None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/providers/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/system.py b/slider-agent/src/main/python/resource_management/core/providers/system.py
index 3475d6a..23000f5 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/system.py
@@ -22,9 +22,9 @@ Slider Agent
from __future__ import with_statement
-import grp
+#import grp
import os
-import pwd
+#import pwd
import time
import shutil
from resource_management.core import shell
@@ -38,10 +38,10 @@ def _coerce_uid(user):
try:
uid = int(user)
except ValueError:
- try:
- uid = pwd.getpwnam(user).pw_uid
- except KeyError:
- raise Fail("User %s doesn't exist." % user)
+ #try:
+ # uid = pwd.getpwnam(user).pw_uid
+ #except KeyError:
+ raise Fail("User %s doesn't exist." % user)
return uid
@@ -49,10 +49,10 @@ def _coerce_gid(group):
try:
gid = int(group)
except ValueError:
- try:
- gid = grp.getgrnam(group).gr_gid
- except KeyError:
- raise Fail("Group %s doesn't exist." % group)
+ #try:
+ # gid = grp.getgrnam(group).gr_gid
+ #except KeyError:
+ raise Fail("Group %s doesn't exist." % group)
return gid
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/providers/windows/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/windows/__init__.py
new file mode 100644
index 0000000..49fddbd
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/__init__.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/providers/windows/service.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/service.py b/slider-agent/src/main/python/resource_management/core/providers/windows/service.py
new file mode 100644
index 0000000..4e73a2d
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/service.py
@@ -0,0 +1,65 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+from resource_management.core.providers import Provider
+from resource_management.core.base import Fail
+import win32service
+import time
+
+
+_schSCManager = win32service.OpenSCManager(None, None, win32service.SC_MANAGER_ALL_ACCESS)
+
+
+class ServiceProvider(Provider):
+ def action_start(self):
+ self._service_handle = self._service_handle if hasattr(self, "_service_handle") else \
+ win32service.OpenService(_schSCManager, self.resource.service_name, win32service.SERVICE_ALL_ACCESS)
+ if not self.status():
+ win32service.StartService(self._service_handle, None)
+ self.wait_status(win32service.SERVICE_RUNNING)
+
+ def action_stop(self):
+ self._service_handle = self._service_handle if hasattr(self, "_service_handle") else \
+ win32service.OpenService(_schSCManager, self.resource.service_name, win32service.SERVICE_ALL_ACCESS)
+ if self.status():
+ win32service.ControlService(self._service_handle, win32service.SERVICE_CONTROL_STOP)
+ self.wait_status(win32service.SERVICE_STOPPED)
+
+ def action_restart(self):
+ self._service_handle = win32service.OpenService(_schSCManager, self.resource.service_name,
+ win32service.SERVICE_ALL_ACCESS)
+ self.action_stop()
+ self.action_start()
+
+ def action_reload(self):
+ raise Fail("Reload for Service resource not supported on windows")
+
+ def status(self):
+ if win32service.QueryServiceStatusEx(self._service_handle)["CurrentState"] == win32service.SERVICE_RUNNING:
+ return True
+ return False
+
+ def get_current_status(self):
+ return win32service.QueryServiceStatusEx(self._service_handle)["CurrentState"]
+
+ def wait_status(self, status, timeout=5):
+ begin = time.time()
+ while self.get_current_status() != status and (timeout == 0 or time.time() - begin < timeout):
+ time.sleep(1)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
new file mode 100644
index 0000000..1e8142e
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -0,0 +1,142 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+from resource_management.core.base import Fail
+from resource_management.core import ExecuteTimeoutException
+import time
+import os
+import subprocess
+
+
+def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None):
+ # TODO implement timeout, logoutput, wait_for_finish
+ proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ cwd=cwd, env=env, shell=False)
+ out = proc.communicate()[0].strip()
+ code = proc.returncode
+ if logoutput and out:
+ Logger.info(out)
+ return code, out
+
+# see msdn Icacls doc for rights
+def _set_file_acl(file, user, rights):
+ acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights)
+ acls_remove_cmd = "icacls {0} /remove {1}".format(file, user)
+ code, out = _call_command(acls_remove_cmd)
+ if code != 0:
+ raise Fail("Can not remove rights for path {0} and user {1}".format(file, user))
+ code, out = _call_command(acls_modify_cmd)
+ if code != 0:
+ raise Fail("Can not set rights {0} for path {1} and user {2}".format(file, user))
+ else:
+ return
+
+class FileProvider(Provider):
+ def action_create(self):
+ path = self.resource.path
+
+ if os.path.isdir(path):
+ raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
+
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
+
+ write = False
+ content = self._get_content()
+ if not os.path.exists(path):
+ write = True
+ reason = "it doesn't exist"
+ elif self.resource.replace:
+ if content is not None:
+ with open(path, "rb") as fp:
+ old_content = fp.read()
+ if content != old_content:
+ write = True
+ reason = "contents don't match"
+ if self.resource.backup:
+ self.resource.env.backup_file(path)
+
+ if write:
+ Logger.info("Writing %s because %s" % (self.resource, reason))
+ with open(path, "wb") as fp:
+ if content:
+ fp.write(content)
+
+ _set_file_acl(self.resource.path, self.resource.owner, self.resource.mode)
+
+ def action_delete(self):
+ path = self.resource.path
+
+ if os.path.isdir(path):
+ raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
+
+ if os.path.exists(path):
+ Logger.info("Deleting %s" % self.resource)
+ os.unlink(path)
+
+ def _get_content(self):
+ content = self.resource.content
+ if content is None:
+ return None
+ elif isinstance(content, basestring):
+ return content
+ elif hasattr(content, "__call__"):
+ return content()
+ raise Fail("Unknown source type for %s: %r" % (self, content))
+
+class ExecuteProvider(Provider):
+ def action_run(self):
+ if self.resource.creates:
+ if os.path.exists(self.resource.creates):
+ return
+
+ Logger.debug("Executing %s" % self.resource)
+
+ if self.resource.path != []:
+ if not self.resource.environment:
+ self.resource.environment = {}
+
+ self.resource.environment['PATH'] = os.pathsep.join(self.resource.path)
+
+ for i in range(0, self.resource.tries):
+ try:
+ _call_command(self.resource.command, logoutput=self.resource.logoutput,
+ cwd=self.resource.cwd, env=self.resource.environment,
+ wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout)
+ break
+ except Fail as ex:
+ if i == self.resource.tries - 1: # last try
+ raise ex
+ else:
+ Logger.info("Retrying after %d seconds. Reason: %s" % (self.resource.try_sleep, str(ex)))
+ time.sleep(self.resource.try_sleep)
+ except ExecuteTimeoutException:
+ err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (
+ self.resource.command, self.resource.timeout)
+
+ if self.resource.on_timeout:
+ Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg))
+ _call_command(self.resource.on_timeout)
+ else:
+ raise Fail(err_msg)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py b/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
index ad30707..cc0610d 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
@@ -28,4 +28,4 @@ from resource_management.libraries.functions.check_process_status import *
from resource_management.libraries.functions.is_empty import *
from resource_management.libraries.functions.substitute_vars import *
from resource_management.libraries.functions.os_check import *
-from resource_management.libraries.functions.get_port_from_url import *
\ No newline at end of file
+from resource_management.libraries.functions.get_port_from_url import *
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
index abfceb8..8c11d93 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
@@ -31,41 +31,65 @@ def linux_distribution():
PYTHON_VER = sys.version_info[0] * 10 + sys.version_info[1]
if PYTHON_VER < 26:
- linux_distribution = platform.dist()
+ (distname, version, id) = platform.dist()
else:
- linux_distribution = platform.linux_distribution()
+ (distname, version, id) = platform.linux_distribution()
- return linux_distribution
+ return (platform.system(), os.name, distname, version, id)
+def windows_distribution():
+ # Only support Windows Server 64 bit
+ (win_release, win_version, win_csd, win_ptype) = platform.win32_ver()
+ #if win_version
+ return (platform.system(), os.name, win_release, win_version, win_ptype)
class OS_CONST_TYPE(type):
+ # os platforms
+ LINUX_OS = 'linux'
+ WINDOWS_OS = 'windows'
+
# os families
REDHAT_FAMILY = 'redhat'
DEBIAN_FAMILY = 'debian'
SUSE_FAMILY = 'suse'
+ WINSRV_FAMILY = 'winsrv'
# Declare here os type mapping
OS_FAMILY_COLLECTION = [
- {'name': REDHAT_FAMILY,
- 'os_list':
- ['redhat', 'fedora', 'centos', 'oraclelinux',
- 'ascendos', 'amazon', 'xenserver', 'oel', 'ovs',
- 'cloudlinux', 'slc', 'scientific', 'psbm',
- 'centos linux']
- },
- {'name': DEBIAN_FAMILY,
- 'os_list': ['ubuntu', 'debian']
- },
- {'name': SUSE_FAMILY,
- 'os_list': ['sles', 'sled', 'opensuse', 'suse']
- }
- ]
+ {'name': REDHAT_FAMILY,
+ 'os_list':
+ ['redhat', 'fedora', 'centos', 'oraclelinux',
+ 'ascendos', 'amazon', 'xenserver', 'oel', 'ovs',
+ 'cloudlinux', 'slc', 'scientific', 'psbm',
+ 'centos linux']
+ },
+ {'name': DEBIAN_FAMILY,
+ 'os_list': ['ubuntu', 'debian']
+ },
+ {'name': SUSE_FAMILY,
+ 'os_list': ['sles', 'sled', 'opensuse', 'suse']
+ }
+ ]
+ WIN_OS_FAMILY_COLLECTION = [
+ {'name': WINSRV_FAMILY,
+ 'os_list':
+ ['2008Server', '2012Server']
+ },
+ ]
# Would be generated from Family collection definition
OS_COLLECTION = []
def __init__(cls, name, bases, dct):
- for item in cls.OS_FAMILY_COLLECTION:
- cls.OS_COLLECTION += item['os_list']
+ if platform.system() == 'Windows':
+ for item in cls.WIN_OS_FAMILY_COLLECTION:
+ cls.OS_COLLECTION += item['os_list']
+ else:
+ if platform.system() == 'Mac':
+ raise Exception("MacOS not supported. Exiting...")
+ else:
+ dist = linux_distribution()
+ for item in cls.OS_FAMILY_COLLECTION:
+ cls.OS_COLLECTION += item['os_list']
def __getattr__(cls, name):
"""
@@ -81,17 +105,45 @@ class OS_CONST_TYPE(type):
else:
raise Exception("Unknown class property '%s'" % name)
+def get_os_distribution():
+ if platform.system() == 'Windows':
+ dist = windows_distribution()
+ else:
+ if platform.system() == 'Darwin':
+ dist = ("Darwin", "TestOnly", "1.1.1", "1.1.1", "1.1")
+ else:
+ # Linux
+ # Read content from /etc/*-release file
+ # Full release name
+ dist = linux_distribution()
+ return dist
class OSConst:
__metaclass__ = OS_CONST_TYPE
class OSCheck:
+ _dist = get_os_distribution()
+
+ @staticmethod
+ def get_os_os():
+ """
+ Return values:
+ windows, linux
+
+ In case cannot detect - exit.
+ """
+ # Read content from /etc/*-release file
+ # Full release name
+ os_os = OSCheck._dist[0].lower()
+
+ return os_os
@staticmethod
def get_os_type():
"""
Return values:
+ 2008server, 2012server,
redhat, fedora, centos, oraclelinux, ascendos,
amazon, xenserver, oel, ovs, cloudlinux, slc, scientific, psbm,
ubuntu, debian, sles, sled, opensuse, suse ... and others
@@ -100,15 +152,14 @@ class OSCheck:
"""
# Read content from /etc/*-release file
# Full release name
- dist = linux_distribution()
- operatingSystem = dist[0].lower()
+ operatingSystem = OSCheck._dist[2].lower()
# special cases
if os.path.exists('/etc/oracle-release'):
return 'oraclelinux'
elif operatingSystem.startswith('suse linux enterprise server'):
return 'sles'
- elif operatingSystem.startswith('red hat enterprise linux server'):
+ elif operatingSystem.startswith('red hat enterprise linux'):
return 'redhat'
if operatingSystem != '':
@@ -124,11 +175,14 @@ class OSCheck:
In case cannot detect raises exception( from self.get_operating_system_type() ).
"""
- os_family = OSCheck.get_os_type()
- for os_family_item in OSConst.OS_FAMILY_COLLECTION:
- if os_family in os_family_item['os_list']:
- os_family = os_family_item['name']
- break
+ if(OSCheck._dist[0] == 'Windows'):
+ os_family = OSConst.WIN_OS_FAMILY_COLLECTION[0]['name']
+ else:
+ os_family = OSCheck.get_os_type()
+ for os_family_item in OSConst.OS_FAMILY_COLLECTION:
+ if os_family in os_family_item['os_list']:
+ os_family = os_family_item['name']
+ break
return os_family.lower()
@@ -139,15 +193,12 @@ class OSCheck:
In case cannot detect raises exception.
"""
- # Read content from /etc/*-release file
- # Full release name
- dist = linux_distribution()
- dist = dist[1]
+ dist = OSCheck._dist[3]
if dist:
return dist
else:
- raise Exception("Cannot detect os version. Exiting...")
+ raise Exception("Cannot detect os version from " + repr(OSCheck._dist) + " Exiting...")
@staticmethod
def get_os_major_version():
@@ -165,8 +216,7 @@ class OSCheck:
In case cannot detect raises exception.
"""
- dist = linux_distribution()
- dist = dist[2].lower()
+ dist = OSCheck._dist[4].lower()
if dist:
return dist
@@ -217,6 +267,54 @@ class OSCheck:
pass
return False
+ @staticmethod
+ def is_windows_family():
+ """
+ Return true if it is so or false if not
+
+ This is safe check for windows family, doesn't generate exception
+ """
+ try:
+ if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
+ return True
+ except Exception:
+ pass
+ return False
+
+ @staticmethod
+ def is_linux_os():
+ """
+ Return true if it is so or false if not
+
+ This is safe check for linux os, doesn't generate exception
+ """
+ try:
+ if OSCheck.get_os_os() == OSConst.LINUX_OS:
+ return True
+ except Exception:
+ pass
+ return False
+
+ @staticmethod
+ def is_windows_os():
+ """
+ Return true if it is so or false if not
+
+ This is safe check for windows os, doesn't generate exception
+ """
+ try:
+ if OSCheck.get_os_os() == OSConst.WINDOWS_OS:
+ return True
+ except Exception:
+ pass
+ return False
+
+
+# OS info
+OS_VERSION = OSCheck().get_os_major_version()
+OS_TYPE = OSCheck.get_os_type()
+OS_FAMILY = OSCheck.get_os_family()
+OS_OS = OSCheck.get_os_os()
if __name__ == "__main__":
main()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py b/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py
index 973958b..1dfeef7 100644
--- a/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py
@@ -30,6 +30,8 @@ PROVIDERS = dict(
debian=dict(
Repository="resource_management.libraries.providers.repository.DebianRepositoryProvider",
),
+ winsrv=dict(
+ ),
default=dict(
ExecuteHadoop="resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider",
TemplateConfig="resource_management.libraries.providers.template_config.TemplateConfigProvider",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py b/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py
index 5817879..7750d25 100644
--- a/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py
@@ -42,7 +42,7 @@ class MonitorWebserverProvider(Provider):
def get_serivice_params(self):
self.system = System.get_instance()
- if self.system.os_family == "suse":
+ if self.system.os_family in ["suse","debian"]:
self.service_name = "apache2"
self.httpd_conf_dir = '/etc/apache2'
else:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/script.py b/slider-agent/src/main/python/resource_management/libraries/script/script.py
index 624d65e..00b80b4 100644
--- a/slider-agent/src/main/python/resource_management/libraries/script/script.py
+++ b/slider-agent/src/main/python/resource_management/libraries/script/script.py
@@ -24,6 +24,7 @@ import os
import sys
import json
import logging
+import shutil
from resource_management.core.environment import Environment
from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
@@ -32,6 +33,7 @@ from resource_management.core.resources import Tarball
from resource_management.core.resources import Directory
from resource_management.libraries.script.config_dictionary import ConfigDictionary
from resource_management.libraries.script.repo_installer import RepoInstaller
+from resource_management.core.logger import Logger
USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL>
@@ -178,12 +180,23 @@ class Script(object):
Directory(install_location, action = "delete")
Directory(install_location)
Tarball(tarball, location=install_location)
+ elif type.lower() == "folder":
+ if name.startswith(os.path.sep):
+ src = name
+ else:
+ basedir = env.config.basedir
+ src = os.path.join(basedir, name)
+ dest = config['configurations']['global']['app_install_dir']
+ Directory(dest, action = "delete")
+ Logger.info("Copying from " + src + " to " + dest)
+ shutil.copytree(src, dest)
else:
if not repo_installed:
RepoInstaller.install_repos(config)
repo_installed = True
Package(name)
- except KeyError:
+ except KeyError, e:
+ Logger.info("Error installing packages. " + repr(e))
pass # No reason to worry
#RepoInstaller.remove_repos(config)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-agent/src/test/python/agent/TestShell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestShell.py b/slider-agent/src/test/python/agent/TestShell.py
index 8ef72b3..8caed7b 100644
--- a/slider-agent/src/test/python/agent/TestShell.py
+++ b/slider-agent/src/test/python/agent/TestShell.py
@@ -42,12 +42,6 @@ class TestShell(unittest.TestCase):
return linux_dist
- @patch("os.setuid")
- def test_changeUid(self, os_setUIDMock):
- shell.threadLocal.uid = 9999
- shell.changeUid()
- self.assertTrue(os_setUIDMock.called)
-
def test_kill_process_with_children(self):
dist = self.linux_distribution()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 6479019..7561c9d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -55,6 +55,8 @@ import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.OSPackage;
+import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -756,9 +758,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setRole(roleName);
Map<String, String> hostLevelParams = new TreeMap<>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
- hostLevelParams.put(PACKAGE_LIST, "[{\"type\":\"tarball\",\"name\":\"" +
- appConf.getGlobalOptions().getMandatoryOption(
- PACKAGE_LIST) + "\"}]");
+ hostLevelParams.put(PACKAGE_LIST, getPackageList());
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -770,6 +770,31 @@ public class AgentProviderService extends AbstractProviderService implements
response.addExecutionCommand(cmd);
}
+ private String getPackageList() {
+ String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}";
+ String pkgListFormatString = "[%s]";
+ List<String> packages = new ArrayList();
+ Application application = getMetainfo().getApplication();
+ if (application != null) {
+ List<OSSpecific> osSpecifics = application.getOSSpecifics();
+ if (osSpecifics != null && osSpecifics.size() > 0) {
+ for (OSSpecific osSpecific : osSpecifics) {
+ if (osSpecific.getOsType().equals("any")) {
+ for (OSPackage osPackage : osSpecific.getPackages()) {
+ packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
+ }
+ }
+ }
+ }
+ }
+
+ if (packages.size() > 0) {
+ return String.format(pkgListFormatString, StringUtils.join(",", packages));
+ } else {
+ return "[]";
+ }
+ }
+
private void prepareExecutionCommand(ExecutionCommand cmd) {
cmd.setTaskId(taskId.incrementAndGet());
cmd.setCommandId(cmd.getTaskId() + "-1");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee00fccb/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
index 5238d90..109843a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -143,13 +143,13 @@ public class SecurityUtils {
try {
Set<PosixFilePermission> perms =
PosixFilePermissions.fromString("rwx------");
- Files.setPosixFilePermissions(Paths.get(secDirFile.toURI()), perms);
- Files.setPosixFilePermissions(Paths.get(dbDir.toURI()), perms);
- Files.setPosixFilePermissions(Paths.get(newCertsDir.toURI()), perms);
+// Files.setPosixFilePermissions(Paths.get(secDirFile.toURI()), perms);
+// Files.setPosixFilePermissions(Paths.get(dbDir.toURI()), perms);
+// Files.setPosixFilePermissions(Paths.get(newCertsDir.toURI()), perms);
File indexFile = new File(dbDir, "index.txt");
indexFile.createNewFile();
- SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath());
+ SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath().replace('\\', '/'));
} catch (IOException e) {
LOG.error("Unable to create SSL configuration directories/files", e);