You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/07/12 19:40:44 UTC
[15/37] incubator-slider git commit: SLIDER-1115 stdout/stderr files
have errors
SLIDER-1115 stdout/stderr files have errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66257d77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66257d77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66257d77
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 66257d771055a96ba0a4d403f41dac4b3da994cf
Parents: e6999d0
Author: Yu Liu <yu...@apache.org>
Authored: Wed Jun 1 11:03:17 2016 -0700
Committer: Yu Liu <yu...@apache.org>
Committed: Wed Jun 1 11:03:17 2016 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/agent/ActionQueue.py | 4 ++--
slider-agent/src/main/python/agent/Constants.py | 4 ++--
.../src/main/python/agent/DockerManager.py | 16 +++++++---------
.../src/main/python/agent/YarnDockerManager.py | 17 ++++++++---------
.../core/providers/windows/system.py | 14 ++++++--------
.../main/python/resource_management/core/shell.py | 17 +++++++----------
6 files changed, 32 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 107d6c3..be6af53 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -72,8 +72,8 @@ class ActionQueue(threading.Thread):
self.customServiceOrchestrator = CustomServiceOrchestrator(config,
controller,
self.queueOutAgentToggleLogger)
- self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
- self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
+ self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator)
+ self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator)
def stop(self):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/Constants.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index 6ede66e..524db6a 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -34,7 +34,7 @@ ZK_QUORUM="zk_quorum"
ZK_REG_PATH="zk_reg_path"
AUTO_GENERATED="auto_generated"
MAX_AM_CONNECT_RETRIES = 10
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application_"
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application"
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = ".out"
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application_"
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application"
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = ".err"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/DockerManager.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/DockerManager.py b/slider-agent/src/main/python/agent/DockerManager.py
index 023a163..48085fe 100644
--- a/slider-agent/src/main/python/agent/DockerManager.py
+++ b/slider-agent/src/main/python/agent/DockerManager.py
@@ -31,9 +31,10 @@ class DockerManager():
stored_command = ''
container_id = ''
- def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator):
self.tmpdir = tmpdir
self.workroot = workroot
+ self.logDir = logDir
self.customServiceOrchestrator = customServiceOrchestrator
def execute_command(self, command, store_command=False):
@@ -128,14 +129,11 @@ class DockerManager():
docker_command = self.add_additional_param_to_command(docker_command, additional_param)
#adding redirecting stdout stderr to file
logger.info("docker run command: " + str(docker_command))
- outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
-
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
return self.execute_command_on_linux(docker_command, stdoutFile, stderrFile)
def add_docker_run_options_to_command(self, docker_command, options):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/YarnDockerManager.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py
index abdb2fb..4a4d6de 100644
--- a/slider-agent/src/main/python/agent/YarnDockerManager.py
+++ b/slider-agent/src/main/python/agent/YarnDockerManager.py
@@ -24,6 +24,7 @@ import subprocess
import Constants
import time
import traceback
+from AgentConfig import AgentConfig
from resource_management import *
logger = logging.getLogger()
@@ -33,9 +34,10 @@ class YarnDockerManager(Script):
stored_command = ''
container_id = ''
- def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator):
self.tmpdir = tmpdir
self.workroot = workroot
+ self.logDir = logDir
self.customServiceOrchestrator = customServiceOrchestrator
def execute_command(self, command, store_command=False):
@@ -168,14 +170,11 @@ class YarnDockerManager(Script):
#extracting param needed by docker run from the command passed from AM
startCommand = self.extract_config_from_command(command, 'docker.startCommand')
#adding redirecting stdout stderr to file
- outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
-
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
returncode,out,err = self.execute_command_on_linux(startCommand, False,
stdoutFile, stderrFile)
return returncode,out,err
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index b85c020..a3d9126 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -30,9 +30,9 @@ import subprocess
import shutil
from resource_management.libraries.script import Script
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log'
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err'
def _merge_env(env1, env2, merge_keys=['PYTHONPATH']):
@@ -74,14 +74,12 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=
# TODO implement user
Logger.info("Executing %s" % (command))
#adding redirecting stdout stderr to file
- outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
- errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+ errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True,
cwd=cwd, env=env, shell=False)
code = None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index f21dbbf..4a383c7 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -31,9 +31,9 @@ from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
import time
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log'
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err'
def checked_call(command, logoutput=False,
@@ -69,14 +69,11 @@ def _call(command, logoutput=False, throw_on_failure=True,
"""
command = ["/bin/bash","--login","-c", command]
#adding redirecting stdout stderr to file
- outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
-
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True,
cwd=cwd, env=env, shell=False,