You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by fb...@apache.org on 2014/12/29 21:41:02 UTC
[2/2] ambari git commit: AMBARI-8770 [WinGA] Add windows server
assembly for AMS
AMBARI-8770 [WinGA] Add windows server assembly for AMS
Windows build for 2 Ambari Metrics service: Host Monitoring and Timeline Service (Collector).
+ Added Windows profiles to the Maven project files
+ Added the necessary Windows assemblies
+ Created Windows service skeletons
+ Host Monitoring: added OS-independent process termination handler
+ Collector: added debugging support for the Java process
+ Fixed services shutdown, especially when joining spawned threads
Windows-specific:
+ Moved the assembly descriptors to ambari-metrics-assembly
+ Fixed comments in the configuration files
+ Added soft dependencies on the embedded HBase service
+ Added support for the embedded HBase service setup
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19e972cf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19e972cf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19e972cf
Branch: refs/heads/trunk
Commit: 19e972cf98423200bf30518803f066669f2dcd2a
Parents: e7303b1
Author: Florian Barca <fb...@hortonworks.com>
Authored: Mon Dec 29 12:40:40 2014 -0800
Committer: Florian Barca <fb...@hortonworks.com>
Committed: Mon Dec 29 12:40:40 2014 -0800
----------------------------------------------------------------------
ambari-agent/conf/windows/service_wrapper.py | 2 +-
.../src/main/python/ambari_commons/os_utils.py | 17 +-
.../main/python/ambari_commons/os_windows.py | 21 +-
.../src/main/python/ambari_commons/xml_utils.py | 33 +++
ambari-metrics/ambari-metrics-assembly/pom.xml | 77 ++++++-
.../src/main/assembly/collector-windows.xml | 101 ++++++++
.../src/main/assembly/monitor-windows.xml | 84 +++++++
.../src/main/assembly/monitor.xml | 2 +-
.../src/main/assembly/sink-windows.xml | 60 +++++
.../conf/windows/ambari-metrics-monitor.cmd | 17 ++
.../conf/windows/metric_groups.conf | 19 ++
.../conf/windows/metric_monitor.ini | 30 +++
.../ambari-metrics-host-monitoring/pom.xml | 146 ++++++++----
.../src/main/python/amhm_service.py | 231 +++++++++++++++++++
.../src/main/python/core/__init__.py | 3 +-
.../src/main/python/core/config_reader.py | 66 +++++-
.../src/main/python/core/controller.py | 27 ++-
.../src/main/python/core/emitter.py | 17 +-
.../src/main/python/core/stop_handler.py | 138 +++++++++++
.../src/main/python/main.py | 58 ++++-
.../conf/windows/ambari-metrics-collector.cmd | 17 ++
.../conf/windows/ams-env.cmd | 16 ++
.../conf/windows/ams-site.xml | 25 ++
.../conf/windows/ams.properties | 17 ++
.../conf/windows/log4j.properties | 29 +++
.../ambari-metrics-timelineservice/pom.xml | 31 +++
.../python/ambari_metrics_collector/__init__.py | 21 ++
.../ambari_metrics_collector/properties.py | 223 ++++++++++++++++++
.../serviceConfiguration.py | 152 ++++++++++++
.../src/main/python/amc_service.py | 174 ++++++++++++++
.../src/main/python/embedded_hbase_service.py | 201 ++++++++++++++++
.../src/main/python/main.py | 214 +++++++++++++++++
ambari-metrics/pom.xml | 27 +++
.../main/python/ambari-server-state/Entities.py | 17 +-
pom.xml | 2 +
35 files changed, 2210 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-agent/conf/windows/service_wrapper.py
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/windows/service_wrapper.py b/ambari-agent/conf/windows/service_wrapper.py
index 40be1d0..5eb06c4 100644
--- a/ambari-agent/conf/windows/service_wrapper.py
+++ b/ambari-agent/conf/windows/service_wrapper.py
@@ -92,7 +92,7 @@ class AmbariAgentService(AmbariService):
# Soft dependency on the Windows Time service
ensure_time_service_is_started()
- self.heartbeat_stop_handler = HeartbeatStopHandlers(self._heventSvcStop)
+ self.heartbeat_stop_handler = HeartbeatStopHandlers(AmbariAgentService._heventSvcStop)
self.ReportServiceStatus(win32service.SERVICE_RUNNING)
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-common/src/main/python/ambari_commons/os_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py
index 3f4819d..942a920 100644
--- a/ambari-common/src/main/python/ambari_commons/os_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/os_utils.py
@@ -48,14 +48,17 @@ def quote_path(filepath):
filepath_ret = filepath
return filepath_ret
-def search_file(filename, search_path, pathsep=os.pathsep):
- """ Given a search path, find file with requested name """
+def _search_file(filename, search_path, pathsep):
for path in string.split(search_path, pathsep):
candidate = os.path.join(path, filename)
if os.path.exists(candidate):
return os.path.abspath(candidate)
return None
+def search_file(filename, search_path, pathsep=os.pathsep):
+ """ Given a search path, find file with requested name """
+ return _search_file(filename, search_path, pathsep)
+
def copy_file(src, dest_file):
try:
shutil.copyfile(src, dest_file)
@@ -103,9 +106,7 @@ def get_password(prompt):
return os_getpass(prompt)
def find_in_path(file):
- dirs = os.environ["PATH"].split(os.pathsep)
- for dir in dirs:
- full_path = os.path.join(dir, file)
- if os.path.exists(full_path):
- return full_path
- raise Exception("File {} not found in PATH".format(file))
\ No newline at end of file
+ full_path = _search_file(file, os.environ["PATH"], os.pathsep)
+ if full_path is None:
+ raise Exception("File {0} not found in PATH".format(file))
+ return full_path
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-common/src/main/python/ambari_commons/os_windows.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py
index 2fb98e4..7918a2f 100644
--- a/ambari-common/src/main/python/ambari_commons/os_windows.py
+++ b/ambari-common/src/main/python/ambari_commons/os_windows.py
@@ -402,7 +402,7 @@ class WinService(win32serviceutil.ServiceFramework):
# _svc_display_name_ = The service display name
# _svc_description_ = The service description
- _heventSvcStop = win32event.CreateEvent(None, 0, 0, None)
+ _heventSvcStop = win32event.CreateEvent(None, 1, 0, None)
_hmtxOut = win32event.CreateMutex(None, False, None) #[fbarca] Python doesn't support critical sections
def __init__(self, *args):
@@ -418,17 +418,18 @@ class WinService(win32serviceutil.ServiceFramework):
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
- win32event.SetEvent(self._heventSvcStop)
+ win32event.SetEvent(WinService._heventSvcStop)
# Service code entry point. Override it to implement the intended functionality.
def ServiceMain(self):
#Default implementation, does nothing.
- win32event.WaitForSingleObject(self._heventSvcStop, win32event.INFINITE)
+ win32event.WaitForSingleObject(WinService._heventSvcStop, win32event.INFINITE)
pass
- def DefCtrlCHandler(self):
+ @staticmethod
+ def DefCtrlCHandler():
print_info_msg("Ctrl+C handler invoked. Stopping.")
- win32event.SetEvent(self._heventSvcStop)
+ win32event.SetEvent(WinService._heventSvcStop)
pass
#username domain\\username : The Username the service is to run under
@@ -438,7 +439,7 @@ class WinService(win32serviceutil.ServiceFramework):
#perfmonini file: .ini file to use for registering performance monitor data
#perfmondll file: .dll file to use when querying the service for performance data, default = perfmondata.dll
@classmethod
- def Install(cls, startupMode = "auto", username = None, password = None, interactive = False,
+ def Install(cls, classPath = None, startupMode = "auto", username = None, password = None, interactive = False,
perfMonIni = None, perfMonDll = None):
installArgs = [sys.argv[0], "--startup=" + startupMode]
if username is not None and username:
@@ -452,7 +453,8 @@ class WinService(win32serviceutil.ServiceFramework):
if perfMonDll is not None and perfMonDll:
installArgs.append("--perfmondll=" + perfMonDll)
installArgs.append("install")
- win32serviceutil.HandleCommandLine(cls, None, installArgs)
+
+ win32serviceutil.HandleCommandLine(cls, classPath, installArgs)
@classmethod
def Start(cls, waitSecs = 30):
@@ -483,11 +485,12 @@ class WinService(win32serviceutil.ServiceFramework):
def CheckForStop(self):
#Check for stop event to be signaled
- return win32event.WAIT_OBJECT_0 == win32event.WaitForSingleObject(self._heventSvcStop, 1)
+ return win32event.WAIT_OBJECT_0 == win32event.WaitForSingleObject(WinService._heventSvcStop, 1)
def _StopOrWaitForChildProcessToFinish(self, childProcess):
#Wait for the child process to finish or for the stop event to be signaled
- if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], False, win32event.INFINITE)):
+ if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([WinService._heventSvcStop, childProcess._handle],
+ False, win32event.INFINITE)):
# The OS only detaches the child process when the master process exits.
# We must kill it manually.
try:
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-common/src/main/python/ambari_commons/xml_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/xml_utils.py b/ambari-common/src/main/python/ambari_commons/xml_utils.py
new file mode 100644
index 0000000..31b2968
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_commons/xml_utils.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+#* Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+
+class ConvertToXml:
+ "Template class, allow to output fields in xml format"
+ def getField(self):
+ return [name for name, obj in inspect.getmembers(self)
+ if not name.startswith("__") and not inspect.isroutine(obj)]
+
+ def attributesToXml(self):
+ result = ""
+ listOfAttr = self.getField()
+ for attrName in listOfAttr:
+ result += "<" + attrName + ">"
+ result += str(getattr(self, attrName))
+ result += "</" + attrName + ">\n"
+ return result
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml
index 1e020cb..0a47b8a 100644
--- a/ambari-metrics/ambari-metrics-assembly/pom.xml
+++ b/ambari-metrics/ambari-metrics-assembly/pom.xml
@@ -26,9 +26,9 @@
<version>0.1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
-
<artifactId>ambari-metrics-assembly</artifactId>
<packaging>pom</packaging>
+ <version>0.1.0-SNAPSHOT</version>
<properties>
<collector.dir>${project.basedir}/../ambari-metrics-timelineservice</collector.dir>
@@ -48,6 +48,33 @@
<build>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>parse-version</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>parse-version</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>ambariVersion</name>
+ <value>${project.version}</value>
+ <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
+ <replacement>$1.$2.$3</replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
@@ -94,7 +121,7 @@
<finalName>ambari-metrics-collector-${project.version}</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
- <descriptor>src/main/assembly/collector.xml</descriptor>
+ <descriptor>${assemblydescriptor.collector}</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
@@ -110,7 +137,7 @@
<finalName>ambari-metrics-monitor-${project.version}</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
- <descriptor>src/main/assembly/monitor.xml</descriptor>
+ <descriptor>${assemblydescriptor.monitor}</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
@@ -126,7 +153,7 @@
<finalName>ambari-metrics-hadoop-sink-${project.version}</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
- <descriptor>src/main/assembly/sink.xml</descriptor>
+ <descriptor>${assemblydescriptor.sink}</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
@@ -710,6 +737,48 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>windows</id>
+ <activation>
+ <os>
+ <family>win</family>
+ </os>
+ </activation>
+ <properties>
+ <envClassifier>win</envClassifier>
+ <dirsep>\</dirsep>
+ <pathsep>;</pathsep>
+ <executable.python>python</executable.python>
+ <executable.shell>cmd</executable.shell>
+ <fileextension.shell>cmd</fileextension.shell>
+ <fileextension.dot.shell-default>.cmd</fileextension.dot.shell-default>
+ <assemblydescriptor.collector>src/main/assembly/collector-windows.xml</assemblydescriptor.collector>
+ <assemblydescriptor.monitor>src/main/assembly/monitor-windows.xml</assemblydescriptor.monitor>
+ <assemblydescriptor.sink>src/main/assembly/sink-windows.xml</assemblydescriptor.sink>
+ <packagingFormat>jar</packagingFormat>
+ </properties>
+ </profile>
+ <profile>
+ <id>linux</id>
+ <activation>
+ <os>
+ <family>unix</family>
+ </os>
+ </activation>
+ <properties>
+ <envClassifier>linux</envClassifier>
+ <dirsep>/</dirsep>
+ <pathsep>:</pathsep>
+ <executable.python>${project.basedir}/../ambari-common/src/main/unix/ambari-python-wrap</executable.python>
+ <executable.shell>sh</executable.shell>
+ <fileextension.shell>sh</fileextension.shell>
+ <fileextension.dot.shell-default></fileextension.dot.shell-default>
+ <assemblydescriptor.collector>src/main/assembly/collector.xml</assemblydescriptor.collector>
+ <assemblydescriptor.monitor>src/main/assembly/monitor.xml</assemblydescriptor.monitor>
+ <assemblydescriptor.sink>src/main/assembly/sink.xml</assemblydescriptor.sink>
+ <packagingFormat>jar</packagingFormat>
+ </properties>
+ </profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
new file mode 100644
index 0000000..3faa085
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
+ <id>collector</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${collector.dir}/target/embedded/${hbase.folder}</directory>
+ <outputDirectory>hbase</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${collector.dir}/conf/windows</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>ambari-metrics-collector.cmd</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${collector.dir}/conf/windows</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>ams.properties</include>
+ <include>ams-env.cmd</include>
+ <include>ams-site.xml</include>
+ <include>log4j.properties</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${collector.dir}/target/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${collector.dir}/src/main/python</directory>
+ <outputDirectory>/sbin</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${collector.dir}/src/main/python/ambari_metrics_collector</directory>
+ <outputDirectory>/sbin/ambari_metrics_collector</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons</directory>
+ <outputDirectory>/sbin/ambari_commons</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons/resources</directory>
+ <outputDirectory>/sbin/ambari_commons/resources</outputDirectory>
+ <includes>
+ <include>*.json</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <unpack>false</unpack>
+ <outputDirectory>hbase/lib</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>org.antlr:antlr*</include>
+ <include>org.apache.phoenix:phoenix-core</include>
+ </includes>
+ </dependencySet>
+ <dependencySet>
+ <unpack>false</unpack>
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>org.apache.ambari:ambari-metrics-timelineservice</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml
new file mode 100644
index 0000000..3b877a7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml
@@ -0,0 +1,84 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>windows-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${monitor.dir}/src/main/python</directory>
+ <outputDirectory>/sbin</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons</directory>
+ <outputDirectory>/sbin/ambari_commons</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons/resources</directory>
+ <outputDirectory>/sbin/ambari_commons/resources</outputDirectory>
+ <includes>
+ <include>*.json</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${monitor.dir}/src/main/python/core</directory>
+ <outputDirectory>/sbin/core</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${monitor.dir}/target/psutil_build</directory>
+ <outputDirectory>/sbin/psutil/build</outputDirectory>
+ <includes>
+ <include>*.egg</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${monitor.dir}/conf/windows</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>metric_groups.conf</include>
+ <include>metric_monitor.ini</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${monitor.dir}/conf/windows</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>ambari-metrics-monitor.cmd</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <excludes>
+ <exclude>*</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml
index 3513972..5944616 100644
--- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml
@@ -31,8 +31,8 @@
<outputDirectory>site-packages/resource_monitoring</outputDirectory>
</fileSet>
<fileSet>
- <outputDirectory>conf</outputDirectory>
<directory>${monitor.dir}/conf/unix</directory>
+ <outputDirectory>conf</outputDirectory>
<includes>
<include>metric_groups.conf</include>
<include>metric_monitor.ini</include>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml
new file mode 100644
index 0000000..a65d6f2
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
+ <id>hadoop-sink</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${hadoop-sink.dir}/src/main/conf</directory>
+ <outputDirectory>hadoop-sink/conf</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${flume-sink.dir}/src/main/conf</directory>
+ <outputDirectory>hadoop-sink/conf</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${storm-sink.dir}/src/main/conf</directory>
+ <outputDirectory>hadoop-sink/conf</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>${hadoop-sink.dir}/target/ambari-metrics-hadoop-sink-with-common-${project.version}.jar</source>
+ <outputDirectory>hadoop-sink</outputDirectory>
+ </file>
+ <file>
+ <source>${flume-sink.dir}/target/ambari-metrics-flume-sink-with-common-${project.version}.jar</source>
+ <outputDirectory>hadoop-sink</outputDirectory>
+ </file>
+ <file>
+ <source>${storm-sink.dir}/target/ambari-metrics-storm-sink-with-common-${project.version}.jar</source>
+ <outputDirectory>hadoop-sink</outputDirectory>
+ </file>
+ </files>
+
+
+
+
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd
new file mode 100644
index 0000000..115b6a6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd
@@ -0,0 +1,17 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@echo off
+python.exe -u %~dp0\sbin\amhm_service.py %* 2>&1
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf
new file mode 100644
index 0000000..6d5a62f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf
@@ -0,0 +1,19 @@
+{
+ "host_metric_groups": {
+ "all": {
+ "collect_every": "10",
+ "metrics": [
+ {
+ "name": "bytes_out",
+ "value_threshold": "128"
+ }
+ ]
+ }
+ },
+ "process_metric_groups": {
+ "": {
+ "collect_every": "15",
+ "metrics": []
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini
new file mode 100644
index 0000000..bc2b461
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini
@@ -0,0 +1,30 @@
+;
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+
+[default]
+debug_level = INFO
+metrics_server = {{ams_collector_host_single}}:{{ams_collector_port}}
+enable_time_threshold = false
+enable_value_threshold = false
+
+[emitter]
+send_interval = 60
+
+[collector]
+collector_sleep_interval = 5
+max_queue_size = 5000
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
index c213057..c2f322c 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
+++ b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
@@ -32,6 +32,7 @@
<resmonitor.install.dir>
/usr/lib/python2.6/site-packages/resource_monitoring
</resmonitor.install.dir>
+ <final.name>${project.artifactId}-${project.version}</final.name>
</properties>
<build>
<plugins>
@@ -82,55 +83,12 @@
<version>3.0</version>
</plugin>
<plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>${project.basedir}/../../ambari-project/src/main/assemblies/empty.xml</descriptor>
- </descriptors>
- </configuration>
- <executions>
- <execution>
- <id>build-tarball</id>
- <phase>none</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <id>psutils-compile</id>
- <phase>process-test-classes</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target name="psutils-compile">
- <exec dir="${basedir}/src/main/python/psutil" executable="${project.basedir}/../../ambari-common/src/main/unix/ambari-python-wrap" failonerror="true">
- <arg value="setup.py" />
- <arg value="build" />
- <arg value="--build-platlib" />
- <arg value="${basedir}/target/psutil_build" />
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<executions>
<execution>
<configuration>
- <executable>${project.basedir}/../../ambari-common/src/main/unix/ambari-python-wrap</executable>
+ <executable>${executable.python}</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
<argument>unitTests.py</argument>
@@ -153,6 +111,8 @@
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>conf/unix/metric_groups.conf</exclude>
+ <exclude>conf/windows/metric_groups.conf</exclude>
<exclude>src/main/python/psutil/**</exclude>
<exclude>.pydevproject</exclude>
</excludes>
@@ -168,4 +128,102 @@
</plugin>
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>windows</id>
+ <activation>
+ <os>
+ <family>win</family>
+ </os>
+ </activation>
+ <properties>
+ <envClassifier>win</envClassifier>
+ <dirsep>\</dirsep>
+ <pathsep>;</pathsep>
+ <executable.python>python</executable.python>
+ <executable.shell>cmd</executable.shell>
+ <fileextension.shell>cmd</fileextension.shell>
+ <fileextension.dot.shell-default>.cmd</fileextension.dot.shell-default>
+ <assemblydescriptor>src/main/assemblies/amhm-windows.xml</assemblydescriptor>
+ <packagingFormat>jar</packagingFormat>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>psutils-compile</id>
+ <phase>process-test-classes</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target name="psutils-compile">
+ <exec dir="${basedir}/src/main/python/psutil" executable="python" failonerror="true">
+ <arg value="setup.py" />
+ <arg value="bdist_egg" />
+ <arg value="--bdist-dir" />
+ <arg value="${basedir}/target/psutil_build_temp" />
+ <arg value="--dist-dir" />
+ <arg value="${basedir}/target/psutil_build" />
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>linux</id>
+ <activation>
+ <os>
+ <family>unix</family>
+ </os>
+ </activation>
+ <properties>
+ <envClassifier>linux</envClassifier>
+ <dirsep>/</dirsep>
+ <pathsep>:</pathsep>
+ <executable.python>${project.basedir}/../../ambari-common/src/main/unix/ambari-python-wrap</executable.python>
+ <executable.shell>sh</executable.shell>
+ <fileextension.shell>sh</fileextension.shell>
+ <fileextension.dot.shell-default></fileextension.dot.shell-default>
+ <assemblydescriptor>src/main/assemblies/empty.xml</assemblydescriptor>
+ <packagingFormat>jar</packagingFormat>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>psutils-compile</id>
+ <phase>process-test-classes</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target name="psutils-compile">
+ <exec dir="${basedir}/src/main/python/psutil" executable="python" failonerror="true">
+ <arg value="setup.py" />
+ <arg value="build" />
+ <arg value="--build-platlib" />
+ <arg value="${basedir}/target/psutil_build" />
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py
new file mode 100644
index 0000000..0f8daab
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import glob
+
+import optparse
+import os
+import sys
+
+from ambari_commons.ambari_service import AmbariService
+from ambari_commons.exceptions import FatalException, NonFatalException
+from ambari_commons.logging_utils import print_warning_msg, print_error_msg, print_info_msg
+from ambari_commons.os_utils import search_file, run_os_command
+from ambari_commons.os_windows import SvcStatusCallback
+from core.config_reader import SERVER_OUT_FILE, SERVICE_USERNAME_KEY, SERVICE_PASSWORD_KEY, \
+ SETUP_ACTION, START_ACTION, STOP_ACTION, RESTART_ACTION, STATUS_ACTION
+from core.stop_handler import bind_signal_handlers, StopHandler
+from main import server_process_main
+
+
+#
+# Windows-specific service implementation. This class will be instantiated directly by pythonservice.exe.
+#
+class AMHostMonitoringService(AmbariService):
+ AmbariService._svc_name_ = "AmbariMetricsHostMonitoring"
+ AmbariService._svc_display_name_ = "Ambari Metrics Host Monitoring"
+ AmbariService._svc_description_ = "Ambari Metrics Host Monitoring Service"
+
+ AmbariService._AdjustServiceVersion()
+
+ # Adds the necessary script dir to the Python's modules path.
+ # Modify this as the deployed product's dir structure changes.
+ def _adjustPythonPath(self, current_dir):
+ python_path = os.path.join(current_dir, "sbin")
+ sys.path.insert(0, python_path)
+ pass
+
+ def SvcDoRun(self):
+ scmStatus = SvcStatusCallback(self)
+
+ self.redirect_output_streams()
+
+ stopHandler = StopHandler(AMHostMonitoringService._heventSvcStop)
+ bind_signal_handlers(stopHandler)
+
+ AMHostMonitoringService.set_ctrl_c_handler(ctrlHandler)
+
+ server_process_main(stopHandler, scmStatus)
+ pass
+
+ def _InitOptionsParser(self):
+ return init_options_parser()
+
+ def redirect_output_streams(self):
+ self._RedirectOutputStreamsToFile(SERVER_OUT_FILE)
+ pass
+
+
+def ctrlHandler(ctrlType):
+ AMHostMonitoringService.DefCtrlCHandler()
+ return True
+
+
+def svcsetup():
+ AMHostMonitoringService.set_ctrl_c_handler(ctrlHandler)
+ # we don't save password between 'setup' runs, so we can't run Install every time. We run 'setup' only if user and
+ # password provided or if service not installed
+ if (SERVICE_USERNAME_KEY in os.environ and SERVICE_PASSWORD_KEY in os.environ):
+ AMHostMonitoringService.Install(username=os.environ[SERVICE_USERNAME_KEY], password=os.environ[SERVICE_PASSWORD_KEY])
+ elif AMHostMonitoringService.QueryStatus() == "not installed":
+ AMHostMonitoringService.Install()
+ pass
+
+
+#
+# Starts the Ambari Metrics Collector. The server can start as a service or standalone process.
+# args:
+# options.is_process = True - start the server as a process. For now, there is no restrictions for the number of
+# server instances that can run like this.
+# options.is_process = False - start the server in normal mode, as a Windows service. If the Ambari Metrics Collector
+# is not registered as a service, the function fails. By default, only one instance of the service can
+# possibly run.
+#
+def start(options):
+ AMHostMonitoringService.set_ctrl_c_handler(ctrlHandler)
+
+ if options.is_process:
+ #Run as a normal process. Invoke the ServiceMain directly.
+ stopHandler = StopHandler(AMHostMonitoringService._heventSvcStop)
+ bind_signal_handlers(stopHandler)
+ server_process_main(stopHandler)
+ else:
+ AMHostMonitoringService.Start()
+
+#
+# Stops the Ambari Metrics Collector. Ineffective when the server is started as a standalone process.
+#
+def stop():
+ AMHostMonitoringService.Stop()
+
+#
+# Prints the Ambari Metrics Collector service status.
+#
+def svcstatus(options):
+ options.exit_message = None
+
+ statusStr = AMHostMonitoringService.QueryStatus()
+ print "Ambari Metrics Collector is " + statusStr
+
+
+def init_options_parser():
+ parser = optparse.OptionParser(usage="usage: %prog action [options]", )
+ parser.add_option('-d', '--debug', action="store_true", dest='debug', default=False,
+ help="Start Ambari Metrics Host Monitoring in debug mode")
+ parser.add_option('-p', '--process', action="store_true", dest='is_process', default=False,
+ help="Start Ambari Metrics Host Monitoring as a normal process, not as a service")
+
+ # --help reserved for help
+ return parser
+
+def find_python_exe_path():
+ paths = "." + os.pathsep + os.environ["PATH"]
+
+ # Find python.exe by attempting to load it as a resource dll
+ python_path = search_file("python.exe", paths)
+ return os.path.dirname(python_path)
+
+
+def find_psutil_egg():
+ abs_subdir = os.path.join(os.getcwd(), "sbin", "psutil", "build")
+ egg_files = glob.glob(os.path.join(abs_subdir, "psutil*-win-amd64.egg"))
+ if egg_files is None or len(egg_files) == 0:
+ err = "Unable to find the expected psutil egg file in {0}. " \
+ "Verify that the installation carried out correctly.".format(abs_subdir)
+ raise FatalException(1, err)
+ if len(egg_files) > 1:
+ err = "Multiple psutil egg files found in {0}".format(abs_subdir)
+ print_warning_msg(err)
+ #Return the latest
+ return egg_files[len(egg_files) - 1]
+
+
+def setup_psutil():
+ python_exe_path = find_python_exe_path()
+ egg_file = find_psutil_egg()
+ cmd = [os.path.join(python_exe_path, "Scripts", "easy_install"), "--upgrade", egg_file]
+
+ retval, std_out, std_err = run_os_command(cmd)
+ if std_err is not None and std_err != '':
+ print_warning_msg(std_err)
+ print_info_msg(std_out)
+ if 0 != retval:
+ err = "Psutil egg installation failed. Exit code={0}, err output={1}".format(retval, std_err)
+ raise FatalException(1, err)
+
+ # Now the egg should be installed in the site_packages dir
+
+def win_main():
+ parser = init_options_parser()
+ (options, args) = parser.parse_args()
+
+ options.warnings = []
+ options.exit_message = None
+
+ if options.debug:
+ sys.frozen = 'windows_exe' # Fake py2exe so we can debug
+
+ if len(args) == 0:
+ print parser.print_help()
+ parser.error("No action entered")
+
+ action = args[0]
+
+ try:
+ if action == SETUP_ACTION:
+ setup_psutil()
+ svcsetup()
+ elif action == START_ACTION:
+ start(options)
+ elif action == STOP_ACTION:
+ stop()
+ elif action == RESTART_ACTION:
+ stop()
+ start(options)
+ elif action == STATUS_ACTION:
+ svcstatus(options)
+ else:
+ parser.error("Invalid action")
+
+ if options.warnings:
+ for warning in options.warnings:
+ print_warning_msg(warning)
+ pass
+ options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action
+ pass
+ except FatalException as e:
+ if e.reason is not None:
+ print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
+ sys.exit(e.code)
+ except NonFatalException as e:
+ options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action
+ if e.reason is not None:
+ print_warning_msg(e.reason)
+
+ if options.exit_message is not None:
+ print options.exit_message
+
+ sys.exit(0)
+
+if __name__ == "__main__":
+ try:
+ win_main()
+ except (KeyboardInterrupt, EOFError):
+ print("\nAborting ... Keyboard Interrupt.")
+ sys.exit(1)
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py
index 996120f..f607ec1 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py
@@ -19,8 +19,9 @@ limitations under the License.
"""
import os, sys
+
path = os.path.abspath(__file__)
-path = os.path.join(os.path.dirname(os.path.dirname(path)), "psutil/build/")
+path = os.path.normpath(os.path.join(os.path.dirname(path), "..", "psutil", "build"))
for dir in os.walk(path).next()[1]:
if 'lib' in dir:
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
index daabf37..463c98c 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
@@ -22,10 +22,67 @@ import ConfigParser
import StringIO
import json
import os
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl
+
+
+#
+# Abstraction for OS-dependent configuration defaults
+#
+class ConfigDefaults(object):
+ def get_config_file_path(self):
+ pass
+ def get_metric_file_path(self):
+ pass
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class ConfigDefaultsWindows(ConfigDefaults):
+ def __init__(self):
+ self._CONFIG_FILE_PATH = "conf\\metric_monitor.ini"
+ self._METRIC_FILE_PATH = "conf\\metric_groups.conf"
+ pass
+
+ def get_config_file_path(self):
+ return self._CONFIG_FILE_PATH
+ def get_metric_file_path(self):
+ return self._METRIC_FILE_PATH
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class ConfigDefaultsLinux(ConfigDefaults):
+ def __init__(self):
+ self._CONFIG_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_monitor.ini"
+ self._METRIC_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_groups.conf"
+ pass
+
+ def get_config_file_path(self):
+ return self._CONFIG_FILE_PATH
+ def get_metric_file_path(self):
+ return self._METRIC_FILE_PATH
+
+configDefaults = ConfigDefaults()
+
config = ConfigParser.RawConfigParser()
-CONFIG_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_monitor.ini"
-METRIC_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_groups.conf"
+
+CONFIG_FILE_PATH = configDefaults.get_config_file_path()
+METRIC_FILE_PATH = configDefaults.get_metric_file_path()
+
+OUT_DIR = os.path.join(os.sep, "var", "log", "ambari-metrics-host-monitoring")
+SERVER_OUT_FILE = OUT_DIR + os.sep + "ambari-metrics-host-monitoring.out"
+SERVER_LOG_FILE = OUT_DIR + os.sep + "ambari-metrics-host-monitoring.log"
+
+PID_DIR = os.path.join(os.sep, "var", "run", "ambari-metrics-host-monitoring")
+PID_OUT_FILE = PID_DIR + os.sep + "ambari-metrics-host-monitoring.pid"
+EXITCODE_OUT_FILE = PID_DIR + os.sep + "ambari-metrics-host-monitoring.exitcode"
+
+SERVICE_USERNAME_KEY = "TMP_AMHM_USERNAME"
+SERVICE_PASSWORD_KEY = "TMP_AMHM_PASSWORD"
+
+SETUP_ACTION = "setup"
+START_ACTION = "start"
+STOP_ACTION = "stop"
+RESTART_ACTION = "restart"
+STATUS_ACTION = "status"
config_content = """
[default]
@@ -96,6 +153,11 @@ class Configuration:
self.metric_groups = json.load(open(METRIC_FILE_PATH))
else:
print 'No metric configs found at {0}'.format(METRIC_FILE_PATH)
+ self.metric_groups = \
+ {
+ 'host_metric_groups': [],
+ 'process_metric_groups': []
+ }
pass
def getConfig(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
index 51f0980..e0ef804 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
@@ -33,7 +33,7 @@ logger = logging.getLogger()
class Controller(threading.Thread):
- def __init__(self, config):
+ def __init__(self, config, stop_handler):
# Process initialization code
threading.Thread.__init__(self)
logger.debug('Initializing Controller thread.')
@@ -48,27 +48,42 @@ class Controller(threading.Thread):
self.metric_collector = MetricsCollector(self.event_queue, self.application_metric_map)
self.server_url = config.get_server_address()
self.sleep_interval = config.get_collector_sleep_interval()
+ self._stop_handler = stop_handler
self.initialize_events_cache()
- self.emitter = Emitter(self.config, self.application_metric_map)
+ self.emitter = Emitter(self.config, self.application_metric_map, stop_handler)
+ self._t = None
def run(self):
logger.info('Running Controller thread: %s' % threading.currentThread().getName())
- # Wake every 5 seconds to push events to the queue
+
+ self.start_emitter()
+
+ # Wake every 5 seconds to push events to the queue
while True:
if (self.event_queue.full()):
logger.warn('Event Queue full!! Suspending further collections.')
else:
self.enqueque_events()
pass
- time.sleep(self.sleep_interval)
+ #Wait for the service stop event instead of sleeping blindly
+ if 0 == self._stop_handler.wait(self.sleep_interval):
+ logger.info('Shutting down Controller thread')
+ break
+
+ if not self._t is None:
+ self._t.cancel()
+ self._t.join(5)
+
+ #The emitter thread should have stopped by now, just ensure it has shut down properly
+ self.emitter.join(5)
pass
# TODO: Optimize to not use Timer class and use the Queue instead
def enqueque_events(self):
# Queue events for up to a minute
for event in self.events_cache:
- t = Timer(event.get_collect_interval(), self.metric_collector.process_event, args=(event,))
- t.start()
+ self._t = Timer(event.get_collect_interval(), self.metric_collector.process_event, args=(event,))
+ self._t.start()
pass
def initialize_events_cache(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
index be83250..c3fd543 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
@@ -32,12 +32,13 @@ class Emitter(threading.Thread):
"""
Wake up every send interval seconds and empty the application metric map.
"""
- def __init__(self, config, application_metric_map):
+ def __init__(self, config, application_metric_map, stop_handler):
threading.Thread.__init__(self)
logger.debug('Initializing Emitter thread.')
self.lock = threading.Lock()
self.collector_address = config.get_server_address()
self.send_interval = config.get_send_interval()
+ self._stop_handler = stop_handler
self.application_metric_map = application_metric_map
def run(self):
@@ -45,10 +46,16 @@ class Emitter(threading.Thread):
while True:
try:
self.submit_metrics()
- time.sleep(self.send_interval)
+ #Wait for the service stop event instead of sleeping blindly
+ if 0 == self._stop_handler.wait(self.send_interval):
+ logger.info('Shutting down Emitter thread')
+ return
except Exception, e:
logger.warn('Unable to emit events. %s' % str(e))
- time.sleep(self.RETRY_SLEEP_INTERVAL)
+ #Wait for the service stop event instead of sleeping blindly
+ if 0 == self._stop_handler.wait(self.RETRY_SLEEP_INTERVAL):
+ logger.info('Shutting down Emitter thread - abort retry')
+ return
logger.info('Retrying emit after %s seconds.' % self.RETRY_SLEEP_INTERVAL)
pass
@@ -69,7 +76,9 @@ class Emitter(threading.Thread):
logger.warn("Error sending metrics to server. Retrying after {0} "
"...".format(self.RETRY_SLEEP_INTERVAL))
retry_count += 1
- time.sleep(self.RETRY_SLEEP_INTERVAL)
+ #Wait for the service stop event instead of sleeping blindly
+ if 0 == self._stop_handler.wait(self.RETRY_SLEEP_INTERVAL):
+ return
pass
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py
new file mode 100644
index 0000000..bfb6957
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import os
+import signal
+import threading
+import traceback
+
+from ambari_commons import OSConst, OSCheck
+from ambari_commons.exceptions import FatalException
+from ambari_commons.os_family_impl import OsFamilyImpl
+
+
+logger = logging.getLogger()
+
+_handler = None
+
+
+class StopHandler(object):
+ def set_stop(self):
+ pass
+
+ def wait(self, timeout=None):
+ return -1
+
+
+#
+# Windows implementation
+#
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class StopHandlerWindows(StopHandler):
+ def __init__(self, stopEvent=None):
+ import win32event
+ # Event used to gracefully stop the process
+ if stopEvent is None:
+ # Allow standalone testing
+ self._heventStop = win32event.CreateEvent(None, 0, 0, None)
+ else:
+ # Allow one unique event per process
+ self._heventStop = stopEvent
+
+ def set_stop(self):
+ import win32event
+ win32event.SetEvent(self._heventStop)
+
+ def wait(self, timeout=None):
+ '''
+ :param timeout: Time to wait, in seconds.
+ :return: 0 == stop event signaled, -1 = timeout
+ '''
+ import win32event
+
+ if timeout is None:
+ timeout = win32event.INFINITE
+ else:
+ timeout = timeout * 1000
+
+ result = win32event.WaitForSingleObject(self._heventStop, timeout)
+ if(win32event.WAIT_OBJECT_0 != result and win32event.WAIT_TIMEOUT != result):
+ raise FatalException(-1, "Error waiting for stop event: " + str(result))
+ if (win32event.WAIT_TIMEOUT == result):
+ return -1
+ logger.info("Stop event received")
+ return result # 0 -> stop
+
+
+#
+# Linux implementation
+#
+def signal_handler(signum, frame):
+ global _handler
+ _handler.set_stop()
+
+def debug(sig, frame):
+ """Interrupt running process, and provide a python prompt for
+ interactive debugging."""
+ d = {'_frame': frame} # Allow access to frame object.
+ d.update(frame.f_globals) # Unless shadowed by global
+ d.update(frame.f_locals)
+
+ message = "Signal received : entering python shell.\nTraceback:\n"
+ message += ''.join(traceback.format_stack(frame))
+ logger.info(message)
+
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class StopHandlerLinux(StopHandler):
+ def __init__(self, stopEvent=None):
+ # Event used to gracefully stop the process
+ if stopEvent is None:
+ # Allow standalone testing
+ self.stop_event = threading.Event()
+ else:
+ # Allow one unique event per process
+ self.stop_event = stopEvent
+
+ def set_stop(self):
+ self.stop_event.set()
+
+ def wait(self, timeout=None):
+ # Stop process when stop event received
+ if self.stop_event.wait(timeout):
+ logger.info("Stop event received")
+ return 0
+ # Timeout
+ return -1
+
+
+def bind_signal_handlers(new_handler=None):
+ if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+ signal.signal(signal.SIGUSR1, debug)
+
+ if new_handler is None:
+ global _handler
+ _handler = StopHandler()
+ else:
+ _handler = new_handler
+ return _handler
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py
index 09ae7e4..37e77e5 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py
@@ -18,27 +18,67 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
-import core
-from core.controller import Controller
-from core.config_reader import Configuration
import logging
-import signal
+import os
import sys
+from ambari_commons.os_utils import remove_file
+
+from core.controller import Controller
+from core.config_reader import Configuration, PID_OUT_FILE, SERVER_LOG_FILE, SERVER_OUT_FILE
+from core.stop_handler import bind_signal_handlers
+
+
logger = logging.getLogger()
+
+def save_pid(pid, pidfile):
+ """
+ Save pid to pidfile.
+ """
+ try:
+ pfile = open(pidfile, "w")
+ pfile.write("%s\n" % pid)
+ except IOError:
+ pass
+ finally:
+ try:
+ pfile.close()
+ except:
+ pass
+
+
def main(argv=None):
# Allow Ctrl-C
- signal.signal(signal.SIGINT, signal.SIG_DFL)
+ stop_handler = bind_signal_handlers()
+
+ server_process_main(stop_handler)
+
+def server_process_main(stop_handler, scmStatus=None):
+ if scmStatus is not None:
+ scmStatus.reportStartPending()
+
+ save_pid(os.getpid(), PID_OUT_FILE)
config = Configuration()
- controller = Controller(config)
-
+ controller = Controller(config, stop_handler)
+
_init_logging(config)
-
+
logger.info('Starting Server RPC Thread: %s' % ' '.join(sys.argv))
controller.start()
- controller.start_emitter()
+
+ print "Server out at: " + SERVER_OUT_FILE
+ print "Server log at: " + SERVER_LOG_FILE
+
+ if scmStatus is not None:
+ scmStatus.reportStarted()
+
+ #The controller thread finishes when the stop event is signaled
+ controller.join()
+
+ remove_file(PID_OUT_FILE)
+ pass
def _init_logging(config):
_levels = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd
new file mode 100644
index 0000000..2cfff6f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd
@@ -0,0 +1,17 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@echo off
+python.exe -u %~dp0\sbin\main.py %* 2>&1
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd
new file mode 100644
index 0000000..af99fc4
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd
@@ -0,0 +1,16 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Set environment variables here.
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml
new file mode 100644
index 0000000..c2dd100
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+
+ <!-- Site specific AMS configuration properties -->
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties
new file mode 100644
index 0000000..3df60b0
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties
new file mode 100644
index 0000000..d354a27
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Define some default values that can be overridden by system properties
+# Root logger option
+log4j.rootLogger=INFO,file
+
+# Direct log messages to a log file
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.File=\\var\\log\\ambari-metrics-collector\\ambari-metrics-collector.log
+log4j.appender.file.MaxFileSize=80MB
+log4j.appender.file.MaxBackupIndex=60
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index 196d7a1..46504a7 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -42,6 +42,33 @@
<build>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>parse-version</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>parse-version</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>ambariVersion</name>
+ <value>${project.version}</value>
+ <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
+ <replacement>$1.$2.$3</replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
@@ -59,6 +86,10 @@
</executions>
</plugin>
<plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.0</version>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py
new file mode 100644
index 0000000..38e1a5e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py
new file mode 100644
index 0000000..8e00762
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import re
+import time
+
+#Apache License Header
+ASF_LICENSE_HEADER = '''
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+
+# A Python replacement for java.util.Properties
+# Based on http://code.activestate.com/recipes
+# /496795-a-python-replacement-for-javautilproperties/
+class Properties(object):
+ def __init__(self, props=None):
+ self._props = {}
+ self._origprops = {}
+ self._keymap = {}
+
+ self.othercharre = re.compile(r'(?<!\\)(\s*\=)|(?<!\\)(\s*\:)')
+ self.othercharre2 = re.compile(r'(\s*\=)|(\s*\:)')
+ self.bspacere = re.compile(r'\\(?!\s$)')
+
+ def __parse(self, lines):
+ lineno = 0
+ i = iter(lines)
+ for line in i:
+ lineno += 1
+ line = line.strip()
+ if not line:
+ continue
+ if line[0] == '#':
+ continue
+ escaped = False
+ sepidx = -1
+ flag = 0
+ m = self.othercharre.search(line)
+ if m:
+ first, last = m.span()
+ start, end = 0, first
+ flag = 1
+ wspacere = re.compile(r'(?<![\\\=\:])(\s)')
+ else:
+ if self.othercharre2.search(line):
+ wspacere = re.compile(r'(?<![\\])(\s)')
+ start, end = 0, len(line)
+ m2 = wspacere.search(line, start, end)
+ if m2:
+ first, last = m2.span()
+ sepidx = first
+ elif m:
+ first, last = m.span()
+ sepidx = last - 1
+ while line[-1] == '\\':
+ nextline = i.next()
+ nextline = nextline.strip()
+ lineno += 1
+ line = line[:-1] + nextline
+ if sepidx != -1:
+ key, value = line[:sepidx], line[sepidx + 1:]
+ else:
+ key, value = line, ''
+ self.process_pair(key, value)
+
+ def process_pair(self, key, value):
+ oldkey = key
+ oldvalue = value
+ keyparts = self.bspacere.split(key)
+ strippable = False
+ lastpart = keyparts[-1]
+ if lastpart.find('\\ ') != -1:
+ keyparts[-1] = lastpart.replace('\\', '')
+ elif lastpart and lastpart[-1] == ' ':
+ strippable = True
+ key = ''.join(keyparts)
+ if strippable:
+ key = key.strip()
+ oldkey = oldkey.strip()
+ oldvalue = self.unescape(oldvalue)
+ value = self.unescape(value)
+ self._props[key] = None if value is None else value.strip()
+ if self._keymap.has_key(key):
+ oldkey = self._keymap.get(key)
+ self._origprops[oldkey] = None if oldvalue is None else oldvalue.strip()
+ else:
+ self._origprops[oldkey] = None if oldvalue is None else oldvalue.strip()
+ self._keymap[key] = oldkey
+
+ def unescape(self, value):
+ newvalue = value
+ if not value is None:
+ newvalue = value.replace('\:', ':')
+ newvalue = newvalue.replace('\=', '=')
+ return newvalue
+
+ def removeOldProp(self, key):
+ if self._origprops.has_key(key):
+ del self._origprops[key]
+ pass
+
+ def removeProp(self, key):
+ if self._props.has_key(key):
+ del self._props[key]
+ pass
+
+ def load(self, stream):
+ if type(stream) is not file:
+ raise TypeError, 'Argument should be a file object!'
+ if stream.mode != 'r':
+ raise ValueError, 'Stream should be opened in read-only mode!'
+ try:
+ self.fileName = os.path.abspath(stream.name)
+ lines = stream.readlines()
+ self.__parse(lines)
+ except IOError:
+ raise
+
+ def get_property(self, key):
+ return self._props.get(key, '')
+
+ def propertyNames(self):
+ return self._props.keys()
+
+ def getPropertyDict(self):
+ return self._props
+
+ def __getitem__(self, name):
+ return self.get_property(name)
+
+ def __getattr__(self, name):
+ try:
+ return self.__dict__[name]
+ except KeyError:
+ if hasattr(self._props, name):
+ return getattr(self._props, name)
+
+ def sort_props(self):
+ tmp_props = {}
+ for key in sorted(self._props.iterkeys()):
+ tmp_props[key] = self._props[key]
+ self._props = tmp_props
+ pass
+
+ def sort_origprops(self):
+ tmp_props = self._origprops.copy()
+ self._origprops.clear()
+ for key in sorted(tmp_props.iterkeys()):
+ self._origprops[key] = tmp_props[key]
+ pass
+
+ def store(self, out, header=""):
+ """ Write the properties list to the stream 'out' along
+ with the optional 'header' """
+ if out.mode[0] != 'w':
+ raise ValueError, 'Steam should be opened in write mode!'
+ try:
+ out.write(''.join(('#', ASF_LICENSE_HEADER, '\n')))
+ out.write(''.join(('#', header, '\n')))
+ # Write timestamp
+ tstamp = time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime())
+ out.write(''.join(('#', tstamp, '\n')))
+ # Write properties from the pristine dictionary
+ for prop, val in self._origprops.items():
+ if val is not None:
+ out.write(''.join((prop, '=', val, '\n')))
+ out.close()
+ except IOError:
+ raise
+
+ def store_ordered(self, out, header=""):
+ """ Write the properties list to the stream 'out' along
+ with the optional 'header' """
+ if out.mode[0] != 'w':
+ raise ValueError, 'Steam should be opened in write mode!'
+ try:
+ out.write(''.join(('#', ASF_LICENSE_HEADER, '\n')))
+ out.write(''.join(('#', header, '\n')))
+ # Write timestamp
+ tstamp = time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime())
+ out.write(''.join(('#', tstamp, '\n')))
+ # Write properties from the pristine dictionary
+ for key in sorted(self._origprops.iterkeys()):
+ val = self._origprops[key]
+ if val is not None:
+ out.write(''.join((key, '=', val, '\n')))
+ out.close()
+ except IOError:
+ raise
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e972cf/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/serviceConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/serviceConfiguration.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/serviceConfiguration.py
new file mode 100644
index 0000000..5e73e24
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/serviceConfiguration.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+
+from ambari_commons.exceptions import FatalException
+from ambari_commons.logging_utils import print_info_msg
+from ambari_commons.os_utils import search_file
+from ambari_metrics_collector.properties import Properties
+
+
+AMS_CONF_VAR = "AMS_CONF"
+DEFAULT_CONF_DIR = "conf"
+AMS_PROPERTIES_FILE = "ams.properties"
+
+JAVA_HOME = "JAVA_HOME"
+
+DEBUG_MODE_KEY = "ams.server.debug"
+SUSPEND_START_MODE_KEY = "ams.server.debug.suspend.start"
+
+SERVER_OUT_FILE_KEY = "ams.output.file.path"
+
+DEFAULT_LIBS_DIR = "lib"
+
+EMBEDDED_HBASE_MASTER_SERVICE = "ams_hbase_master"
+
+EMBEDDED_HBASE_SUBDIR = "hbase"
+
+JAVA_EXE_SUBPATH = "bin\\java.exe"
+
+JAVA_HEAP_MAX_DEFAULT = "-Xmx1g"
+
+HADOOP_HEAPSIZE = "HADOOP_HEAPSIZE"
+HADOOP_HEAPSIZE_DEFAULT = "1024"
+
+DEBUG_MODE = False
+SUSPEND_START_MODE = False
+
+OUT_DIR = "\\var\\log\\ambari-metrics-collector"
+SERVER_OUT_FILE = OUT_DIR + "\\ambari-metrics-collector.out"
+SERVER_LOG_FILE = OUT_DIR + "\\ambari-metrics-collector.log"
+
+PID_DIR = "\\var\\run\\ambari-metrics-collector"
+PID_OUT_FILE = PID_DIR + "\\ambari-metrics-collector.pid"
+EXITCODE_OUT_FILE = PID_DIR + "\\ambari-metrics-collector.exitcode"
+
+SERVICE_USERNAME_KEY = "TMP_AMC_USERNAME"
+SERVICE_PASSWORD_KEY = "TMP_AMC_PASSWORD"
+
+SETUP_ACTION = "setup"
+START_ACTION = "start"
+STOP_ACTION = "stop"
+RESTART_ACTION = "restart"
+STATUS_ACTION = "status"
+
+def get_conf_dir():
+ try:
+ conf_dir = os.environ[AMS_CONF_VAR]
+ except KeyError:
+ conf_dir = DEFAULT_CONF_DIR
+ return conf_dir
+
+def find_properties_file():
+ conf_file = search_file(AMS_PROPERTIES_FILE, get_conf_dir())
+ if conf_file is None:
+ err = 'File %s not found in search path $%s: %s' % (AMS_PROPERTIES_FILE,
+ AMS_CONF_VAR, get_conf_dir())
+ print err
+ raise FatalException(1, err)
+ else:
+ print_info_msg('Loading properties from ' + conf_file)
+ return conf_file
+
+# Load AMC properties and return dict with values
+def get_properties():
+ conf_file = find_properties_file()
+
+ properties = None
+ try:
+ properties = Properties()
+ properties.load(open(conf_file))
+ except (Exception), e:
+ print 'Could not read "%s": %s' % (conf_file, e)
+ return -1
+ return properties
+
+def get_value_from_properties(properties, key, default=""):
+ try:
+ value = properties.get_property(key)
+ if not value:
+ value = default
+ except:
+ return default
+ return value
+
+def get_java_cp():
+ conf_dir = get_conf_dir()
+ conf_dir = os.path.abspath(conf_dir) + os.pathsep + os.path.join(os.path.abspath(DEFAULT_LIBS_DIR), "*")
+ if conf_dir.find(' ') != -1:
+ conf_dir = '"' + conf_dir + '"'
+ return conf_dir
+
+def find_jdk():
+ try:
+ java_home = os.environ[JAVA_HOME]
+ except Exception:
+ # No JAVA_HOME set
+ err = "ERROR: JAVA_HOME is not set and could not be found."
+ raise FatalException(1, err)
+
+ if not os.path.isdir(java_home):
+ err = "ERROR: JAVA_HOME {0} does not exist.".format(java_home)
+ raise FatalException(1, err)
+
+ java_exe = os.path.join(java_home, JAVA_EXE_SUBPATH)
+ if not os.path.isfile(java_exe):
+ err = "ERROR: {0} is not executable.".format(java_exe)
+ raise FatalException(1, err)
+
+ return java_home
+
+def get_java_exe_path():
+ jdk_path = find_jdk()
+ java_exe = os.path.join(jdk_path, JAVA_EXE_SUBPATH)
+ return java_exe
+
+def build_jvm_args():
+ try:
+ # check envvars which might override default args
+ hadoop_heapsize = os.environ[HADOOP_HEAPSIZE]
+ java_heap_max = "-Xms{0}m".format(hadoop_heapsize)
+ except Exception:
+ java_heap_max = JAVA_HEAP_MAX_DEFAULT
+
+ return java_heap_max