You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2016/03/02 21:30:30 UTC
ambari git commit: Updates to LogSearch Stack service definition,
including role_command_order changes (Oliver Szabo via rnettleton)
Repository: ambari
Updated Branches:
refs/heads/branch-dev-logsearch cb25daa33 -> 69adbd527
Updates to LogSearch Stack service definition, including role_command_order changes (Oliver Szabo via rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/69adbd52
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/69adbd52
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/69adbd52
Branch: refs/heads/branch-dev-logsearch
Commit: 69adbd5270582aa51151dd87b69475e92b1d7f6e
Parents: cb25daa
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Wed Mar 2 15:29:03 2016 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Wed Mar 2 15:29:03 2016 -0500
----------------------------------------------------------------------
.../LOGSEARCH/0.5.0/metainfo.xml | 189 ++++++++++---------
.../LOGSEARCH/0.5.0/metrics.json | 59 ++++++
.../0.5.0/package/scripts/logfeeder.py | 11 +-
.../0.5.0/package/scripts/logsearch.py | 11 +-
.../LOGSEARCH/0.5.0/package/scripts/solr.py | 11 +-
.../0.5.0/package/scripts/status_params.py | 1 -
.../LOGSEARCH/0.5.0/widgets.json | 173 +++++++++++++++++
.../stacks/HDP/2.4/role_command_order.json | 3 +
8 files changed, 347 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index 8d7f665..3429304 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -16,98 +16,103 @@
limitations under the License.
-->
<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>LOGSEARCH</name>
- <displayName>Log Search</displayName>
- <comment>Log aggregation, analysis, and visualization for Hadoop</comment>
- <version>0.5.0</version>
-
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>LOGSEARCH</name>
+ <displayName>Log Search</displayName>
+ <comment>Log aggregation, analysis, and visualization for Hadoop</comment>
+ <version>0.5.0</version>
- <components>
-
- <component>
- <name>LOGSEARCH_MASTER</name>
- <timelineAppid>logsearch</timelineAppid>
- <displayName>Log Search</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <commandScript>
- <script>scripts/logsearch.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>5000</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>UPDATE_LOGSEARCH</name>
- <commandScript>
- <script>scripts/logsearch.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
+ <components>
- </component>
-
- <component>
- <name>LOGSEARCH_SOLR</name>
- <displayName>Solr</displayName>
- <category>MASTER</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/solr.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>5000</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>LOGSEARCH_LOGFEEDER</name>
- <timelineAppid>logfeeder</timelineAppid>
- <displayName>Log Feeder</displayName>
- <category>SLAVE</category>
- <cardinality>ALL</cardinality>
- <commandScript>
- <script>scripts/logfeeder.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>5000</timeout>
- </commandScript>
-
- <customCommands>
- <customCommand>
- <name>UPDATE_LOGFEEDER</name>
- <commandScript>
- <script>scripts/logfeeder.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
-
- </component>
-
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>redhat6</osFamily>
- <packages>
- <package><name>wget</name></package>
- <package><name>tar</name></package>
- <package><name>zip</name></package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <configuration-dependencies>
- <config-type>logsearch-config</config-type>
- <config-type>solr-config</config-type>
- <config-type>logfeeder-config</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>false</restartRequiredAfterChange>
- <widgetsFileName>widgets.json</widgetsFileName>
- <metricsFileName>metrics.json</metricsFileName>
- </service>
- </services>
+ <component>
+ <name>LOGSEARCH_MASTER</name>
+ <timelineAppid>logsearch</timelineAppid>
+ <displayName>Log Search</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/logsearch.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>5000</timeout>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>UPDATE_LOGSEARCH</name>
+ <commandScript>
+ <script>scripts/logsearch.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+
+ </component>
+
+ <component>
+ <name>LOGSEARCH_SOLR</name>
+ <displayName>Solr</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>5000</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>LOGSEARCH_LOGFEEDER</name>
+ <timelineAppid>logfeeder</timelineAppid>
+ <displayName>Log Feeder</displayName>
+ <category>SLAVE</category>
+ <cardinality>ALL</cardinality>
+ <commandScript>
+ <script>scripts/logfeeder.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>5000</timeout>
+ </commandScript>
+
+ <customCommands>
+ <customCommand>
+ <name>UPDATE_LOGFEEDER</name>
+ <commandScript>
+ <script>scripts/logfeeder.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+
+ </component>
+
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>wget</name>
+ </package>
+ <package>
+ <name>tar</name>
+ </package>
+ <package>
+ <name>zip</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <configuration-dependencies>
+ <config-type>logsearch-config</config-type>
+ <config-type>solr-config</config-type>
+ <config-type>logfeeder-config</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>false</restartRequiredAfterChange>
+ <widgetsFileName>widgets.json</widgetsFileName>
+ <metricsFileName>metrics.json</metricsFileName>
+ </service>
+ </services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json
new file mode 100644
index 0000000..a6c20c7
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json
@@ -0,0 +1,59 @@
+{
+ "LOGSEARCH_LOGFEEDER": {
+ "Component": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "default": {
+ "metrics/filter/error/grok": {
+ "metric": "filter.error.grok",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/filter/error/truncate": {
+ "metric": "filter.error.truncate",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/input/files/count": {
+ "metric": "input.files.count",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/input/files/read_lines": {
+ "metric": "input.files.read_lines",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/input/files/read_bytes": {
+ "metric": "input.files.read_bytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/output/kafka/write_logs": {
+ "metric": "output.kafka.write_logs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/output/kafka/write_bytes": {
+ "metric": "output.kafka.write_bytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/output/solr/write_logs": {
+ "metric": "output.solr.write_logs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/output/solr/write_bytes": {
+ "metric": "output.solr.write_bytes",
+ "pointInTime": false,
+ "temporal": true
+ }
+
+ }
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
index 8da6b82..052ce52 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
@@ -19,9 +19,8 @@ limitations under the License.
import sys, os, pwd, grp, signal, time
from resource_management import *
-from subprocess import call
-class Master(Script):
+class LogFeeder(Script):
#Call setup.sh to install the service
def install(self, env):
@@ -72,7 +71,7 @@ class Master(Script):
Execute('cd ' + params.logfeeder_dir + '; tar -xvf logfeeder.tar.gz', user=params.logfeeder_user)
- def configure(self, env):
+ def configure(self, env, upgrade_type=None):
import params
env.set_params(params)
@@ -89,7 +88,7 @@ class Master(Script):
#Call start.sh to start the service
- def start(self, env):
+ def start(self, env, upgrade_type=None):
#import properties defined in -config.xml file from params class
import params
@@ -114,7 +113,7 @@ class Master(Script):
Execute(cmd, user=params.logfeeder_user)
#Called to stop the service using the pidfile
- def stop(self, env):
+ def stop(self, env, upgrade_type=None):
import params
#import status properties defined in -env.xml file from status_params class
@@ -152,4 +151,4 @@ class Master(Script):
if __name__ == "__main__":
- Master().execute()
+ LogFeeder().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
index 594d85f..3f4f02e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
@@ -19,9 +19,8 @@ limitations under the License.
import sys, os, pwd, grp, signal, time
from resource_management import *
-from subprocess import call
-class Master(Script):
+class LogSearch(Script):
#Call setup.sh to install the service
def install(self, env):
@@ -72,7 +71,7 @@ class Master(Script):
Execute('cd ' + params.logsearch_dir + '; tar -xvf logsearch-portal.tar.gz', user=params.logsearch_user)
- def configure(self, env):
+ def configure(self, env, upgrade_type=None):
import params
env.set_params(params)
@@ -93,7 +92,7 @@ class Master(Script):
File(format("{params.logsearch_dir}/solr_configsets/audit_logs/conf/solrconfig.xml"), content=file_content, owner=params.logsearch_user)
#Call start.sh to start the service
- def start(self, env):
+ def start(self, env, upgrade_type=None):
#import properties defined in -config.xml file from params class
import params
@@ -151,7 +150,7 @@ class Master(Script):
Execute(cmd, user=params.logsearch_user)
#Called to stop the service using the pidfile
- def stop(self, env):
+ def stop(self, env, upgrade_type=None):
import params
#import status properties defined in -env.xml file from status_params class
@@ -188,4 +187,4 @@ class Master(Script):
self.start(env)
if __name__ == "__main__":
- Master().execute()
+ LogSearch().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
index cdbfabd..ec8eeb2 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
@@ -19,9 +19,8 @@ limitations under the License.
import sys, os, pwd, grp, signal, time
from resource_management import *
-from subprocess import call
-class Master(Script):
+class Solr(Script):
#Call setup_solr.sh to install the service
def install(self, env):
@@ -88,7 +87,7 @@ class Master(Script):
- def configure(self, env):
+ def configure(self, env, upgrade_type=None):
import params
env.set_params(params)
@@ -111,7 +110,7 @@ class Master(Script):
#Call start.sh to start the service
- def start(self, env):
+ def start(self, env, upgrade_type=None):
#import properties defined in -config.xml file from params class
import params
@@ -143,7 +142,7 @@ class Master(Script):
Execute(cmd, user=params.solr_user)
#Called to stop the service using the pidfile
- def stop(self, env):
+ def stop(self, env, upgrade_type=None):
import params
#import status properties defined in -env.xml file from status_params class
@@ -174,4 +173,4 @@ class Master(Script):
if __name__ == "__main__":
- Master().execute()
+ Solr().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
index f9ed87c..86e8f2e 100755
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
@@ -20,7 +20,6 @@ limitations under the License.
"""
from resource_management import *
-import glob
# config object that holds the status related configurations declared in the -env.xml file
config = Script.get_config()
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json
new file mode 100644
index 0000000..379122a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/widgets.json
@@ -0,0 +1,173 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_logsearch_dashboard",
+ "display_name": "Logsearch Dashboard",
+ "section_name": "LOGSEARCH_SUMMARY",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "Files Monitoring",
+ "description": "Files that are been monitored",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "input.files.count",
+ "metric_path": "metrics/input/files/count",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Files count",
+ "value": "${input.files.count}"
+ }
+ ],
+ "properties": {
+ }
+ },
+ {
+ "widget_name": "Logs Published",
+ "description": "Logs Published",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "output.solr.write_logs._sum",
+ "metric_path": "metrics/output/solr/write_logs",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ },
+ {
+ "name": "output.kafka.write_logs._sum",
+ "metric_path": "metrics/output/kafka/write_logs",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Logs to Solr",
+ "value": "${output.solr.write_logs._sum/1000}"
+ },
+ {
+ "name": "Logs to Kafka",
+ "value": "${output.kafka.write_logs._sum/1000}"
+ }
+ ],
+ "properties": {
+ "display_unit": "K",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Lines Read",
+ "description": "Lines Read",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "input.files.read_lines._sum",
+ "metric_path": "metrics/input/files/read_lines",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ }
+
+ ],
+ "values": [
+ {
+ "name": "Lines read from File",
+ "value": "${input.files.read_lines._sum/1024}"
+ }
+ ],
+ "properties": {
+ "display_unit": "K",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Read/Write Bytes",
+ "description": "Bytes read and written",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "input.files.read_bytes._sum",
+ "metric_path": "metrics/input/files/read_bytes",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ },
+ {
+ "name": "output.solr.write_bytes._sum",
+ "metric_path": "metrics/output/solr/write_bytes",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ },
+ {
+ "name": "output.kafka.write_bytes._sum",
+ "metric_path": "metrics/output/kafka/write_bytes",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Data read from File",
+ "value": "${input.files.read_bytes._sum/(1024*1024)}"
+ },
+ {
+ "name": "Data sent to Solr",
+ "value": "${output.solr.write_bytes._sum/(1024*1024)}"
+ },
+ {
+ "name": "Data sent to Kafka",
+ "value": "${output.kafka.write_bytes._sum/(1024*1024)}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Parse Errors",
+ "description": "Parse Errors",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "filter.error.grok._sum",
+ "metric_path": "metrics/filter/error/grok",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ },
+ {
+ "name": "filter.error.truncate._sum",
+ "metric_path": "metrics/filter/error/truncate",
+ "service_name": "LOGSEARCH",
+ "component_name": "LOGSEARCH_LOGFEEDER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Grok Errors",
+ "value": "${filter.error.grok._sum}"
+ },
+ {
+ "name": "Logs Truncated",
+ "value": "${filter.error.truncate._sum}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/69adbd52/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json
index 14b2563..9e13808 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/role_command_order.json
@@ -3,6 +3,9 @@
"_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
"general_deps" : {
"_comment" : "dependencies for all cases",
+ "LOGSEARCH_SOLR-START" : ["ZOOKEEPER_SERVER-START"],
+ "LOGSEARCH_MASTER-START": ["LOGSEARCH_SOLR-START"],
+ "LOGSEARCH_LOGFEEDER-START": ["LOGSEARCH_SOLR-START", "LOGSEARCH_MASTER-START"],
"SPARK_THRIFTSERVER-START" : ["HIVE_SERVER-START"]
}
}