You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/06/24 21:06:15 UTC
[1/2] ambari git commit: AMBARI-17424. Add missing components to
LogSearch Part 2 (Miklos Gergely via oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 6e5422f62 -> d370a4467
AMBARI-17424. Add missing components to LogSearch Part 2 (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca2e62fe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca2e62fe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca2e62fe
Branch: refs/heads/branch-2.4
Commit: ca2e62fe048a4e35d87043c687027e4ab7b896b2
Parents: 6e5422f
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Fri Jun 24 17:18:18 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Fri Jun 24 23:05:41 2016 +0200
----------------------------------------------------------------------
.../src/main/resources/HadoopServiceConfig.json | 29 ++++++++++
.../FLUME/1.4.0.2.0/metainfo.xml | 6 +++
.../LOGSEARCH/0.5.0/metainfo.xml | 6 +++
.../LOGSEARCH/0.5.0/package/scripts/params.py | 6 ++-
.../templates/input.config-flume.json.j2 | 56 ++++++++++++++++++++
.../templates/input.config-hbase.json.j2 | 29 ++++++++++
.../templates/input.config-logsearch.json.j2 | 30 +++++++++++
.../templates/input.config-zeppelin.json.j2 | 56 ++++++++++++++++++++
.../ZEPPELIN/0.6.0.2.5/metainfo.xml | 6 +++
.../stacks/HDP/2.3/services/HBASE/metainfo.xml | 6 +++
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 4 +-
11 files changed, 230 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
index 73400e3..abe692d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
@@ -100,6 +100,18 @@
]
},
+ "flume":{
+ "label":"Flume",
+ "components":[
+ {
+ "name":"flume_handler"
+ }
+ ],
+ "dependencies":[
+
+ ]
+
+ },
"hbase":{
"label":"HBase",
"components":[
@@ -108,6 +120,9 @@
},
{
"name":"hbase_regionserver"
+ },
+ {
+ "name":"hbase_phoenix_server"
}
],
@@ -272,6 +287,9 @@
},
{
"name":"logsearch_perf"
+ },
+ {
+ "name":"logsearch_solr"
}
],
@@ -425,6 +443,17 @@
]
},
+ "zeppelin":{
+ "label":"Zeppelin",
+ "components":[
+ {
+ "name":"zeppelin"
+ }
+ ],
+ "dependencies":[
+
+ ]
+ },
"zookeeper":{
"label":"ZooKeeper",
"components":[
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
index 0396c28..b4f1058 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
@@ -35,6 +35,12 @@
<scriptType>PYTHON</scriptType>
<timeout>1200</timeout>
</commandScript>
+ <logs>
+ <log>
+ <logId>flume_handler</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index e3a8f98..ffa9e13 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -84,6 +84,12 @@
<script>scripts/logsearch_solr.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <logs>
+ <log>
+ <logId>logsearch_solr</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
<dependencies>
<dependency>
<name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index d21a97f..4bc1004 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -247,6 +247,7 @@ metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_d
atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')
accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')
falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')
+flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flume')
hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')
hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')
hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive')
@@ -259,6 +260,7 @@ ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ra
storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm')
yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')
mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')
+zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')
#####################################
@@ -328,8 +330,8 @@ logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.c
logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower()
logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5)
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka',
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in
logfeeder_supported_services]
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
new file mode 100644
index 0000000..d1ea632
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"flume_handler",
+ "rowtype":"service",
+ "path":"{{flume_log_dir}}/flume.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "flume_handler"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
index fb47e77..42c5683 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
@@ -26,6 +26,11 @@
"type":"hbase_regionserver",
"rowtype":"service",
"path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+ },
+ {
+ "type":"hbase_phoenix_server",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/phoenix-hbase-server.log"
}
],
@@ -55,6 +60,30 @@
}
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hbase_phoenix_server"
+ ]
+
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
index ba151b9..31c37bf 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
@@ -31,6 +31,11 @@
"type":"logsearch_perf",
"rowtype":"service",
"path":"{{logsearch_log_dir}}/logsearch-performance.json"
+ },
+ {
+ "type":"logsearch_solr",
+ "rowtype":"service",
+ "path":"{{logsearch_solr_log_dir}}/solr.log"
}
],
@@ -49,6 +54,31 @@
}
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "logsearch_solr"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
new file mode 100644
index 0000000..09cd8b3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"zeppelin",
+ "rowtype":"service",
+ "path":"{{zeppelin_log_dir}}/zeppelin-zeppelin-*.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "zeppelin"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
index c40a593..4f19b43 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
@@ -53,6 +53,12 @@ limitations under the License.
</auto-deploy>
</dependency>
</dependencies>
+ <logs>
+ <log>
+ <logId>zeppelin</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
index 0614cf7..d7401d8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
@@ -33,6 +33,12 @@
<script>scripts/phoenix_queryserver.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <logs>
+ <log>
+ <logId>hbase_phoenix_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 0613770..26e8934 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -98,8 +98,8 @@ class TestLogFeeder(RMFTestCase):
encoding='utf-8'
)
- logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+ logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka',
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag
in logfeeder_supported_services]
[2/2] ambari git commit: AMBARI-17423. Add missing components to
LogSearch Part1 (oleewere)
Posted by ol...@apache.org.
AMBARI-17423. Add missing components to LogSearch Part1 (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d370a446
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d370a446
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d370a446
Branch: refs/heads/branch-2.4
Commit: d370a4467bdebf43d3f32721eae9955d390081c7
Parents: ca2e62f
Author: oleewere <ol...@gmail.com>
Authored: Fri Jun 24 17:50:27 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Fri Jun 24 23:05:42 2016 +0200
----------------------------------------------------------------------
.../src/main/resources/HadoopServiceConfig.json | 467 +++++++++----------
.../AMBARI_METRICS/0.1.0/metainfo.xml | 12 +
.../LOGSEARCH/0.5.0/package/scripts/params.py | 18 +-
.../0.5.0/package/templates/grok-patterns.j2 | 3 +
.../package/templates/input.config-ams.json.j2 | 85 +++-
.../package/templates/input.config-hdfs.json.j2 | 21 +-
.../package/templates/input.config-hive.json.j2 | 37 +-
.../package/templates/input.config-hst.json.j2 | 93 ++++
.../templates/input.config-spark.json.j2 | 66 +++
.../templates/input.config-spark2.json.j2 | 66 +++
.../package/templates/input.config-yarn.json.j2 | 12 +-
.../common-services/SPARK/1.2.1/metainfo.xml | 6 +
.../common-services/SPARK/1.4.1/metainfo.xml | 6 +
.../common-services/SPARK2/2.0.0/metainfo.xml | 12 +
.../stacks/HDP/2.5/services/SPARK/metainfo.xml | 6 +
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 4 +-
16 files changed, 651 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
index abe692d..d407d82 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
@@ -1,472 +1,459 @@
{
- "service":{
- "accumulo":{
- "label":"Accumulo",
- "components":[
+ "service": {
+ "accumulo": {
+ "label": "Accumulo",
+ "components": [
{
- "name":"accumulo_gc"
+ "name": "accumulo_gc"
},
{
- "name":"accumulo_master"
+ "name": "accumulo_master"
},
{
- "name":"accumulo_monitor"
+ "name": "accumulo_monitor"
},
{
- "name":"accumulo_tracer"
+ "name": "accumulo_tracer"
},
{
- "name":"accumulo_tserver"
+ "name": "accumulo_tserver"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "atlas":{
- "label":"Atlas",
- "components":[
+ "atlas": {
+ "label": "Atlas",
+ "components": [
{
- "name":"atlas_app"
+ "name": "atlas_app"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "ambari":{
- "label":"Ambari",
- "components":[
+ "ambari": {
+ "label": "Ambari",
+ "components": [
{
- "name":"ambari_agent"
+ "name": "ambari_agent"
},
{
- "name":"ambari_server"
+ "name": "ambari_server"
},
{
- "name":"ambari_alerts"
+ "name": "ambari_alerts"
},
{
- "name":"ambari_audit"
+ "name": "ambari_audit"
},
{
- "name":"ambari_config_changes"
+ "name": "ambari_config_changes"
},
{
- "name":"ambari_eclipseling"
+ "name": "ambari_eclipselink"
},
{
- "name":"ambari_server_check_database"
+ "name": "ambari_server_check_database"
}
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "ams":{
- "label":"AMS",
- "components":[
+ "ams": {
+ "label": "AMS",
+ "components": [
{
- "name":"ams_hbase_master"
+ "name": "ams_hbase_master"
},
{
- "name":"ams_hbase_regionserver"
+ "name": "ams_hbase_regionserver"
},
{
- "name":"ams_collector"
+ "name": "ams_collector"
+ },
+ {
+ "name": "ams_monitor"
+ },
+ {
+ "name": "ams_grafana"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "falcon":{
- "label":"Falcon",
- "components":[
+ "falcon": {
+ "label": "Falcon",
+ "components": [
{
- "name":"falcon_app"
+ "name": "falcon_app"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "flume":{
- "label":"Flume",
- "components":[
+ "flume": {
+ "label": "Flume",
+ "components": [
{
- "name":"flume_handler"
+ "name": "flume_handler"
}
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "hbase":{
- "label":"HBase",
- "components":[
+ "hbase": {
+ "label": "HBase",
+ "components": [
{
- "name":"hbase_master"
+ "name": "hbase_master"
},
{
- "name":"hbase_regionserver"
+ "name": "hbase_regionserver"
},
{
- "name":"hbase_phoenix_server"
+ "name": "hbase_phoenix_server"
}
-
],
- "dependencies":[
+ "dependencies": [
{
- "service":"hdfs",
- "components":[
+ "service": "hdfs",
+ "components": [
"hdfs_namenode"
]
-
}
-
]
-
},
- "hdfs":{
- "label":"HDFS",
- "components":[
+ "hdfs": {
+ "label": "HDFS",
+ "components": [
{
- "name":"hdfs_datanode"
+ "name": "hdfs_datanode"
},
{
- "name":"hdfs_namenode"
+ "name": "hdfs_namenode"
},
{
- "name":"hdfs_journalnode"
+ "name": "hdfs_journalnode"
},
{
- "name":"hdfs_secondarynamenode"
+ "name": "hdfs_secondarynamenode"
},
{
- "name":"hdfs_zkfc"
+ "name": "hdfs_zkfc"
},
{
- "name":"hdfs_audit",
- "rowtype":"audit"
+ "name": "hdfs_nfs3"
+ },
+ {
+ "name": "hdfs_audit",
+ "rowtype": "audit"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "hive":{
- "label":"Hive",
- "components":[
+ "hive": {
+ "label": "Hive",
+ "components": [
+ {
+ "name": "hive_hiveserver2"
+ },
{
- "name":"hive_hiveserver2"
+ "name": "hive_metastore"
},
{
- "name":"hive_metastore"
+ "name": "webhcat_server"
}
-
],
- "dependencies":[
+ "dependencies": [
{
- "service":"hdfs",
- "components":[
+ "service": "hdfs",
+ "components": [
"hdfs_namenode"
]
-
}
-
]
-
},
- "kafka":{
- "label":"Kafka",
- "components":[
+ "kafka": {
+ "label": "Kafka",
+ "components": [
{
- "name":"kafka_controller"
+ "name": "kafka_controller"
},
{
- "name":"kafka_request"
+ "name": "kafka_request"
},
{
- "name":"kafka_logcleaner"
+ "name": "kafka_logcleaner"
},
{
- "name":"kafka_server"
+ "name": "kafka_server"
},
{
- "name":"kafka_statechange"
+ "name": "kafka_statechange"
}
-
],
- "dependencies":[
+ "dependencies": [
{
- "service":"zookeeper",
- "components":[
+ "service": "zookeeper",
+ "components": [
"zookeeper"
]
-
}
-
]
-
},
- "knox":{
- "label":"Knox",
- "components":[
+ "knox": {
+ "label": "Knox",
+ "components": [
{
- "name":"knox_gateway"
+ "name": "knox_gateway"
},
{
- "name":"knox_cli"
+ "name": "knox_cli"
},
{
- "name":"knox_ldap"
+ "name": "knox_ldap"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "nifi":{
- "label":"NiFi",
- "components":[
+ "nifi": {
+ "label": "NiFi",
+ "components": [
{
- "name":"nifi_app"
+ "name": "nifi_app"
},
{
- "name":"nifi_bootstrap"
+ "name": "nifi_bootstrap"
},
{
- "name":"nifi_setup"
+ "name": "nifi_setup"
},
{
- "name":"nifi_user"
+ "name": "nifi_user"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "mapred":{
- "label":"MapReduce",
- "components":[
+ "mapred": {
+ "label": "MapReduce",
+ "components": [
{
- "name":"mapred_historyserver"
+ "name": "mapred_historyserver"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "logsearch":{
- "label":"Logsearch",
- "components":[
+ "logsearch": {
+ "label": "Logsearch",
+ "components": [
{
- "name":"logsearch_app"
+ "name": "logsearch_app"
},
{
- "name":"logsearch_feeder"
+ "name": "logsearch_feeder"
},
{
- "name":"logsearch_perf"
+ "name": "logsearch_perf"
},
{
- "name":"logsearch_solr"
+ "name": "logsearch_solr"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "ranger":{
- "label":"Ranger",
- "components":[
+ "ranger": {
+ "label": "Ranger",
+ "components": [
{
- "name":"ranger_admin"
+ "name": "ranger_admin"
},
{
- "name":"ranger_dbpatch"
+ "name": "ranger_dbpatch"
},
{
- "name":"ranger_kms"
+ "name": "ranger_kms"
},
{
- "name":"ranger_usersync"
+ "name": "ranger_usersync"
}
-
],
- "dependencies":[
+ "dependencies": [
{
- "service":"hdfs",
- "required":"optional",
- "components":[
+ "service": "hdfs",
+ "required": "optional",
+ "components": [
"hdfs_namenode"
]
-
},
{
- "service":"hbase",
- "required":"optional",
- "components":[
+ "service": "hbase",
+ "required": "optional",
+ "components": [
"hbase_master",
"hbase_regionserver"
]
-
},
{
- "service":"hive",
- "required":"optional",
- "components":[
+ "service": "hive",
+ "required": "optional",
+ "components": [
"hive_hiveserver2"
]
-
},
{
- "service":"kafka",
- "required":"optional",
- "components":[
+ "service": "kafka",
+ "required": "optional",
+ "components": [
"kafka_ranger"
]
-
},
{
- "service":"knox",
- "required":"optional",
- "components":[
+ "service": "knox",
+ "required": "optional",
+ "components": [
"knox_gateway"
]
-
},
{
- "service":"storm",
- "required":"optional",
- "components":[
+ "service": "storm",
+ "required": "optional",
+ "components": [
"storm_supervisor"
]
-
},
{
- "service":"yarn",
- "required":"optional",
- "components":[
+ "service": "yarn",
+ "required": "optional",
+ "components": [
"yarn_resourcemanager"
]
-
}
-
]
-
},
- "oozie":{
- "label":"Oozie",
- "components":[
+ "oozie": {
+ "label": "Oozie",
+ "components": [
{
- "name":"oozie_app"
+ "name": "oozie_app"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "yarn":{
- "label":"YARN",
- "components":[
+ "hst": {
+ "label": "SmartSense",
+ "components": [
{
- "name":"yarn_nodemanager"
+ "name": "hst_server"
},
{
- "name":"yarn_resourcemanager"
+ "name": "hst_agent"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+ "spark": {
+ "label": "Spark",
+ "components": [
+ {
+ "name": "spark_jobhistory_server"
},
{
- "name":"yarn_timelineserver"
+ "name": "spark_thriftserver"
},
{
- "name":"yarn_historyserver"
+ "name": "livy_server"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+ "spark2": {
+ "label": "Spark 2",
+ "components": [
+ {
+ "name": "spark2_jobhistory_server"
},
{
- "name":"yarn_jobsummary"
+ "name": "spark2_thriftserver"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "storm":{
- "label":"Storm",
- "components":[
+ "storm": {
+ "label": "Storm",
+ "components": [
+ {
+ "name": "storm_drpc"
+ },
+ {
+ "name": "storm_logviewer"
+ },
+ {
+ "name": "storm_nimbus"
+ },
{
- "name":"storm_drpc"
+ "name": "storm_supervisor"
},
{
- "name":"storm_logviewer"
+ "name": "storm_ui"
+ },
+ {
+ "name": "storm_worker"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+ "yarn": {
+ "label": "YARN",
+ "components": [
+ {
+ "name": "yarn_nodemanager"
},
{
- "name":"storm_nimbus"
+ "name": "yarn_resourcemanager"
},
{
- "name":"storm_supervisor"
+ "name": "yarn_timelineserver"
},
{
- "name":"storm_ui"
+ "name": "yarn_historyserver"
},
{
- "name":"storm_worker"
+ "name": "yarn_jobsummary"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
},
- "zeppelin":{
- "label":"Zeppelin",
- "components":[
+ "zeppelin": {
+ "label": "Zeppelin",
+ "components": [
{
- "name":"zeppelin"
+ "name": "zeppelin"
}
],
- "dependencies":[
-
+ "dependencies": [
]
},
- "zookeeper":{
- "label":"ZooKeeper",
- "components":[
+ "zookeeper": {
+ "label": "ZooKeeper",
+ "components": [
{
- "name":"zookeeper"
+ "name": "zookeeper"
}
-
],
- "dependencies":[
-
+ "dependencies": [
]
-
}
-
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index 48dda02..95eb414 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -87,6 +87,12 @@
<scriptType>PYTHON</scriptType>
<timeout>1200</timeout>
</commandScript>
+ <logs>
+ <log>
+ <logId>ams_monitor</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
<component>
@@ -105,6 +111,12 @@
<config-type>ams-grafana-ini</config-type>
</configuration-dependencies>
</component>
+ <logs>
+ <log>
+ <logId>ams_grafana</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</components>
<osSpecifics>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 4bc1004..23a9c09 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -240,9 +240,11 @@ logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['conte
ambari_server_log_dir = '/var/log/ambari-server'
ambari_agent_log_dir = '/var/log/ambari-agent'
knox_log_dir = '/var/log/knox'
+hst_log_dir = '/var/log/hst'
-metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log')
-metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log')
+metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')
+metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor')
+metrics_grafana_log_dir = default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana')
atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')
accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')
@@ -251,6 +253,7 @@ flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flu
hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')
hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')
hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive')
+hcat_log_dir = default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat')
kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')
nifi_log_dir = default('/configurations/nifi-env/nifi_node_log_dir', '/var/log/nifi')
oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')
@@ -262,6 +265,13 @@ yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/
mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')
zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')
+spark_log_dir = default('/configurations/spark-env/spark_log_dir', '/var/log/spark')
+livy_log_dir = default('/configurations/livy-env/livy_log_dir', '/var/log/livy')
+spark2_log_dir = default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')
+
+hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs')
+mapred_user = default('configurations/mapred-env/mapred_user', 'mapred')
+yarn_user = default('configurations/yarn-env/yarn_user', 'yarn')
#####################################
# Logsearch auth configs
@@ -330,8 +340,8 @@ logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.c
logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower()
logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5)
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper']
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'kafka',
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in
logfeeder_supported_services]
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
index c85a06a..874c59d 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
@@ -142,3 +142,6 @@ LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|
# Custom
USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}
+SPARK_DATESTAMP %{YEAR}[/-]%{MONTHNUM2}[/-]%{MONTHDAY} %{TIME}
+CUSTOM_DATESTAMP %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}
+CUSTOM_SEPARATOR %{SPACE}\|%{SPACE}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
index 34c1ed4..3dbe6b2 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
@@ -31,8 +31,17 @@
"type":"ams_collector",
"rowtype":"service",
"path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
+ },
+ {
+ "type":"ams_monitor",
+ "rowtype":"service",
+ "path":"{{metrics_monitor_log_dir}}/ambari-metrics-monitor.out"
+ },
+ {
+ "type":"ams_grafana",
+ "rowtype":"service",
+ "path":"{{metrics_grafana_log_dir}}/grafana.log"
}
-
],
"filter":[
{
@@ -85,7 +94,79 @@
}
- }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_grafana"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{DATESTAMP})",
+ "message_pattern":"(?m)^%{DATESTAMP}%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy/MM/dd HH:mm:ss"
+ }
+
+ },
+ "level":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"I",
+ "post_value":"INFO"
+ },
+ "map_fieldvalue":{
+ "pre_value":"W",
+ "post_value":"WARN"
+ },
+ "map_fieldvalue":{
+ "pre_value":"D",
+ "post_value":"DEBUG"
+ },
+ "map_fieldvalue":{
+ "pre_value":"E",
+ "post_value":"ERROR"
+ },
+ "map_fieldvalue":{
+ "pre_value":"F",
+ "post_value":"FATAL"
+ }
+ }
+ ]
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_monitor"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{DATESTAMP:logtime})",
+ "message_pattern":"(?m)^%{DATESTAMP:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
index 730ef99..13ff9d1 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
@@ -20,27 +20,32 @@
{
"type":"hdfs_datanode",
"rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log"
+ "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-datanode-*.log"
},
{
"type":"hdfs_namenode",
"rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log"
+ "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-namenode-*.log"
},
{
"type":"hdfs_journalnode",
"rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log"
+ "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-journalnode-*.log"
},
{
"type":"hdfs_secondarynamenode",
"rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log"
+ "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-secondarynamenode-*.log"
},
{
"type":"hdfs_zkfc",
"rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log"
+ "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-zkfc-*.log"
+ },
+ {
+ "type":"hdfs_nfs3",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-nfs3-*.log"
},
{
"type":"hdfs_audit",
@@ -52,7 +57,7 @@
"repoType":"1",
"repo":"hdfs"
},
- "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log"
+ "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hdfs-audit.log"
}
],
@@ -66,7 +71,8 @@
"hdfs_journalnode",
"hdfs_secondarynamenode",
"hdfs_namenode",
- "hdfs_zkfc"
+ "hdfs_zkfc",
+ "hdfs_nfs3"
]
}
@@ -80,7 +86,6 @@
"map_date":{
"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
}
-
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
index 131ae86..f0fbf0a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
@@ -26,8 +26,12 @@
"type":"hive_metastore",
"rowtype":"service",
"path":"{{hive_log_dir}}/hivemetastore.log"
+ },
+ {
+ "type": "webhcat_server",
+ "rowntype":"service",
+ "path":"{{hcat_log_dir}}/webhcat.log"
}
-
],
"filter":[
{
@@ -55,6 +59,37 @@
}
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "webhcat_server"
+ ]
+
+ }
+
+ },
+ "log4j_format":" %-5p | %d{DATE} | %c | %m%n",
+ "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})",
+ "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+ }
+
+ }
+
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2
new file mode 100644
index 0000000..ee19f14
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2
@@ -0,0 +1,93 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+ {
+ "input":[
+
+ {
+ "type":"hst_server",
+ "rowtype":"service",
+ "path":"{{hst_log_dir}}/hst-server.log"
+ },
+ {
+ "type":"hst_agent",
+ "rowtype":"service",
+ "path":"{{hst_log_dir}}/hst-agent.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hst_agent"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{LOGLEVEL:level}%{SPACE}%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{LOGLEVEL:level}%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVAFILE:file}%{GREEDYDATA:log_message}}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+
+ }
+
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hst_server"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{CUSTOM_DATESTAMP:logtime})",
+ "message_pattern":"(?m)^%{CUSTOM_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+ }
+
+ }
+ }
+
+ ]
+
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2
new file mode 100644
index 0000000..80be6ee
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2
@@ -0,0 +1,66 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+ {
+ "input":[
+
+ {
+ "type":"spark_jobhistory_server",
+ "rowtype":"service",
+ "path":"{{spark_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out"
+ },
+ {
+ "type":"spark_thriftserver",
+ "rowtype":"service",
+ "path":"{{spark_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "spark_jobhistory_server",
+ "spark_thriftserver"
+ ]
+
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})",
+ "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yy/MM/dd HH:mm:ss"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+
+ }
+
+ }
+ }
+ ]
+
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2
new file mode 100644
index 0000000..20bd946
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2
@@ -0,0 +1,66 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+ {
+ "input":[
+
+ {
+ "type":"spark2_jobhistory_server",
+ "rowtype":"service",
+ "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out"
+ },
+ {
+ "type":"spark2_thriftserver",
+ "rowtype":"service",
+ "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "spark2_jobhistory_server",
+ "spark2_thriftserver"
+ ]
+
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})",
+ "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yy/MM/dd HH:mm:ss"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+
+ }
+
+ }
+ }
+ ]
+
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
index fb35620..3d9677e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
@@ -20,32 +20,32 @@
{
"type":"mapred_historyserver",
"rowtype":"service",
- "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log"
+ "path":"{{mapred_log_dir_prefix}}/{{mapred_user}}/mapred-{{mapred_user}}-historyserver*.log"
},
{
"type":"yarn_nodemanager",
"rowtype":"service",
- "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log"
+ "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-nodemanager-*.log"
},
{
"type":"yarn_resourcemanager",
"rowtype":"service",
- "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log"
+ "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-resourcemanager-*.log"
},
{
"type":"yarn_timelineserver",
"rowtype":"service",
- "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log"
+ "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-timelineserver-*.log"
},
{
"type":"yarn_historyserver",
"rowtype":"service",
- "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log"
+ "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-historyserver-*.log"
},
{
"type":"yarn_jobsummary",
"rowtype":"service",
- "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log"
+ "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/hadoop-mapreduce.jobsummary.log"
}
],
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml
index 86af492..1a35793 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml
@@ -61,6 +61,12 @@
</auto-deploy>
</dependency>
</dependencies>
+ <logs>
+ <log>
+ <logId>spark_jobhistory_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
<commandScript>
<script>scripts/job_history_server.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml
index 99bab04..bc9f7a4 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml
@@ -62,6 +62,12 @@
</auto-deploy>
</dependency>
</dependencies>
+ <logs>
+ <log>
+ <logId>spark_thriftserver</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
<commandScript>
<script>scripts/spark_thrift_server.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml
index 783f049..c83c83c 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml
@@ -59,6 +59,12 @@
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
+ <logs>
+ <log>
+ <logId>spark2_jobhistory_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
<component>
<name>SPARK2_THRIFTSERVER</name>
@@ -101,6 +107,12 @@
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
+ <logs>
+ <log>
+ <logId>spark2_thriftserver</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
<component>
<name>SPARK2_CLIENT</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml
index 67303b6..8b14fd1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml
@@ -59,6 +59,12 @@
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
+ <logs>
+ <log>
+ <logId>livy_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d370a446/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 26e8934..63afcac 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -98,8 +98,8 @@ class TestLogFeeder(RMFTestCase):
encoding='utf-8'
)
- logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper']
+ logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'kafka',
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag
in logfeeder_supported_services]