You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/06/24 21:03:45 UTC
[2/2] ambari git commit: AMBARI-17424. Add missing components to
LogSearch Part 2 (Miklos Gergely via oleewere)
AMBARI-17424. Add missing components to LogSearch Part 2 (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4207e601
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4207e601
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4207e601
Branch: refs/heads/trunk
Commit: 4207e60186c42215ead3283285b814af9fe37067
Parents: 36d1f86
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Fri Jun 24 17:18:18 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Fri Jun 24 23:00:12 2016 +0200
----------------------------------------------------------------------
.../src/main/resources/HadoopServiceConfig.json | 29 ++++++++++
.../FLUME/1.4.0.2.0/metainfo.xml | 6 +++
.../LOGSEARCH/0.5.0/metainfo.xml | 6 +++
.../LOGSEARCH/0.5.0/package/scripts/params.py | 6 ++-
.../templates/input.config-flume.json.j2 | 56 ++++++++++++++++++++
.../templates/input.config-hbase.json.j2 | 29 ++++++++++
.../templates/input.config-logsearch.json.j2 | 30 +++++++++++
.../templates/input.config-zeppelin.json.j2 | 56 ++++++++++++++++++++
.../ZEPPELIN/0.6.0.2.5/metainfo.xml | 6 +++
.../stacks/HDP/2.3/services/HBASE/metainfo.xml | 6 +++
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 4 +-
11 files changed, 230 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
index 73400e3..abe692d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
@@ -100,6 +100,18 @@
]
},
+ "flume":{
+ "label":"Flume",
+ "components":[
+ {
+ "name":"flume_handler"
+ }
+ ],
+ "dependencies":[
+
+ ]
+
+ },
"hbase":{
"label":"HBase",
"components":[
@@ -108,6 +120,9 @@
},
{
"name":"hbase_regionserver"
+ },
+ {
+ "name":"hbase_phoenix_server"
}
],
@@ -272,6 +287,9 @@
},
{
"name":"logsearch_perf"
+ },
+ {
+ "name":"logsearch_solr"
}
],
@@ -425,6 +443,17 @@
]
},
+ "zeppelin":{
+ "label":"Zeppelin",
+ "components":[
+ {
+ "name":"zeppelin"
+ }
+ ],
+ "dependencies":[
+
+ ]
+ },
"zookeeper":{
"label":"ZooKeeper",
"components":[
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
index 0396c28..b4f1058 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
@@ -35,6 +35,12 @@
<scriptType>PYTHON</scriptType>
<timeout>1200</timeout>
</commandScript>
+ <logs>
+ <log>
+ <logId>flume_handler</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index e3a8f98..ffa9e13 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -84,6 +84,12 @@
<script>scripts/logsearch_solr.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <logs>
+ <log>
+ <logId>logsearch_solr</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
<dependencies>
<dependency>
<name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index d21a97f..4bc1004 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -247,6 +247,7 @@ metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_d
atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')
accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')
falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')
+flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flume')
hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')
hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')
hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive')
@@ -259,6 +260,7 @@ ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ra
storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm')
yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')
mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')
+zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')
#####################################
@@ -328,8 +330,8 @@ logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.c
logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower()
logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5)
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka',
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in
logfeeder_supported_services]
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
new file mode 100644
index 0000000..d1ea632
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"flume_handler",
+ "rowtype":"service",
+ "path":"{{flume_log_dir}}/flume.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "flume_handler"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
index fb47e77..42c5683 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
@@ -26,6 +26,11 @@
"type":"hbase_regionserver",
"rowtype":"service",
"path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+ },
+ {
+ "type":"hbase_phoenix_server",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/phoenix-hbase-server.log"
}
],
@@ -55,6 +60,30 @@
}
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hbase_phoenix_server"
+ ]
+
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
index ba151b9..31c37bf 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
@@ -31,6 +31,11 @@
"type":"logsearch_perf",
"rowtype":"service",
"path":"{{logsearch_log_dir}}/logsearch-performance.json"
+ },
+ {
+ "type":"logsearch_solr",
+ "rowtype":"service",
+ "path":"{{logsearch_solr_log_dir}}/solr.log"
}
],
@@ -49,6 +54,31 @@
}
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "logsearch_solr"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
new file mode 100644
index 0000000..09cd8b3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"zeppelin",
+ "rowtype":"service",
+ "path":"{{zeppelin_log_dir}}/zeppelin-zeppelin-*.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "zeppelin"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
index c40a593..4f19b43 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
@@ -53,6 +53,12 @@ limitations under the License.
</auto-deploy>
</dependency>
</dependencies>
+ <logs>
+ <log>
+ <logId>zeppelin</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
index 0614cf7..d7401d8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
@@ -33,6 +33,12 @@
<script>scripts/phoenix_queryserver.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <logs>
+ <log>
+ <logId>hbase_phoenix_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 0613770..26e8934 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -98,8 +98,8 @@ class TestLogFeeder(RMFTestCase):
encoding='utf-8'
)
- logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+ logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka',
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag
in logfeeder_supported_services]