You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/06/08 17:59:44 UTC
ambari git commit: AMBARI-17089. Log Search capability for Nifi
(Miklos Gergely via oleewere)
Repository: ambari
Updated Branches:
refs/heads/trunk 6a4cd4774 -> 1c8bfce48
AMBARI-17089. Log Search capability for Nifi (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1c8bfce4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1c8bfce4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1c8bfce4
Branch: refs/heads/trunk
Commit: 1c8bfce485edf41ca04738b9898be469ca7f2fb0
Parents: 6a4cd47
Author: oleewere <ol...@gmail.com>
Authored: Wed Jun 8 19:55:49 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Jun 8 19:55:49 2016 +0200
----------------------------------------------------------------------
.../LOGSEARCH/0.5.0/package/scripts/params.py | 4 +-
.../package/templates/input.config-nifi.json.j2 | 98 ++++++++++++++++++++
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 2 +-
3 files changed, 102 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c8bfce4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 34583ba..7acdec2 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -230,6 +230,8 @@ hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hba
hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')
hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive')
kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')
+nifi_master_log_dir = default('/configurations/nifi-bootstrap-env/nifi_master_log_dir', '/var/log/nifi_master')
+nifi_node_log_dir = default('/configurations/nifi-bootstrap-env/nifi_nod_log_dir', '/var/log/nifi_node')
oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')
ranger_usersync_log_dir = default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')
ranger_admin_log_dir = default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')
@@ -308,7 +310,7 @@ logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/
logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5)
logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in
logfeeder_supported_services]
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c8bfce4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-nifi.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-nifi.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-nifi.json.j2
new file mode 100644
index 0000000..1bbab20
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-nifi.json.j2
@@ -0,0 +1,98 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"nifi_app_master",
+ "rowtype":"service",
+ "path":"{{nifi_master_log_dir}}/nifi-app.log"
+ },
+ {
+ "type":"nifi_bootstrap_master",
+ "rowtype":"service",
+ "path":"{{nifi_master_log_dir}}/nifi-bootstrap.log"
+ },
+ {
+ "type":"nifi_setup_master",
+ "rowtype":"service",
+ "path":"{{nifi_master_log_dir}}/nifi-setup.log"
+ },
+ {
+ "type":"nifi_user_master",
+ "rowtype":"service",
+ "path":"{{nifi_master_log_dir}}/nifi-user.log"
+ },
+ {
+ "type":"nifi_app_node",
+ "rowtype":"service",
+ "path":"{{nifi_node_log_dir}}/nifi-app.log"
+ },
+ {
+ "type":"nifi_bootstrap_node",
+ "rowtype":"service",
+ "path":"{{nifi_node_log_dir}}/nifi-bootstrap.log"
+ },
+ {
+ "type":"nifi_setup_node",
+ "rowtype":"service",
+ "path":"{{nifi_node_log_dir}}/nifi-setup.log"
+ },
+ {
+ "type":"nifi_user_node",
+ "rowtype":"service",
+ "path":"{{nifi_node_log_dir}}/nifi-user.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "nifi_app_master",
+ "nifi_bootstrap_master",
+ "nifi_setup_master",
+ "nifi_user_master",
+ "nifi_app_node",
+ "nifi_bootstrap_node",
+ "nifi_setup_node",
+ "nifi_user_node"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c8bfce4/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 54e08e4..981319c 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -89,7 +89,7 @@ class TestLogFeeder(RMFTestCase):
)
logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
- 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+ 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag
in logfeeder_supported_services]