You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2017/04/12 13:02:34 UTC
[3/5] ambari git commit: AMBARI-20578 Log Search Configuration API
(mgergely)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/input.config-ambari-metrics.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/input.config-ambari-metrics.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/input.config-ambari-metrics.json.j2
new file mode 100644
index 0000000..ef823b2
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/input.config-ambari-metrics.json.j2
@@ -0,0 +1,169 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"ams_hbase_master",
+ "rowtype":"service",
+ "path":"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/hbase-ams-master-*.log"
+ },
+ {
+ "type":"ams_hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/hbase-ams-regionserver-*.log"
+ },
+ {
+ "type":"ams_collector",
+ "rowtype":"service",
+ "path":"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/ambari-metrics-collector.log"
+ },
+ {
+ "type":"ams_monitor",
+ "rowtype":"service",
+ "path":"{{default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor')}}/ambari-metrics-monitor.out"
+ },
+ {
+ "type":"ams_grafana",
+ "rowtype":"service",
+ "path":"{{default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana')}}/grafana.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_collector"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %p %c: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_hbase_master",
+ "ams_hbase_regionserver"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_grafana"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{DATESTAMP:logtime})",
+ "message_pattern":"(?m)^%{DATESTAMP:logtime}%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy/MM/dd HH:mm:ss"
+ }
+ },
+ "level":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"I",
+ "post_value":"INFO"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"W",
+ "post_value":"WARN"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"D",
+ "post_value":"DEBUG"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"E",
+ "post_value":"ERROR"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"F",
+ "post_value":"FATAL"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_monitor"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ },
+ "level":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml
deleted file mode 100644
index 71a08fb..0000000
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>Atlas</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>ATLAS_SERVER:atlas_app</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"atlas_app",
- "rowtype":"service",
- "path":"{{default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')}}/application.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "atlas_app"
- ]
- }
- },
- "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/input.config-atlas.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/input.config-atlas.json.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/input.config-atlas.json.j2
new file mode 100644
index 0000000..2d977b9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/input.config-atlas.json.j2
@@ -0,0 +1,48 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"atlas_app",
+ "rowtype":"service",
+ "path":"{{default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')}}/application.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "atlas_app"
+ ]
+ }
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml
deleted file mode 100644
index 6b43ba6..0000000
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>Falcon</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>FALCON_SERVER:falcon_app</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"falcon_app",
- "rowtype":"service",
- "path":"{{default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')}}/falcon.application.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "falcon_app"
- ]
- }
- },
- "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/input.config-falcon.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/input.config-falcon.json.j2 b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/input.config-falcon.json.j2
new file mode 100644
index 0000000..7c5aede
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/input.config-falcon.json.j2
@@ -0,0 +1,48 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"falcon_app",
+ "rowtype":"service",
+ "path":"{{default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')}}/falcon.application.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "falcon_app"
+ ]
+ }
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml
deleted file mode 100644
index 98e6db8..0000000
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>Flume</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>FLUME:flume_handler</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"flume_handler",
- "rowtype":"service",
- "path":"{{default('/configurations/flume-env/flume_log_dir', '/var/log/flume')}}/flume.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "flume_handler"
- ]
- }
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
- "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/input.config-flume.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/input.config-flume.json.j2 b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/input.config-flume.json.j2
new file mode 100644
index 0000000..12d7aad
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/input.config-flume.json.j2
@@ -0,0 +1,53 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{# flume.conf: Add your flume configuration here and start flume
+# Note if you are using the Windows srvice or Unix service
+# provided by the HDP distribution, they will assume the
+# agent's name in this file to be 'a1'
+#}
+{
+ "input":[
+ {
+ "type":"flume_handler",
+ "rowtype":"service",
+ "path":"{{default('/configurations/flume-env/flume_log_dir', '/var/log/flume')}}/flume.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "flume_handler"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml
deleted file mode 100644
index 321ea4e..0000000
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml
+++ /dev/null
@@ -1,111 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>HBase</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>HBASE_MASTER:hbase_master;HBASE_REGIONSERVER:hbase_regionserver;PHOENIX_QUERY_SERVER:hbase_phoenix_server</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"hbase_master",
- "rowtype":"service",
- "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-master-*.log"
- },
- {
- "type":"hbase_regionserver",
- "rowtype":"service",
- "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-regionserver-*.log"
- },
- {
- "type":"hbase_phoenix_server",
- "rowtype":"service",
- "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/phoenix-*-server.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hbase_master",
- "hbase_regionserver"
- ]
- }
- },
- "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hbase_phoenix_server"
- ]
- }
- },
- "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/input.config-hbase.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/input.config-hbase.json.j2
new file mode 100644
index 0000000..94fbc64
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/input.config-hbase.json.j2
@@ -0,0 +1,79 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hbase_master",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-master-*.log"
+ },
+ {
+ "type":"hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-regionserver-*.log"
+ },
+ {
+ "type":"hbase_phoenix_server",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/phoenix-*-server.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hbase_master",
+ "hbase_regionserver"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hbase_phoenix_server"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml
deleted file mode 100644
index d85a028..0000000
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml
+++ /dev/null
@@ -1,248 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>HDFS</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>NAMENODE:hdfs_namenode;DATANODE:hdfs_datanode;SECONDARY_NAMENODE:hdfs_secondarynamenode;JOURNALNODE:hdfs_journalnode;ZKFC:hdfs_zkfc;NFS_GATEWAY:hdfs_nfs3</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"hdfs_datanode",
- "rowtype":"service",
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-datanode-*.log"
- },
- {
- "type":"hdfs_namenode",
- "rowtype":"service",
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-namenode-*.log"
- },
- {
- "type":"hdfs_journalnode",
- "rowtype":"service",
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-journalnode-*.log"
- },
- {
- "type":"hdfs_secondarynamenode",
- "rowtype":"service",
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-secondarynamenode-*.log"
- },
- {
- "type":"hdfs_zkfc",
- "rowtype":"service",
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-zkfc-*.log"
- },
- {
- "type":"hdfs_nfs3",
- "rowtype":"service",
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-nfs3-*.log"
- },
- {
- "type":"hdfs_audit",
- "rowtype":"audit",
- "is_enabled":"true",
- "add_fields":{
- "logType":"HDFSAudit",
- "enforcer":"hadoop-acl",
- "repoType":"1",
- "repo":"hdfs"
- },
- "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hdfs-audit.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hdfs_datanode",
- "hdfs_journalnode",
- "hdfs_secondarynamenode",
- "hdfs_namenode",
- "hdfs_zkfc",
- "hdfs_nfs3"
- ]
- }
- },
- "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hdfs_audit"
- ]
- }
- },
- "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "evtTime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- },
- {
- "filter":"keyvalue",
- "sort_order":1,
- "conditions":{
- "fields":{
- "type":[
- "hdfs_audit"
- ]
- }
- },
- "source_field":"log_message",
- "value_split":"=",
- "field_split":"\t",
- "post_map_values":{
- "src":{
- "map_fieldname":{
- "new_fieldname":"resource"
- }
- },
- "ip":{
- "map_fieldname":{
- "new_fieldname":"cliIP"
- }
- },
- "allowed":[
- {
- "map_fieldvalue":{
- "pre_value":"true",
- "post_value":"1"
- }
- },
- {
- "map_fieldvalue":{
- "pre_value":"false",
- "post_value":"0"
- }
- },
- {
- "map_fieldname":{
- "new_fieldname":"result"
- }
- }
- ],
- "cmd":{
- "map_fieldname":{
- "new_fieldname":"action"
- }
- },
- "proto":{
- "map_fieldname":{
- "new_fieldname":"cliType"
- }
- },
- "callerContext":{
- "map_fieldname":{
- "new_fieldname":"req_caller_id"
- }
- }
- }
- },
- {
- "filter":"grok",
- "sort_order":2,
- "source_field":"ugi",
- "remove_source_field":"false",
- "conditions":{
- "fields":{
- "type":[
- "hdfs_audit"
- ]
- }
- },
- "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
- "post_map_values":{
- "user":{
- "map_fieldname":{
- "new_fieldname":"reqUser"
- }
- },
- "x_user":{
- "map_fieldname":{
- "new_fieldname":"reqUser"
- }
- },
- "p_user":{
- "map_fieldname":{
- "new_fieldname":"reqUser"
- }
- },
- "k_user":{
- "map_fieldname":{
- "new_fieldname":"proxyUsers"
- }
- },
- "p_authType":{
- "map_fieldname":{
- "new_fieldname":"authType"
- }
- },
- "k_authType":{
- "map_fieldname":{
- "new_fieldname":"proxyAuthType"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/input.config-hdfs.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/input.config-hdfs.json.j2
new file mode 100644
index 0000000..ed7abf6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/input.config-hdfs.json.j2
@@ -0,0 +1,216 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hdfs_datanode",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-datanode-*.log"
+ },
+ {
+ "type":"hdfs_namenode",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-namenode-*.log"
+ },
+ {
+ "type":"hdfs_journalnode",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-journalnode-*.log"
+ },
+ {
+ "type":"hdfs_secondarynamenode",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-secondarynamenode-*.log"
+ },
+ {
+ "type":"hdfs_zkfc",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-zkfc-*.log"
+ },
+ {
+ "type":"hdfs_nfs3",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-nfs3-*.log"
+ },
+ {
+ "type":"hdfs_audit",
+ "rowtype":"audit",
+ "is_enabled":"true",
+ "add_fields":{
+ "logType":"HDFSAudit",
+ "enforcer":"hadoop-acl",
+ "repoType":"1",
+ "repo":"hdfs"
+ },
+ "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hdfs-audit.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_datanode",
+ "hdfs_journalnode",
+ "hdfs_secondarynamenode",
+ "hdfs_namenode",
+ "hdfs_zkfc",
+ "hdfs_nfs3"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "evtTime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"keyvalue",
+ "sort_order":1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+ }
+ },
+ "source_field":"log_message",
+ "value_split":"=",
+ "field_split":"\t",
+ "post_map_values":{
+ "src":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+ },
+ "ip":{
+ "map_fieldname":{
+ "new_fieldname":"cliIP"
+ }
+ },
+ "allowed":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"true",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"false",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+ }
+ ],
+ "cmd":{
+ "map_fieldname":{
+ "new_fieldname":"action"
+ }
+ },
+ "proto":{
+ "map_fieldname":{
+ "new_fieldname":"cliType"
+ }
+ },
+ "callerContext":{
+ "map_fieldname":{
+ "new_fieldname":"req_caller_id"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "sort_order":2,
+ "source_field":"ugi",
+ "remove_source_field":"false",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+ }
+ },
+ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+ "post_map_values":{
+ "user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+ },
+ "x_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+ },
+ "p_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+ },
+ "k_user":{
+ "map_fieldname":{
+ "new_fieldname":"proxyUsers"
+ }
+ },
+ "p_authType":{
+ "map_fieldname":{
+ "new_fieldname":"authType"
+ }
+ },
+ "k_authType":{
+ "map_fieldname":{
+ "new_fieldname":"proxyAuthType"
+ }
+ }
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml
deleted file mode 100644
index c1b971c..0000000
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml
+++ /dev/null
@@ -1,117 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>Hive</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>HIVE_METASTORE:hive_metastore;HIVE_SERVER:hive_hiveserver2;WEBHCAT_SERVER:webhcat_server</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"hive_hiveserver2",
- "rowtype":"service",
- "path":"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hiveserver2.log"
- },
- {
- "type":"hive_metastore",
- "rowtype":"service",
- "path":"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hivemetastore.log"
- },
- {
- "type": "webhcat_server",
- "rowntype":"service",
- "path":"{{default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat')}}/webhcat.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hive_hiveserver2",
- "hive_metastore"
- ]
- }
- },
- "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "webhcat_server"
- ]
- }
- },
- "log4j_format":" %-5p | %d{DATE} | %c | %m%n",
- "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})",
- "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
- }
- },
- "level":{
- "map_fieldvalue":{
- "pre_value":"WARNING",
- "post_value":"WARN"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2
new file mode 100644
index 0000000..8697cf5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2
@@ -0,0 +1,85 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hive_hiveserver2",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hiveserver2.log"
+ },
+ {
+ "type":"hive_metastore",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hivemetastore.log"
+ },
+ {
+ "type": "webhcat_server",
+ "rowntype":"service",
+ "path":"{{default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat')}}/webhcat.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hive_hiveserver2",
+ "hive_metastore"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "webhcat_server"
+ ]
+ }
+ },
+ "log4j_format":" %-5p | %d{DATE} | %c | %m%n",
+ "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})",
+ "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+ }
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml
deleted file mode 100644
index 29a8b36..0000000
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml
+++ /dev/null
@@ -1,124 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>Kafka</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>KAFKA_BROKER:kafka_server,kafka_request,kafka_logcleaner,kafka_controller,kafka_statechange</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"kafka_controller",
- "rowtype":"service",
- "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/controller.log"
- },
- {
- "type":"kafka_request",
- "rowtype":"service",
- "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/kafka-request.log"
- },
- {
- "type":"kafka_logcleaner",
- "rowtype":"service",
- "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/log-cleaner.log"
- },
- {
- "type":"kafka_server",
- "rowtype":"service",
- "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/server.log"
- },
- {
- "type":"kafka_statechange",
- "rowtype":"service",
- "path":"{{kafka_log_dir}}/state-change.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "kafka_controller",
- "kafka_request",
- "kafka_logcleaner"
- ]
- }
- },
- "log4j_format":"[%d] %p %m (%c)%n",
- "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
- "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- },
- {
- "filter":"grok",
- "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
- "conditions":{
- "fields":{
- "type":[
- "kafka_server",
- "kafka_statechange"
- ]
- }
- },
- "log4j_format":"[%d] %p %m (%c)%n",
- "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
- "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2
new file mode 100644
index 0000000..5b8f896
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2
@@ -0,0 +1,92 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"kafka_controller",
+ "rowtype":"service",
+ "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/controller.log"
+ },
+ {
+ "type":"kafka_request",
+ "rowtype":"service",
+ "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/kafka-request.log"
+ },
+ {
+ "type":"kafka_logcleaner",
+ "rowtype":"service",
+ "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/log-cleaner.log"
+ },
+ {
+ "type":"kafka_server",
+ "rowtype":"service",
+ "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/server.log"
+ },
+ {
+ "type":"kafka_statechange",
+ "rowtype":"service",
+ "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/state-change.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_controller",
+ "kafka_request",
+ "kafka_logcleaner"
+ ]
+ }
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_server",
+ "kafka_statechange"
+ ]
+ }
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml
deleted file mode 100644
index 528b011..0000000
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_name</name>
- <display-name>Service name</display-name>
- <description>Service name for Logsearch Portal (label)</description>
- <value>Knox</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>component_mappings</name>
- <display-name>Component mapping</display-name>
- <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
- <value>KNOX_GATEWAY:knox_gateway,knox_cli,knox_ldap</value>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Logfeeder Config</display-name>
- <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
- <value>
-{
- "input":[
- {
- "type":"knox_gateway",
- "rowtype":"service",
- "path":"/var/log/knox/gateway.log"
- },
- {
- "type":"knox_cli",
- "rowtype":"service",
- "path":"/var/log/knox/knoxcli.log"
- },
- {
- "type":"knox_ldap",
- "rowtype":"service",
- "path":"/var/log/knox/ldap.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "knox_gateway",
- "knox_cli",
- "knox_ldap"
- ]
-
- }
- },
- "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
- }
- }
- ]
- }
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/templates/input.config-knox.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/templates/input.config-knox.json.j2
new file mode 100644
index 0000000..6d7cf72
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/templates/input.config-knox.json.j2
@@ -0,0 +1,60 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"knox_gateway",
+ "rowtype":"service",
+ "path":"/var/log/knox/gateway.log"
+ },
+ {
+ "type":"knox_cli",
+ "rowtype":"service",
+ "path":"/var/log/knox/knoxcli.log"
+ },
+ {
+ "type":"knox_ldap",
+ "rowtype":"service",
+ "path":"/var/log/knox/ldap.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "knox_gateway",
+ "knox_cli",
+ "knox_ldap"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml
index 19bbd4a..fce4c10 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml
@@ -31,6 +31,7 @@
<show-property-name>false</show-property-name>
<property-file-name>input.config-ambari.json.j2</property-file-name>
<property-file-type>text</property-file-type>
+ <visible>false</visible>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
deleted file mode 100644
index 30a93a5..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>service_component_mappings</name>
- <display-name>Service Component mapping</display-name>
- <description>Log Search service component logid mapping list (e.g.: SERVICE1=S1_COMPONENT1:logid1,logid2;S1_COMPONENT2:logid3|SERVICE2=...)</description>
- <value></value>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>content</name>
- <display-name>Log Feeder Config</display-name>
- <description>Metadata jinja template for Log Feeder which contains grok patterns for reading service specific logs.</description>
- <value>
- </value>
- <value-attributes>
- <type>content</type>
- <show-property-name>false</show-property-name>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index d00be4e..84a6ad9 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -81,7 +81,6 @@ cluster_name = str(config['clusterName'])
configurations = config['configurations'] # need reference inside logfeeder jinja templates
logserch_meta_configs = get_logsearch_meta_configs(configurations)
logsearch_metadata = get_logsearch_metadata(logserch_meta_configs)
-logfeeder_metadata = get_logfeeder_metadata(logserch_meta_configs)
# for now just pick first collector
if 'metrics_collector_hosts' in config['clusterHostInfo']:
@@ -256,14 +255,6 @@ logsearch_properties['logsearch.auth.simple.enabled'] = 'false'
logsearch_properties['logsearch.protocol'] = logsearch_ui_protocol
-logsearch_acls = ''
-if 'infra-solr-env' in config['configurations'] and security_enabled and not logsearch_use_external_solr:
- acl_infra_solr_principal = get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal'])
- acl_logsearch_principal = get_name_from_principal(config['configurations']['logsearch-env']['logsearch_kerberos_principal'])
- logsearch_acls = format('world:anyone:r,sasl:{acl_infra_solr_principal}:cdrwa,sasl:{acl_logsearch_principal}:cdrwa')
- logsearch_properties['logsearch.solr.zk.acls'] = logsearch_acls
- logsearch_properties['logsearch.solr.audit.logs.zk.acls'] = logsearch_acls
-
# load config values
logsearch_properties = dict(logsearch_properties.items() + dict(config['configurations']['logsearch-properties']).items())
@@ -279,6 +270,9 @@ logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearc
logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url'])
logsearch_properties['logsearch.spnego.kerberos.host'] = logsearch_spnego_host
+if not('logsearch.config.zk_connect_string' in logsearch_properties):
+ logsearch_properties['logsearch.config.zk_connect_string'] = logsearch_solr_zk_quorum
+
if logsearch_solr_kerberos_enabled:
logsearch_properties['logsearch.solr.kerberos.enable'] = 'true'
logsearch_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file
@@ -330,17 +324,7 @@ logfeeder_env_jceks_file = os.path.join(logsearch_logfeeder_conf, 'logfeeder.jce
logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content']
logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content']
-logfeeder_default_services = ['logsearch']
-logfeeder_default_config_file_names = ['global.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services]
-logfeeder_custom_config_file_names = ['input.config-%s.json' % (tag.replace('-logsearch-conf', ''))
- for tag, content in logfeeder_metadata.iteritems() if any(logfeeder_metadata)]
-
-if logfeeder_system_log_enabled:
- default_config_files = ','.join(['output.config.json','input.config-ambari.json'] + logfeeder_default_config_file_names + logfeeder_custom_config_file_names
- + ['input.config-system_messages.json', 'input.config-secure_log.json'])
-else:
- default_config_files = ','.join(['output.config.json','input.config-ambari.json'] + logfeeder_default_config_file_names + logfeeder_custom_config_file_names)
-
+default_config_files = ','.join(['output.config.json','global.config.json'])
logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns']
if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip():
@@ -367,14 +351,19 @@ logfeeder_properties = dict(logfeeder_properties.items() + dict(config['configur
# load derivated values
-logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts'])
+logfeeder_properties['cluster.name'] = cluster_name
+logfeeder_properties['logfeeder.config.dir'] = logsearch_logfeeder_conf
logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files'])
logfeeder_properties['logfeeder.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode
+logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts'])
logfeeder_properties['logfeeder.metrics.collector.protocol'] = metrics_collector_protocol
logfeeder_properties['logfeeder.metrics.collector.port'] = metrics_collector_port
logfeeder_properties['logfeeder.metrics.collector.path'] = '/ws/v1/timeline/metrics'
+if not('logsearch.config.zk_connect_string' in logfeeder_properties):
+ logfeeder_properties['logsearch.config.zk_connect_string'] = logsearch_solr_zk_quorum
+
if logsearch_solr_kerberos_enabled:
if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties:
logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
@@ -387,6 +376,19 @@ logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder'
logfeeder_use_ssl = logsearch_solr_ssl_enabled or metrics_collector_protocol == 'https'
+
+logsearch_acls = ''
+if 'infra-solr-env' in config['configurations'] and security_enabled and not logsearch_use_external_solr:
+ acl_infra_solr_principal = get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal'])
+ acl_logsearch_principal = get_name_from_principal(config['configurations']['logsearch-env']['logsearch_kerberos_principal'])
+ logsearch_acls = format('world:anyone:r,sasl:{acl_infra_solr_principal}:cdrwa,sasl:{acl_logsearch_principal}:cdrwa')
+ logsearch_properties['logsearch.solr.zk.acls'] = logsearch_acls
+ logsearch_properties['logsearch.solr.audit.logs.zk.acls'] = logsearch_acls
+ if not('logsearch.config.zk_acls' in logsearch_properties):
+ logsearch_properties['logsearch.config.zk_acls'] = logsearch_acls
+ if not('logsearch.config.zk_acls' in logfeeder_properties):
+ logfeeder_properties['logsearch.config.zk_acls'] = logsearch_acls
+
#####################################
# Smoke command
#####################################
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ac0ba42/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index e6e55b9..653d604 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -99,6 +99,10 @@ def setup_logfeeder():
encoding="utf-8"
)
+ File(format("{logsearch_logfeeder_conf}/global.config.json"),
+ content=Template("global.config.json.j2")
+ )
+
File(format("{logsearch_logfeeder_conf}/input.config-ambari.json"),
content=InlineTemplate(params.logfeeder_ambari_config_content),
encoding="utf-8"
@@ -109,17 +113,6 @@ def setup_logfeeder():
encoding="utf-8"
)
- for file_name in params.logfeeder_default_config_file_names:
- File(format("{logsearch_logfeeder_conf}/" + file_name),
- content=Template(file_name + ".j2")
- )
-
- File(format("{logsearch_logfeeder_conf}/input.config-logfeeder-custom.json"), action='delete')
- for service, pattern_content in params.logfeeder_metadata.iteritems():
- File(format("{logsearch_logfeeder_conf}/input.config-" + service.replace('-logsearch-conf', '') + ".json"),
- content=InlineTemplate(pattern_content, extra_imports=[default])
- )
-
if params.logfeeder_system_log_enabled:
File(format("{logsearch_logfeeder_conf}/input.config-system_messages.json"),
content=params.logfeeder_system_messages_content