You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/10/27 09:48:08 UTC
[2/4] ambari git commit: AMBARI-18548. Declarative
Logsearch/Logfeeder Component Metadata for Stack Component (oleewere)
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml
new file mode 100644
index 0000000..0bf91ff
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>service_name</name>
+ <display-name>Service name</display-name>
+ <description>Service name for Logsearch Portal (label)</description>
+ <value>Knox</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>component_mappings</name>
+ <display-name>Component mapping</display-name>
+ <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description>
+ <value>KNOX_GATEWAY:knox_gateway,knox_cli,knox_ldap</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>Logfeeder Config</display-name>
+ <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description>
+ <value>
+{
+ "input":[
+ {
+ "type":"knox_gateway",
+ "rowtype":"service",
+ "path":"/var/log/dir/gateway.log"
+ },
+ {
+ "type":"knox_cli",
+ "rowtype":"service",
+ "path":"/var/log/knox/knoxcli.log"
+ },
+ {
+ "type":"knox_ldap",
+ "rowtype":"service",
+ "path":"/var/log/knox/ldap.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "knox_gateway",
+ "knox_cli",
+ "knox_ldap"
+ ]
+
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+ }
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml
new file mode 100644
index 0000000..b8dd4b9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+ <property>
+ <name>logfeeder_system_messages_content</name>
+ <display-name>System messages grok filter</display-name>
+ <description>This is the jinja template for input.config.system_messages.json file</description>
+ <value/>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>system_messages.json</property-file-name>
+ <property-file-type>text</property-file-type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>logfeeder_secure_log_content</name>
+ <display-name>Secure log grok filter</display-name>
+ <description>This is the jinja template for input.config.secure_log.json file</description>
+ <value/>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>secure_log.json</property-file-name>
+ <property-file-type>text</property-file-type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>logfeeder_system_log_enabled</name>
+ <value>false</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
index cfc6c91..fbc29c8 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
@@ -25,7 +25,6 @@ from resource_management.libraries.script.script import Script
from setup_logfeeder import setup_logfeeder
from logsearch_common import kill_process
-
class LogFeeder(Script):
def install(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
index 9c5215d..4410712 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
@@ -24,7 +24,6 @@ from resource_management.libraries.script.script import Script
from setup_logsearch import setup_logsearch
from logsearch_common import kill_process
-
class LogSearch(Script):
def install(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
new file mode 100644
index 0000000..2e9dda3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management import Logger
+
+def __parse_component_mappings(component_mappings):
+ components = list()
+ component_mappings_list = component_mappings.split(';')
+ if component_mappings_list and len(component_mappings_list) > 0:
+ metadata_list = map(lambda x : x.split(':'), component_mappings_list)
+ if metadata_list and len(metadata_list) > 0:
+ for metadata in metadata_list:
+ if (len(metadata) == 2):
+ logids = metadata[1].split(',')
+ components.extend(logids)
+ Logger.info("Found logids for logsearch component %s - (%s) " % (metadata[0], metadata[1]))
+ return components
+
+def get_logsearch_meta_configs(configurations):
+ logsearch_meta_configs = {}
+ for key, value in configurations.iteritems(): # iter on both keys and values
+ if str(key).endswith('logsearch-conf'):
+ logsearch_meta_configs[key] = value
+ Logger.info("Found logsearch config entry : " + key)
+ return logsearch_meta_configs
+
+def get_logfeeder_metadata(logsearch_meta_configs):
+ """
+ get logfeeder pattern metadata list, an element: (e.g.) :
+ ['service_config_name' : 'pattern json content']
+ """
+ logfeeder_contents = {}
+ for key, value in logsearch_meta_configs.iteritems():
+ if 'content' in logsearch_meta_configs[key]:
+ logfeeder_contents[key] = logsearch_meta_configs[key]['content']
+ Logger.info("Found logfeeder pattern content in " + key)
+ return logfeeder_contents
+
+def get_logsearch_metadata(logsearch_meta_configs):
+ """
+ get logsearch metadata list, an element (e.g.) :
+ ['service_name_key' : {component1 : [logid1, logid2]}, {component2 : [logid1, logid2]}]
+ """
+ logsearch_service_component_mappings = {}
+ for key, value in logsearch_meta_configs.iteritems():
+ if 'service_name' in logsearch_meta_configs[key] and 'component_mappings':
+ service_name = logsearch_meta_configs[key]['service_name']
+ component_mappings = __parse_component_mappings(logsearch_meta_configs[key]['component_mappings'])
+ logsearch_service_component_mappings[service_name] = component_mappings
+
+ return logsearch_service_component_mappings
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 4526640..1f5064a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -19,6 +19,7 @@ limitations under the License.
"""
from ambari_commons.constants import AMBARI_SUDO_BINARY
+from logsearch_config_aggregator import get_logfeeder_metadata, get_logsearch_metadata, get_logsearch_meta_configs
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.is_empty import is_empty
@@ -66,6 +67,11 @@ zookeeper_hosts = ",".join(zookeeper_hosts_list)
cluster_name = str(config['clusterName'])
availableServices = config['availableServices']
+configurations = config['configurations'] # need reference inside logfeeder jinja templates
+logserch_meta_configs = get_logsearch_meta_configs(configurations)
+logsearch_metadata = get_logsearch_metadata(logserch_meta_configs)
+logfeeder_metadata = get_logfeeder_metadata(logserch_meta_configs)
+
# for now just pick first collector
if 'metrics_collector_hosts' in config['clusterHostInfo']:
metrics_collector_hosts_list = ",".join(config['clusterHostInfo']['metrics_collector_hosts'])
@@ -139,41 +145,13 @@ logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['conte
# Log dirs
ambari_server_log_dir = '/var/log/ambari-server'
ambari_agent_log_dir = '/var/log/ambari-agent'
-knox_log_dir = '/var/log/knox'
hst_log_dir = '/var/log/hst'
hst_activity_log_dir = '/var/log/smartsense-activity'
-metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')
-metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor')
-metrics_grafana_log_dir = default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana')
-
-atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')
-accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')
-falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')
-flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flume')
-hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')
-hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')
-hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive')
-hcat_log_dir = default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat')
-infra_solr_log_dir = default('configurations/infra-solr-env/infra_solr_log_dir', '/var/log/ambari-infra-solr')
-kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')
-nifi_log_dir = default('/configurations/nifi-env/nifi_node_log_dir', '/var/log/nifi')
-oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')
-ranger_usersync_log_dir = default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')
-ranger_admin_log_dir = default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')
-ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')
-storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm')
-yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')
-mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')
-zeppelin_log_dir = default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
-zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')
-spark_log_dir = default('/configurations/spark-env/spark_log_dir', '/var/log/spark')
-livy_log_dir = default('/configurations/livy-env/livy_log_dir', '/var/log/livy')
-spark2_log_dir = default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')
-
-hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs')
-mapred_user = default('configurations/mapred-env/mapred_user', 'mapred')
-yarn_user = default('configurations/yarn-env/yarn_user', 'yarn')
+# System logs
+logfeeder_system_messages_content = config['configurations']['logfeeder-system_log-env']['logfeeder_system_messages_content']
+logfeeder_secure_log_content = config['configurations']['logfeeder-system_log-env']['logfeeder_secure_log_content']
+logfeeder_system_log_enabled = default('/configurations/logfeeder-system_log-env/logfeeder_system_log_enabled', False)
# Logsearch auth configs
@@ -272,14 +250,17 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe
logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password']
logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type']
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra',
- 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn',
- 'zeppelin', 'zookeeper']
+logfeeder_default_services = ['ambari', 'logsearch']
+logfeeder_default_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services]
+logfeeder_custom_config_file_names = ['input.config-%s.json' % (tag.replace('-logsearch-conf', ''))
+ for tag, content in logfeeder_metadata.iteritems() if any(logfeeder_metadata)]
-logfeeder_config_file_names = \
- ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
+if logfeeder_system_log_enabled:
+ default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names
+ + ['input.config-system_messages.json', 'input.config-secure_log.json'])
+else:
+ default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names)
-default_config_files = ','.join(logfeeder_config_file_names)
logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns']
if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip():
@@ -292,8 +273,6 @@ if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip():
'\n' + \
config['configurations']['logfeeder-grok']['custom_grok_patterns']
-logfeeder_properties = dict(config['configurations']['logfeeder-properties'])
-
# logfeeder properties
# load default values
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index ad16958..e90d623 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -17,6 +17,7 @@ limitations under the License.
"""
+from resource_management.libraries.functions.default import default
from resource_management.core.resources.system import Directory, File
from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate, Template
@@ -61,11 +62,25 @@ def setup_logfeeder():
encoding="utf-8"
)
- for file_name in params.logfeeder_config_file_names:
+ for file_name in params.logfeeder_default_config_file_names:
File(format("{logsearch_logfeeder_conf}/" + file_name),
content=Template(file_name + ".j2")
)
+ for service, pattern_content in params.logfeeder_metadata.iteritems():
+ File(format("{logsearch_logfeeder_conf}/input.config-" + service.replace('-logsearch-conf', '') + ".json"),
+ content=InlineTemplate(pattern_content, extra_imports=[default])
+ )
+
+ if params.logfeeder_system_log_enabled:
+ File(format("{logsearch_logfeeder_conf}/input.config-system_messages.json"),
+ content=params.logfeeder_system_messages_content
+ )
+ File(format("{logsearch_logfeeder_conf}/input.config-secure_log.json"),
+ content=params.logfeeder_secure_log_content
+ )
+
+
if params.security_enabled:
File(format("{logfeeder_jaas_file}"),
content=Template("logfeeder_jaas.conf.j2")
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
index 7982631..7e9153f 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
@@ -15,520 +15,86 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
-{
- "service": {
-{% if "ACCUMULO" in availableServices %}
- "accumulo": {
- "label": "Accumulo",
+ {
+ "service": {
+{% if logsearch_metadata and logsearch_metadata.keys()|length > 0 %}
+ {% for metadata_key, component_list in logsearch_metadata.iteritems() %}
+ "{{ metadata_key.lower() }}": {
+ "label": "{{ metadata_key }}",
"components": [
+ {% for component in component_list %}
{
- "name": "accumulo_gc"
- },
- {
- "name": "accumulo_master"
- },
- {
- "name": "accumulo_monitor"
- },
- {
- "name": "accumulo_tracer"
- },
- {
- "name": "accumulo_tserver"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "ATLAS" in availableServices %}
- "atlas": {
- "label": "Atlas",
- "components": [
- {
- "name": "atlas_app"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
- "ambari": {
- "label": "Ambari",
- "components": [
- {
- "name": "ambari_agent"
- },
- {
- "name": "ambari_server"
- },
- {
- "name": "ambari_alerts"
- },
- {
- "name": "ambari_audit"
- },
- {
- "name": "ambari_config_changes"
- },
- {
- "name": "ambari_eclipselink"
- },
- {
- "name": "ambari_server_check_database"
- }
- ],
- "dependencies": [
- ]
- },
-{% if "AMBARI_METRICS" in availableServices %}
- "ams": {
- "label": "AMS",
- "components": [
- {
- "name": "ams_hbase_master"
- },
- {
- "name": "ams_hbase_regionserver"
- },
- {
- "name": "ams_collector"
- },
- {
- "name": "ams_monitor"
- },
- {
- "name": "ams_grafana"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "FALCON" in availableServices %}
- "falcon": {
- "label": "Falcon",
- "components": [
- {
- "name": "falcon_app"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "FLUME" in availableServices %}
- "flume": {
- "label": "Flume",
- "components": [
- {
- "name": "flume_handler"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "HBASE" in availableServices %}
- "hbase": {
- "label": "HBase",
- "components": [
- {
- "name": "hbase_master"
- },
- {
- "name": "hbase_regionserver"
- },
- {
- "name": "hbase_phoenix_server"
- }
- ],
- "dependencies": [
- {
- "service": "hdfs",
- "components": [
- "hdfs_namenode"
- ]
- }
- ]
- },
-{% endif %}
-{% if "HDFS" in availableServices %}
- "hdfs": {
- "label": "HDFS",
- "components": [
- {
- "name": "hdfs_datanode"
- },
- {
- "name": "hdfs_namenode"
- },
- {
- "name": "hdfs_journalnode"
- },
- {
- "name": "hdfs_secondarynamenode"
- },
- {
- "name": "hdfs_zkfc"
- },
- {
- "name": "hdfs_nfs3"
- },
- {
- "name": "hdfs_audit",
- "rowtype": "audit"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "HIVE" in availableServices %}
- "hive": {
- "label": "Hive",
- "components": [
- {
- "name": "hive_hiveserver2"
- },
- {
- "name": "hive_metastore"
- },
- {
- "name": "webhcat_server"
- }
- ],
- "dependencies": [
- {
- "service": "hdfs",
- "components": [
- "hdfs_namenode"
- ]
- }
- ]
- },
-{% endif %}
-{% if "KAFKA" in availableServices %}
- "kafka": {
- "label": "Kafka",
- "components": [
- {
- "name": "kafka_controller"
- },
- {
- "name": "kafka_request"
- },
- {
- "name": "kafka_logcleaner"
- },
- {
- "name": "kafka_server"
- },
- {
- "name": "kafka_statechange"
- }
- ],
- "dependencies": [
- {
- "service": "zookeeper",
- "components": [
- "zookeeper"
- ]
- }
- ]
- },
-{% endif %}
- "ambari_infra" : {
- "label": "Infra",
- "components" : [
- {
- "name": "infra_solr"
- }
- ],
- "dependencies": [
- ]
- },
-{% if "KNOX" in availableServices %}
- "knox": {
- "label": "Knox",
- "components": [
- {
- "name": "knox_gateway"
- },
- {
- "name": "knox_cli"
- },
- {
- "name": "knox_ldap"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "NIFI" in availableServices %}
- "nifi": {
- "label": "NiFi",
- "components": [
- {
- "name": "nifi_app"
- },
- {
- "name": "nifi_bootstrap"
- },
- {
- "name": "nifi_setup"
- },
- {
- "name": "nifi_user"
- },
- {
- "name": "nifi_ca"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "MAPREDUCE2" in availableServices %}
- "mapred": {
- "label": "MapReduce",
- "components": [
- {
- "name": "mapred_historyserver"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
- "logsearch": {
- "label": "Logsearch",
- "components": [
- {
- "name": "logsearch_app"
- },
- {
- "name": "logsearch_feeder"
- },
- {
- "name": "logsearch_perf"
- }
- ],
- "dependencies": [
- ]
- },
-{% if "RANGER" in availableServices %}
- "ranger": {
- "label": "Ranger",
- "components": [
- {
- "name": "ranger_admin"
- },
- {
- "name": "ranger_dbpatch"
- },
- {
- "name": "ranger_kms"
- },
- {
- "name": "ranger_usersync"
- }
- ],
- "dependencies": [
- {
- "service": "hdfs",
- "required": "optional",
- "components": [
- "hdfs_namenode"
- ]
- },
- {
- "service": "hbase",
- "required": "optional",
- "components": [
- "hbase_master",
- "hbase_regionserver"
- ]
- },
- {
- "service": "hive",
- "required": "optional",
- "components": [
- "hive_hiveserver2"
- ]
- },
- {
- "service": "kafka",
- "required": "optional",
- "components": [
- "kafka_ranger"
- ]
- },
- {
- "service": "knox",
- "required": "optional",
- "components": [
- "knox_gateway"
- ]
- },
- {
- "service": "storm",
- "required": "optional",
- "components": [
- "storm_supervisor"
- ]
- },
- {
- "service": "yarn",
- "required": "optional",
- "components": [
- "yarn_resourcemanager"
- ]
- }
- ]
- },
-{% endif %}
-{% if "OOZIE" in availableServices %}
- "oozie": {
- "label": "Oozie",
- "components": [
- {
- "name": "oozie_app"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "SMARTSENSE" in availableServices %}
- "hst": {
- "label": "SmartSense",
- "components": [
- {
- "name": "hst_server"
- },
- {
- "name": "hst_agent"
- },
- {
- "name": "activity_analyzer"
- },
- {
- "name": "activity_explorer"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "SPARK" in availableServices %}
- "spark": {
- "label": "Spark",
- "components": [
- {
- "name": "spark_jobhistory_server"
- },
- {
- "name": "spark_thriftserver"
- },
- {
- "name": "livy_server"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "SPARK2" in availableServices %}
- "spark2": {
- "label": "Spark 2",
- "components": [
- {
- "name": "spark2_jobhistory_server"
- },
- {
- "name": "spark2_thriftserver"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "STORM" in availableServices %}
- "storm": {
- "label": "Storm",
- "components": [
- {
- "name": "storm_drpc"
- },
- {
- "name": "storm_logviewer"
- },
- {
- "name": "storm_nimbus"
- },
- {
- "name": "storm_supervisor"
- },
- {
- "name": "storm_ui"
- },
- {
- "name": "storm_worker"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "YARN" in availableServices %}
- "yarn": {
- "label": "YARN",
- "components": [
- {
- "name": "yarn_nodemanager"
- },
- {
- "name": "yarn_resourcemanager"
- },
- {
- "name": "yarn_timelineserver"
- },
- {
- "name": "yarn_historyserver"
- },
- {
- "name": "yarn_jobsummary"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "ZEPPELIN" in availableServices %}
- "zeppelin": {
- "label": "Zeppelin",
- "components": [
- {
- "name": "zeppelin"
- }
- ],
- "dependencies": [
- ]
- },
-{% endif %}
-{% if "ZOOKEEPER" in availableServices %}
- "zookeeper": {
- "label": "ZooKeeper",
- "components": [
- {
- "name": "zookeeper"
+ "name": "{{ component }}"
}
+ {% if not loop.last %}
+ ,
+ {% endif %}
+ {% endfor %}
],
"dependencies": [
]
}
-{% endif %}
+ ,
+ {% endfor %}
+{% endif %}
+{% if logfeeder_system_log_enabled %}
+ "system" : {
+ "label" : "System",
+ "components" : [
+ {
+ "name": "system_message"
+ },
+ {
+ "name": "secure_log"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+ "ambari": {
+ "label": "Ambari",
+ "components": [
+ {
+ "name": "ambari_agent"
+ },
+ {
+ "name": "ambari_server"
+ },
+ {
+ "name": "ambari_alerts"
+ },
+ {
+ "name": "ambari_audit"
+ },
+ {
+ "name": "ambari_config_changes"
+ },
+ {
+ "name": "ambari_eclipselink"
+ },
+ {
+ "name": "ambari_server_check_database"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+ "logsearch": {
+ "label": "Logsearch",
+ "components": [
+ {
+ "name": "logsearch_app"
+ },
+ {
+ "name": "logsearch_feeder"
+ },
+ {
+ "name": "logsearch_perf"
+ }
+ ],
+ "dependencies": [
+ ]
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2
deleted file mode 100644
index 1fd93cf..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2
+++ /dev/null
@@ -1,105 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"accumulo_gc",
- "rowtype":"service",
- "path":"{{accumulo_log_dir}}/gc_*.log"
- },
- {
- "type":"accumulo_master",
- "rowtype":"service",
- "path":"{{accumulo_log_dir}}/master_*.log"
- },
- {
- "type":"accumulo_monitor",
- "rowtype":"service",
- "path":"{{accumulo_log_dir}}/monitor_*.log"
- },
- {
- "type":"accumulo_tracer",
- "rowtype":"service",
- "path":"{{accumulo_log_dir}}/tracer_*.log"
- },
- {
- "type":"accumulo_tserver",
- "rowtype":"service",
- "path":"{{accumulo_log_dir}}/tserver_*.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "accumulo_master"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "comment":"This one has one extra space after LEVEL",
- "conditions":{
- "fields":{
- "type":[
- "accumulo_gc",
- "accumulo_monitor",
- "accumulo_tracer",
- "accumulo_tserver"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
index 36507c9..ad4adb2 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
@@ -31,7 +31,7 @@
"type":"ambari_alerts",
"rowtype":"service",
"add_fields":{
- "level":"INFO"
+ "level":"UNKNOWN"
},
"path":"{{ambari_server_log_dir}}/ambari-alerts.log"
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
deleted file mode 100644
index 0bcaf56..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
+++ /dev/null
@@ -1,191 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"ams_hbase_master",
- "rowtype":"service",
- "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log"
- },
- {
- "type":"ams_hbase_regionserver",
- "rowtype":"service",
- "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log"
- },
- {
- "type":"ams_collector",
- "rowtype":"service",
- "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
- },
- {
- "type":"ams_monitor",
- "rowtype":"service",
- "path":"{{metrics_monitor_log_dir}}/ambari-metrics-monitor.out"
- },
- {
- "type":"ams_grafana",
- "rowtype":"service",
- "path":"{{metrics_grafana_log_dir}}/grafana.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ams_collector"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %p %c: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ams_hbase_master",
- "ams_hbase_regionserver"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ams_grafana"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "multiline_pattern":"^(%{DATESTAMP:logtime})",
- "message_pattern":"(?m)^%{DATESTAMP:logtime}%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy/MM/dd HH:mm:ss"
- }
-
- },
- "level":[
- {
- "map_fieldvalue":{
- "pre_value":"I",
- "post_value":"INFO"
- }
- },
- {
- "map_fieldvalue":{
- "pre_value":"W",
- "post_value":"WARN"
- }
- },
- {
- "map_fieldvalue":{
- "pre_value":"D",
- "post_value":"DEBUG"
- }
- },
- {
- "map_fieldvalue":{
- "pre_value":"E",
- "post_value":"ERROR"
- }
- },
- {
- "map_fieldvalue":{
- "pre_value":"F",
- "post_value":"FATAL"
- }
- }
- ]
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ams_monitor"
- ]
-
- }
-
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- },
- "level":[
- {
- "map_fieldvalue":{
- "pre_value":"WARNING",
- "post_value":"WARN"
- }
- }
- ]
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2
deleted file mode 100644
index 53912c8..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2
+++ /dev/null
@@ -1,55 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"atlas_app",
- "rowtype":"service",
- "path":"{{atlas_log_dir}}/application.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "atlas_app"
- ]
-
- }
-
- },
- "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2
deleted file mode 100644
index a40e52d..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2
+++ /dev/null
@@ -1,55 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"falcon_app",
- "rowtype":"service",
- "path":"{{falcon_log_dir}}/falcon.application.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "falcon_app"
- ]
-
- }
-
- },
- "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
deleted file mode 100644
index b31054b..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
+++ /dev/null
@@ -1,56 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"flume_handler",
- "rowtype":"service",
- "path":"{{flume_log_dir}}/flume.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "flume_handler"
- ]
-
- }
-
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
- "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
deleted file mode 100644
index 0c04052..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
+++ /dev/null
@@ -1,91 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"hbase_master",
- "rowtype":"service",
- "path":"{{hbase_log_dir}}/hbase-*-master-*.log"
- },
- {
- "type":"hbase_regionserver",
- "rowtype":"service",
- "path":"{{hbase_log_dir}}/hbase-*-regionserver-*.log"
- },
- {
- "type":"hbase_phoenix_server",
- "rowtype":"service",
- "path":"{{hbase_log_dir}}/phoenix-*-server.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hbase_master",
- "hbase_regionserver"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hbase_phoenix_server"
- ]
-
- }
- },
- "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
deleted file mode 100644
index 3dff6f5..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
+++ /dev/null
@@ -1,251 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"hdfs_datanode",
- "rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-datanode-*.log"
- },
- {
- "type":"hdfs_namenode",
- "rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-namenode-*.log"
- },
- {
- "type":"hdfs_journalnode",
- "rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-journalnode-*.log"
- },
- {
- "type":"hdfs_secondarynamenode",
- "rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-secondarynamenode-*.log"
- },
- {
- "type":"hdfs_zkfc",
- "rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-zkfc-*.log"
- },
- {
- "type":"hdfs_nfs3",
- "rowtype":"service",
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-nfs3-*.log"
- },
- {
- "type":"hdfs_audit",
- "rowtype":"audit",
- "is_enabled":"true",
- "add_fields":{
- "logType":"HDFSAudit",
- "enforcer":"hadoop-acl",
- "repoType":"1",
- "repo":"hdfs"
- },
- "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hdfs-audit.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hdfs_datanode",
- "hdfs_journalnode",
- "hdfs_secondarynamenode",
- "hdfs_namenode",
- "hdfs_zkfc",
- "hdfs_nfs3"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hdfs_audit"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "evtTime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"keyvalue",
- "sort_order":1,
- "conditions":{
- "fields":{
- "type":[
- "hdfs_audit"
- ]
-
- }
-
- },
- "source_field":"log_message",
- "value_split":"=",
- "field_split":"\t",
- "post_map_values":{
- "src":{
- "map_fieldname":{
- "new_fieldname":"resource"
- }
-
- },
- "ip":{
- "map_fieldname":{
- "new_fieldname":"cliIP"
- }
-
- },
- "allowed":[
- {
- "map_fieldvalue":{
- "pre_value":"true",
- "post_value":"1"
- }
-
- },
- {
- "map_fieldvalue":{
- "pre_value":"false",
- "post_value":"0"
- }
-
- },
- {
- "map_fieldname":{
- "new_fieldname":"result"
- }
-
- }
-
- ],
- "cmd":{
- "map_fieldname":{
- "new_fieldname":"action"
- }
-
- },
- "proto":{
- "map_fieldname":{
- "new_fieldname":"cliType"
- }
-
- },
- "callerContext":{
- "map_fieldname":{
- "new_fieldname":"req_caller_id"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "sort_order":2,
- "source_field":"ugi",
- "remove_source_field":"false",
- "conditions":{
- "fields":{
- "type":[
- "hdfs_audit"
- ]
-
- }
-
- },
- "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
- "post_map_values":{
- "user":{
- "map_fieldname":{
- "new_fieldname":"reqUser"
- }
-
- },
- "x_user":{
- "map_fieldname":{
- "new_fieldname":"reqUser"
- }
-
- },
- "p_user":{
- "map_fieldname":{
- "new_fieldname":"reqUser"
- }
-
- },
- "k_user":{
- "map_fieldname":{
- "new_fieldname":"proxyUsers"
- }
-
- },
- "p_authType":{
- "map_fieldname":{
- "new_fieldname":"authType"
- }
-
- },
- "k_authType":{
- "map_fieldname":{
- "new_fieldname":"proxyAuthType"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
deleted file mode 100644
index f96b16f..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
+++ /dev/null
@@ -1,97 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"hive_hiveserver2",
- "rowtype":"service",
- "path":"{{hive_log_dir}}/hiveserver2.log"
- },
- {
- "type":"hive_metastore",
- "rowtype":"service",
- "path":"{{hive_log_dir}}/hivemetastore.log"
- },
- {
- "type": "webhcat_server",
- "rowntype":"service",
- "path":"{{hcat_log_dir}}/webhcat.log"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "hive_hiveserver2",
- "hive_metastore"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "webhcat_server"
- ]
-
- }
-
- },
- "log4j_format":" %-5p | %d{DATE} | %c | %m%n",
- "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})",
- "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
- }
-
- },
- "level":{
- "map_fieldvalue":{
- "pre_value":"WARNING",
- "post_value":"WARN"
- }
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2
deleted file mode 100644
index 20fa49d..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2
+++ /dev/null
@@ -1,56 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"infra_solr",
- "rowtype":"service",
- "path":"{{infra_solr_log_dir}}/solr.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "infra_solr"
- ]
-
- }
-
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2
deleted file mode 100644
index d05d4ad..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2
+++ /dev/null
@@ -1,105 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"kafka_controller",
- "rowtype":"service",
- "path":"{{kafka_log_dir}}/controller.log"
- },
- {
- "type":"kafka_request",
- "rowtype":"service",
- "path":"{{kafka_log_dir}}/kafka-request.log"
- },
- {
- "type":"kafka_logcleaner",
- "rowtype":"service",
- "path":"{{kafka_log_dir}}/log-cleaner.log"
- },
- {
- "type":"kafka_server",
- "rowtype":"service",
- "path":"{{kafka_log_dir}}/server.log"
- },
- {
- "type":"kafka_statechange",
- "rowtype":"service",
- "path":"{{kafka_log_dir}}/state-change.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "kafka_controller",
- "kafka_request",
- "kafka_logcleaner"
- ]
-
- }
-
- },
- "log4j_format":"[%d] %p %m (%c)%n",
- "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
- "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
- "conditions":{
- "fields":{
- "type":[
- "kafka_server",
- "kafka_statechange"
- ]
-
- }
-
- },
- "log4j_format":"[%d] %p %m (%c)%n",
- "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
- "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2
deleted file mode 100644
index df98e25..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2
+++ /dev/null
@@ -1,68 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"knox_gateway",
- "rowtype":"service",
- "path":"{{knox_log_dir}}/gateway.log"
- },
- {
- "type":"knox_cli",
- "rowtype":"service",
- "path":"{{knox_log_dir}}/knoxcli.log"
- },
- {
- "type":"knox_ldap",
- "rowtype":"service",
- "path":"{{knox_log_dir}}/ldap.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "knox_gateway",
- "knox_cli",
- "knox_ldap"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2
deleted file mode 100644
index ddb8198..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2
+++ /dev/null
@@ -1,56 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"oozie_app",
- "rowtype":"service",
- "path":"{{oozie_log_dir}}/oozie.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "oozie_app"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2
deleted file mode 100644
index 7b12869..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2
+++ /dev/null
@@ -1,122 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"ranger_admin",
- "rowtype":"service",
- "path":"{{ranger_admin_log_dir}}/xa_portal.log"
- },
- {
- "type":"ranger_dbpatch",
- "is_enabled":"true",
- "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log"
- },
- {
- "type":"ranger_kms",
- "rowtype":"service",
- "path":"{{ranger_kms_log_dir}}/kms.log"
- },
- {
- "type":"ranger_usersync",
- "rowtype":"service",
- "path":"{{ranger_usersync_log_dir}}/usersync.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ranger_admin",
- "ranger_dbpatch"
- ]
-
- }
-
- },
- "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ranger_kms"
- ]
-
- }
-
- },
- "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
- }
-
- }
-
- }
-
- },
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "ranger_usersync"
- ]
-
- }
-
- },
- "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n",
- "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
- "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"dd MMM yyyy HH:mm:ss"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2
deleted file mode 100644
index 4371276..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2
+++ /dev/null
@@ -1,72 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
- {
- "input":[
-
- {
- "type":"spark_jobhistory_server",
- "rowtype":"service",
- "path":"{{spark_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out"
- },
- {
- "type":"spark_thriftserver",
- "rowtype":"service",
- "path":"{{spark_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out"
- },
- {
- "type":"livy_server",
- "rowtype":"service",
- "path":"{{livy_log_dir}}/livy-livy-server.out"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "spark_jobhistory_server",
- "spark_thriftserver",
- "livy_server"
- ]
-
- }
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})",
- "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yy/MM/dd HH:mm:ss"
- }
-
- },
- "level":{
- "map_fieldvalue":{
- "pre_value":"WARNING",
- "post_value":"WARN"
- }
-
- }
-
- }
- }
- ]
-
- }
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2
deleted file mode 100644
index 01aea36..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2
+++ /dev/null
@@ -1,66 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
- {
- "input":[
-
- {
- "type":"spark2_jobhistory_server",
- "rowtype":"service",
- "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out"
- },
- {
- "type":"spark2_thriftserver",
- "rowtype":"service",
- "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out"
- }
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "spark2_jobhistory_server",
- "spark2_thriftserver"
- ]
-
- }
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})",
- "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yy/MM/dd HH:mm:ss"
- }
-
- },
- "level":{
- "map_fieldvalue":{
- "pre_value":"WARNING",
- "post_value":"WARN"
- }
-
- }
-
- }
- }
- ]
-
- }
http://git-wip-us.apache.org/repos/asf/ambari/blob/db9bb094/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2
deleted file mode 100644
index 07a4c2e..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2
+++ /dev/null
@@ -1,86 +0,0 @@
-{#
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #}
-{
- "input":[
- {
- "type":"storm_drpc",
- "rowtype":"service",
- "path":"{{storm_log_dir}}/drpc.log"
- },
- {
- "type":"storm_logviewer",
- "rowtype":"service",
- "path":"{{storm_log_dir}}/logviewer.log"
- },
- {
- "type":"storm_nimbus",
- "rowtype":"service",
- "path":"{{storm_log_dir}}/nimbus.log"
- },
- {
- "type":"storm_supervisor",
- "rowtype":"service",
- "path":"{{storm_log_dir}}/supervisor.log"
- },
- {
- "type":"storm_ui",
- "rowtype":"service",
- "path":"{{storm_log_dir}}/ui.log"
- },
- {
- "type":"storm_worker",
- "rowtype":"service",
- "path":"{{storm_log_dir}}/*worker*.log"
- }
-
- ],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "storm_drpc",
- "storm_logviewer",
- "storm_nimbus",
- "storm_supervisor",
- "storm_ui",
- "storm_worker"
- ]
-
- }
-
- },
- "log4j_format":"",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
- }
-
- }
-
- }
-
- }
-
- ]
-
-}
\ No newline at end of file