You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/04/09 12:15:51 UTC
[55/61] [abbrv] ambari git commit: AMBARI-15679. Stack definition for
Logsearch (oleewre)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml
new file mode 100644
index 0000000..de67420
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml
@@ -0,0 +1,193 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+
+ <property>
+ <name>logsearch_solr_port</name>
+ <value>8886</value>
+ <description>Solr port</description>
+ <display-name>Logsearch Solr port</display-name>
+ </property>
+
+ <property>
+ <name>logsearch_solr_pid_dir</name>
+ <value>/var/run/ambari-logsearch-solr</value>
+ <description>Solr Process ID Directory</description>
+ <display-name>Logsearch Solr pid dir</display-name>
+ </property>
+
+ <property>
+ <name>logsearch_solr_log_dir</name>
+ <value>/var/log/ambari-logsearch-solr</value>
+ <description>Directory for Solr logs</description>
+ <display-name>Logsearch Solr log dir</display-name>
+ </property>
+
+ <property>
+ <name>logsearch_solr_user</name>
+ <value>solr</value>
+ <property-type>USER</property-type>
+ <description>Solr user</description>
+ <display-name>Logsearch Solr User</display-name>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>logsearch_solr_group</name>
+ <value>solr</value>
+ <property-type>GROUP</property-type>
+ <description>Solr group</description>
+ <display-name>Logsearch Solr Group</display-name>
+ </property>
+
+ <property>
+ <name>logsearch_solr_datadir</name>
+ <value>/opt/logsearch_solr/data</value>
+ <display-name>Logsearch Solr data dir</display-name>
+ <description>Directory for storting Solr index. Make sure you have enough disk space</description>
+ </property>
+
+ <property>
+ <name>logsearch_solr_znode</name>
+ <value>/logsearch</value>
+ <description>Zookeeper znode</description>
+ <display-name>Logsearch Solr ZNode</display-name>
+ </property>
+
+ <property>
+ <name>logsearch_solr_minmem</name>
+ <value>512m</value>
+ <display-name>Logsearch minimum heap size</display-name>
+ <description>Solr minimum heap size e.g.512m </description>
+ </property>
+
+ <property>
+ <name>logsearch_solr_maxmem</name>
+ <value>512m</value>
+ <description>Solr maximum heap size e.g. 512m</description>
+ <display-name>Logsearch maximum heap size</display-name>
+ </property>
+
+
+ <!-- logsearch-solr-env.sh -->
+
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for logsearch-solr-env.sh file</description>
+ <value>
+# By default the script will use JAVA_HOME to determine which java
+# to use, but you can set a specific path for Solr to use without
+# affecting other Java applications on your server/workstation.
+SOLR_JAVA_HOME={{java64_home}}
+
+# Increase Java Min/Max Heap as needed to support your indexing / query needs
+SOLR_JAVA_MEM="-Xms{{logsearch_solr_min_mem}} -Xmx{{logsearch_solr_max_mem}}"
+
+# Enable verbose GC logging
+GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \
+-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"
+
+# These GC settings have shown to work well for a number of common Solr workloads
+GC_TUNE="-XX:NewRatio=3 \
+-XX:SurvivorRatio=4 \
+-XX:TargetSurvivorRatio=90 \
+-XX:MaxTenuringThreshold=8 \
+-XX:+UseConcMarkSweepGC \
+-XX:+UseParNewGC \
+-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \
+-XX:+CMSScavengeBeforeRemark \
+-XX:PretenureSizeThreshold=64m \
+-XX:+UseCMSInitiatingOccupancyOnly \
+-XX:CMSInitiatingOccupancyFraction=50 \
+-XX:CMSMaxAbortablePrecleanTime=6000 \
+-XX:+CMSParallelRemarkEnabled \
+-XX:+ParallelRefProcEnabled"
+
+# Set the ZooKeeper connection string if using an external ZooKeeper ensemble
+# e.g. host1:2181,host2:2181/chroot
+# Leave empty if not using SolrCloud
+ZK_HOST="{{zookeeper_quorum}}{{logsearch_solr_znode}}"
+
+# Set the ZooKeeper client timeout (for SolrCloud mode)
+ZK_CLIENT_TIMEOUT="60000"
+
+# By default the start script uses "localhost"; override the hostname here
+# for production SolrCloud environments to control the hostname exposed to cluster state
+#SOLR_HOST="192.168.1.1"
+
+# By default the start script uses UTC; override the timezone if needed
+#SOLR_TIMEZONE="UTC"
+
+# Set to true to activate the JMX RMI connector to allow remote JMX client applications
+# to monitor the JVM hosting Solr; set to "false" to disable that behavior
+# (false is recommended in production environments)
+ENABLE_REMOTE_JMX_OPTS="false"
+
+# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
+# RMI_PORT=18983
+
+# Anything you add to the SOLR_OPTS variable will be included in the java
+# start command line as-is, in ADDITION to other options. If you specify the
+# -a option on start script, those options will be appended as well. Examples:
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000"
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000"
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true"
+
+# Location where the bin/solr script will save PID files for running instances
+# If not set, the script will create PID files in $SOLR_TIP/bin
+SOLR_PID_DIR={{logsearch_solr_piddir}}
+
+# Path to a directory where Solr creates index files, the specified directory
+# must contain a solr.xml; by default, Solr will use server/solr
+SOLR_HOME={{logsearch_solr_datadir}}
+
+# Solr provides a default Log4J configuration properties file in server/resources
+# however, you may want to customize the log settings and file appender location
+# so you can point the script to use a different log4j.properties file
+LOG4J_PROPS={{logsearch_solr_conf}}/log4j.properties
+
+# Location where Solr should write logs to; should agree with the file appender
+# settings in server/resources/log4j.properties
+SOLR_LOGS_DIR={{logsearch_solr_log_dir}}
+
+# Sets the port Solr binds to, default is 8983
+SOLR_PORT={{logsearch_solr_port}}
+
+# Uncomment to set SSL-related system properties
+# Be sure to update the paths to the correct keystore for your environment
+#SOLR_SSL_OPTS="-Djavax.net.ssl.keyStore=etc/solr-ssl.keystore.jks \
+#-Djavax.net.ssl.keyStorePassword=secret \
+#-Djavax.net.ssl.trustStore=etc/solr-ssl.keystore.jks \
+#-Djavax.net.ssl.trustStorePassword=secret"
+
+# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set
+# and you are using SSL, then the start script will use SOLR_PORT for the SSL port
+#SOLR_SSL_PORT=
+ </value>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml
new file mode 100644
index 0000000..1f19c8d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for log4j.properties</description>
+ <value>
+# Logging level
+solr.log={{logsearch_solr_log_dir}}
+#log4j.rootLogger=INFO, file, CONSOLE
+log4j.rootLogger=INFO, file
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] \u2013 %m%n
+
+#- size rotation with log cleanup.
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.MaxFileSize=10MB
+log4j.appender.file.MaxBackupIndex=9
+
+#- File to log to and log format
+log4j.appender.file.File=${solr.log}/solr.log
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.hadoop=WARN
+
+# set to INFO to enable infostream log messages
+log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF
+ </value>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml
new file mode 100644
index 0000000..01b32b5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for logsearch solr.xml file</description>
+ <value>
+<solr>
+ <solrcloud>
+ <str name="host">${host:}</str>
+ <int name="hostPort">${jetty.port:}</int>
+ <str name="hostContext">${hostContext:solr}</str>
+ <int name="zkClientTimeout">${zkClientTimeout:15000}</int>
+ <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+ </solrcloud>
+</solr>
+ </value>
+ </property>
+
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json
new file mode 100644
index 0000000..6dd4aa7
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json
@@ -0,0 +1,17 @@
+{
+ "services": [
+ {
+ "name": "LOGSEARCH",
+ "identities": [
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "components": [
+ {
+ "name": "LOGSEARCH_SERVER"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
new file mode 100644
index 0000000..9bf486b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>LOGSEARCH</name>
+ <displayName>Log Search</displayName>
+ <comment>Log aggregation, analysis, and visualization for Ambari managed services</comment>
+ <version>0.5.0</version>
+
+ <components>
+
+ <component>
+ <name>LOGSEARCH_SERVER</name>
+ <timelineAppid>logsearch</timelineAppid>
+ <displayName>Log Search Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/logsearch.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>logsearch_app</logId>
+ </log>
+ <log>
+ <logId>logsearch_perf</logId>
+ </log>
+ </logs>
+ </component>
+
+ <component>
+ <name>LOGSEARCH_SOLR</name>
+ <displayName>Solr Instance</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>LOGSEARCH_LOGFEEDER</name>
+ <timelineAppid>logfeeder</timelineAppid>
+ <displayName>Log Feeder</displayName>
+ <category>SLAVE</category>
+ <cardinality>ALL</cardinality>
+ <commandScript>
+ <script>scripts/logfeeder.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>logsearch_feeder</logId>
+ </log>
+ </logs>
+ </component>
+
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>ambari-logsearch-logfeeder</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>ambari-logsearch-portal</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>ambari-logsearch-solr</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_logsearch_solr</condition>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <requiredServices>
+ <service>ZOOKEEPER</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>logsearch-site</config-type>
+ <config-type>solr-site</config-type>
+ <config-type>logfeeder-site</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>false</restartRequiredAfterChange>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
new file mode 100644
index 0000000..ce0155d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from setup_logfeeder import setup_logfeeder
+
+class LogFeeder(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ setup_logfeeder()
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ Execute(format("{logfeeder_dir}/run.sh"),
+ environment = {'LOGFEEDER_INCLUDE': format('{logsearch_logfeeder_conf}/logfeeder-env.sh')},
+ user=params.logfeeder_user
+ )
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ Execute (format("kill `cat {logfeeder_pid_file}`"),
+ user=params.logfeeder_user ,
+ only_if = format("test -f {logfeeder_pid_file}")
+ )
+ File(params.logfeeder_pid_file,
+ action="delete"
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.logfeeder_pid_file)
+
+
+if __name__ == "__main__":
+ LogFeeder().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
new file mode 100644
index 0000000..ac37788
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
@@ -0,0 +1,65 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from setup_logsearch import setup_logsearch
+
+class LogSearch(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ setup_logsearch()
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ Execute(format("{logsearch_dir}/run.sh {logsearch_ui_port}"),
+ environment = {'LOGSEARCH_INCLUDE': format('{logsearch_server_conf}/logsearch-env.sh')},
+ user=params.logsearch_user
+ )
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ Execute (format('kill `cat {logsearch_pid_file}` >/dev/null 2>&1'),
+ user=params.logsearch_user,
+ only_if=format("test -f {logsearch_pid_file}") # TODO: check if process is up
+ )
+ File(params.logsearch_pid_file,
+ action="delete"
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.logsearch_pid_file)
+
+if __name__ == "__main__":
+ LogSearch().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
new file mode 100644
index 0000000..ee39629
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management import *
+import os
+from resource_management.libraries.functions.default import default
+import status_params
+
+def get_port_from_url(address):
+ if not is_empty(address):
+ return address.split(':')[-1]
+ else:
+ return address
+
+# config object that holds the configurations declared in the -site.xml file
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_version = default("/commandParams/version", None)
+sudo = AMBARI_SUDO_BINARY
+
+logsearch_solr_conf = "/etc/ambari-logsearch-solr/conf"
+logsearch_server_conf = "/etc/ambari-logsearch-portal/conf"
+logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf"
+
+logsearch_solr_port = status_params.logsearch_solr_port
+logsearch_solr_piddir = status_params.logsearch_solr_piddir
+logsearch_solr_pidfile = status_params.logsearch_solr_pidfile
+
+#logsearch pid file
+logsearch_pid_dir = status_params.logsearch_pid_dir
+logsearch_pid_file = status_params.logsearch_pid_file
+
+#logfeeder pid file
+logfeeder_pid_dir = status_params.logfeeder_pid_dir
+logfeeder_pid_file = status_params.logfeeder_pid_file
+
+#shared configs
+java64_home = config['hostLevelParams']['java_home']
+#get comma separated list of zookeeper hosts from clusterHostInfo
+zookeeper_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts'])
+cluster_name = str(config['clusterName'])
+
+#for now just pick first collector
+if 'metrics_collector_hosts' in config['clusterHostInfo']:
+ metrics_collector_hosts_list = ",".join(config['clusterHostInfo']['metrics_collector_hosts'])
+ metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address']))
+ metrics_collector_hosts = format('http://{metrics_collector_hosts_list}:{metrics_collector_port}/ws/v1/timeline/metrics')
+else:
+ metrics_collector_hosts = ''
+
+#####################################
+#Solr configs
+#####################################
+
+# Only supporting SolrCloud mode - so hardcode those options
+solr_cloudmode = 'true'
+solr_dir = '/usr/lib/ambari-logsearch-solr'
+solr_bindir = solr_dir + '/bin'
+cloud_scripts = solr_dir +'/server/scripts/cloud-scripts'
+
+logsearch_solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode']
+logsearch_solr_min_mem = format(config['configurations']['logsearch-solr-env']['logsearch_solr_minmem'])
+logsearch_solr_max_mem = format(config['configurations']['logsearch-solr-env']['logsearch_solr_maxmem'])
+logsearch_solr_instance_count = len(config['clusterHostInfo']['logsearch_solr_hosts'])
+logsearch_solr_datadir = format(config['configurations']['logsearch-solr-env']['logsearch_solr_datadir'])
+logsearch_solr_data_resources_dir = os.path.join(logsearch_solr_datadir,'resources')
+logsearch_service_logs_max_retention = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_max_retention']
+logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention']
+
+logsearch_service_logs_fields = config['configurations']['logsearch-site']['logsearch.service.logs.fields']
+
+logsearch_ui_port = config['configurations']['logsearch-site']["logsearch.ui.port"]
+logsearch_debug_enabled = str(config['configurations']['logsearch-env']["logsearch_debug_enabled"]).lower()
+logsearch_debug_port = config['configurations']['logsearch-env']["logsearch_debug_port"]
+logsearch_app_max_memory = config['configurations']['logsearch-env']['logsearch_app_max_memory']
+
+audit_logs_collection_splits_interval_mins = config['configurations']['logsearch-site']['logsearch.audit.logs.split.interval.mins']
+service_logs_collection_splits_interval_mins = config['configurations']['logsearch-site']['logsearch.service.logs.split.interval.mins']
+
+zookeeper_port=default('/configurations/zoo.cfg/clientPort', None)
+#get comma separated list of zookeeper hosts from clusterHostInfo
+index = 0
+zookeeper_quorum = ""
+for host in config['clusterHostInfo']['zookeeper_hosts']:
+ zookeeper_quorum += host + ":"+str(zookeeper_port)
+ index += 1
+ if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+ zookeeper_quorum += ","
+
+logsearch_solr_user = config['configurations']['logsearch-solr-env']['logsearch_solr_user']
+logsearch_solr_group = config['configurations']['logsearch-solr-env']['logsearch_solr_group']
+logsearch_solr_log_dir = config['configurations']['logsearch-solr-env']['logsearch_solr_log_dir']
+logsearch_solr_log = format("{logsearch_solr_log_dir}/solr-install.log")
+
+solr_env_content = config['configurations']['logsearch-solr-env']['content']
+
+solr_xml_content = config['configurations']['logsearch-solr-xml']['content']
+
+solr_log4j_content = config['configurations']['logsearch-solr-log4j']['content']
+
+
+#####################################
+#Logsearch configs
+#####################################
+logsearch_dir = '/usr/lib/ambari-logsearch-portal'
+logsearch_numshards_config = config['configurations']['logsearch-site']['logsearch.collection.numshards']
+
+if logsearch_numshards_config > 0:
+ logsearch_numshards = str(logsearch_numshards_config)
+else:
+ logsearch_numshards = format(str(logsearch_solr_instance_count))
+
+logsearch_repfactor = str(config['configurations']['logsearch-site']['logsearch.collection.replication.factor'])
+
+logsearch_solr_collection_service_logs = default('/configurations/logsearch-site/logsearch.solr.collection.service.logs', 'hadoop_logs')
+logsearch_solr_collection_audit_logs = default('/configurations/logsearch-site/logsearch.solr.collection.audit.logs', 'audit_logs')
+
+solr_audit_logs_use_ranger = default('/configurations/logsearch-env/logsearch_solr_audit_logs_use_ranger', False)
+solr_audit_logs_url = ''
+
+if solr_audit_logs_use_ranger:
+ #In Ranger, this contain the /zkNode also
+ ranger_audit_solr_zookeepers = default('/configurations/ranger-admin-site/ranger.audit.solr.zookeepers', None)
+ #TODO: ranger property already has zk node appended. We need to remove it.
+ #For now, let's assume it is going to be URL
+ solr_audit_logs_url = default('/configurations/ranger-admin-site/ranger.audit.solr.urls', solr_audit_logs_url)
+else:
+ solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', None)
+ solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', None)
+
+ if not(solr_audit_logs_zk_quorum):
+ solr_audit_logs_zk_quorum=zookeeper_quorum
+ if not(solr_audit_logs_zk_node):
+ solr_audit_logs_zk_node=logsearch_solr_znode
+
+ solr_audit_logs_zk_node = format(solr_audit_logs_zk_node)
+ solr_audit_logs_zk_quorum = format(solr_audit_logs_zk_quorum)
+
+
+# logsearch-env configs
+logsearch_user = config['configurations']['logsearch-env']['logsearch_user']
+logsearch_group = config['configurations']['logsearch-env']['logsearch_group']
+logsearch_log_dir = config['configurations']['logsearch-env']['logsearch_log_dir']
+logsearch_log = logsearch_log_dir + '/logsearch.out'
+
+# store the log file for the service from the 'solr.log' property of the 'logsearch-env.xml' file
+logsearch_env_content = config['configurations']['logsearch-env']['content']
+logsearch_service_logs_solrconfig_content = config['configurations']['logsearch-service_logs-solrconfig']['content']
+logsearch_audit_logs_solrconfig_content = config['configurations']['logsearch-audit_logs-solrconfig']['content']
+logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['content']
+
+#Log dirs
+ambari_server_log_dir = '/var/log/ambari-server'
+ambari_agent_log_dir = '/var/log/ambari-agent'
+knox_log_dir = '/var/log/knox'
+
+metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log')
+metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log')
+
+
+atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')
+accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')
+falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir','/var/log/falcon')
+hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir','/var/log/hbase')
+hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix','/var/log/hadoop')
+hive_log_dir = default('/configurations/hive-env/hive_log_dir','/var/log/hive')
+kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir','/var/log/kafka')
+oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir','/var/log/oozie')
+ranger_usersync_log_dir = default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')
+ranger_admin_log_dir = default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')
+ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')
+storm_log_dir = default('/configurations/storm-env/storm_log_dir','/var/log/storm')
+yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix','/var/log/hadoop')
+mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix','/var/log/hadoop')
+zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir','/var/log/zookeeper')
+
+
+#####################################
+#Logfeeder configs
+#####################################
+
+logfeeder_dir = "/usr/lib/ambari-logsearch-logfeeder"
+
+# logfeeder-site configs
+solr_service_logs_enable = default('/configurations/logfeeder-site/logfeeder.solr.service.logs.enable',True)
+solr_audit_logs_enable = default('/configurations/logfeeder-site/logfeeder.solr.audit.logs_enable',True)
+
+kafka_broker_list = default('/configurations/logfeeder-site/logfeeder.kafka.broker.list', '')
+kafka_security_protocol= default('/configurations/logfeeder-site/logfeeder.kafka.security.protocol', '')
+kafka_kerberos_service_name=default('/configurations/logfeeder-site/logfeeder.kafka.kerberos.service.name', '')
+
+kafka_service_logs_enable = default('/configurations/logfeeder-site/logfeeder.kafka.service.logs.enable',False)
+kafka_audit_logs_enable = default('/configurations/logfeeder-site/logfeeder.kafka.audit.logs.enable',False)
+kafka_topic_service_logs = default('/configurations/logfeeder-site/logfeeder.kafka.topic.service.logs','service_logs')
+kafka_topic_audit_logs = default('/configurations/logfeeder-site/logfeeder.kafka.topic.audit.logs','audit_logs')
+
+# logfeeder-env configs
+logfeeder_user = config['configurations']['logfeeder-env']['logfeeder_user']
+logfeeder_group = config['configurations']['logfeeder-env']['logfeeder_group']
+logfeeder_log_dir = config['configurations']['logfeeder-env']['logfeeder_log_dir']
+logfeeder_log = logfeeder_log_dir + '/logfeeder.out'
+logfeeder_max_mem = config['configurations']['logfeeder-env']['logfeeder_max_mem']
+logfeeder_env_content = config['configurations']['logfeeder-env']['content']
+logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content']
+
+logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.checkpoint.folder', '/etc/ambari-logsearch-logfeeder/conf/checkpoints')
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
+ 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+
+logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
+
+logfeeder_config_files = ','.join(logfeeder_config_file_names)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
new file mode 100644
index 0000000..0c1cb6d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def setup_logfeeder():
+ import params
+
+ Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_dir,
+ params.logsearch_logfeeder_conf, params.logfeeder_checkpoint_folder],
+ mode = 0755,
+ cd_access = 'a',
+ owner = params.logfeeder_user,
+ group = params.logfeeder_group,
+ create_parents = True
+ )
+
+
+ File(params.logfeeder_log,
+ mode = 0644,
+ owner = params.logfeeder_user,
+ group = params.logfeeder_group,
+ content = ''
+ )
+
+ File(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
+ content=Template("logfeeder.properties.j2"),
+ owner=params.logfeeder_user
+ )
+
+ File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"),
+ content = InlineTemplate(params.logfeeder_env_content),
+ owner = params.logfeeder_user
+ )
+
+ File(format("{logsearch_logfeeder_conf}/log4j.xml"),
+ content = InlineTemplate(params.logfeeder_log4j_content),
+ owner = params.logfeeder_user
+ )
+
+ File(format("{logsearch_logfeeder_conf}/grok-patterns"),
+ content = Template("grok-patterns.j2"),
+ owner = params.logfeeder_user,
+ encoding = "utf-8"
+ )
+
+ for file_name in params.logfeeder_config_file_names:
+ File(format("{logsearch_logfeeder_conf}/" + file_name),
+ content = Template(file_name + ".j2"),
+ owner = params.logfeeder_user
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
new file mode 100644
index 0000000..7cab1d9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -0,0 +1,93 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import random
+from resource_management import *
+from resource_management.libraries.functions.decorator import retry
+
+def setup_logsearch():
+ import params
+
+ Directory([params.logsearch_log_dir, params.logsearch_pid_dir, params.logsearch_dir, params.logsearch_server_conf],
+ mode=0755,
+ cd_access='a',
+ owner=params.logsearch_user,
+ group=params.logsearch_group,
+ create_parents=True
+ )
+
+ File(params.logsearch_log,
+ mode=0644,
+ owner=params.logsearch_user,
+ group=params.logsearch_group,
+ content=''
+ )
+
+ File(format("{logsearch_server_conf}/logsearch.properties"),
+ content=Template("logsearch.properties.j2"),
+ owner=params.logsearch_user
+ )
+
+ File(format("{logsearch_server_conf}/log4j.xml"),
+ content=InlineTemplate(params.logsearch_app_log4j_content),
+ owner=params.logsearch_user
+ )
+
+ File(format("{logsearch_server_conf}/logsearch-env.sh"),
+ content=InlineTemplate(params.logsearch_env_content),
+ mode=0755,
+ owner=params.logsearch_user
+ )
+
+ File(format("{logsearch_server_conf}/solr_configsets/hadoop_logs/conf/solrconfig.xml"),
+ content=InlineTemplate(params.logsearch_service_logs_solrconfig_content),
+ owner=params.logsearch_user
+ )
+
+ File(format("{logsearch_server_conf}/solr_configsets/audit_logs/conf/solrconfig.xml"),
+ content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content),
+ owner=params.logsearch_user
+ )
+
+ random_num = random.random()
+
+ upload_configuration_dir_to_zk('hadoop_logs', random_num)
+
+ upload_configuration_dir_to_zk('history', random_num)
+
+ upload_configuration_dir_to_zk('audit_logs', random_num)
+
+ Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")),
+ sudo=True
+ )
+
+def upload_configuration_dir_to_zk(config_set, random_num):
+ import params
+
+ tmp_config_set_folder=format('{tmp_dir}/solr_config_{config_set}_{random_num}')
+ zk_cli_prefix = format('export JAVA_HOME={java64_home} ; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_quorum}{logsearch_solr_znode}')
+
+ Execute(format('{zk_cli_prefix} -cmd downconfig -confdir {tmp_config_set_folder} -confname {config_set}'),
+ only_if = format("{zk_cli_prefix} -cmd get /configs/{config_set}"),
+ )
+
+ Execute(format('{zk_cli_prefix} -cmd upconfig -confdir {logsearch_server_conf}/solr_configsets/{config_set}/conf -confname {config_set}'),
+ not_if = format("test -d {tmp_config_set_folder}"),
+ )
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py
new file mode 100644
index 0000000..0192277
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def setup_solr():
+ import params
+
+ Directory([params.solr_dir, params.logsearch_solr_log_dir, params.logsearch_solr_piddir, params.logsearch_solr_conf,
+ params.logsearch_solr_datadir, params.logsearch_solr_data_resources_dir],
+ mode=0755,
+ cd_access='a',
+ owner=params.logsearch_solr_user,
+ group=params.logsearch_solr_group,
+ create_parents=True
+ )
+
+ File(params.logsearch_solr_log,
+ mode=0644,
+ owner=params.logsearch_solr_user,
+ group=params.logsearch_solr_group,
+ content=''
+ )
+
+ File(format("{logsearch_solr_conf}/logsearch-solr-env.sh"),
+ content=InlineTemplate(params.solr_env_content),
+ mode = 0755,
+ owner=params.logsearch_solr_user
+ )
+
+ File(format("{logsearch_solr_datadir}/solr.xml"),
+ content=InlineTemplate(params.solr_xml_content),
+ owner=params.logsearch_solr_user
+ )
+
+ File(format("{logsearch_solr_conf}/log4j.properties"),
+ content=InlineTemplate(params.solr_log4j_content),
+ owner=params.logsearch_solr_user
+ )
+
+ File(format("{logsearch_solr_datadir}/zoo.cfg"),
+ content=Template("zoo.cfg.j2"),
+ owner=params.logsearch_solr_user
+ )
+
+ zk_cli_prefix = format('export JAVA_HOME={java64_home}; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_hosts}')
+ Execute (format('{zk_cli_prefix} -cmd makepath {logsearch_solr_znode}'),
+ not_if = format("{zk_cli_prefix} -cmd get /logsearch"),
+ ignore_failures = True,
+ user=params.logsearch_solr_user
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
new file mode 100644
index 0000000..545505a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py
@@ -0,0 +1,66 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from setup_solr import setup_solr
+
+class Solr(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ setup_solr()
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ Execute(format('{solr_bindir}/solr start -cloud -noprompt -s {logsearch_solr_datadir} >> {logsearch_solr_log} 2>&1'),
+ environment = {'SOLR_INCLUDE': format('{logsearch_solr_conf}/logsearch-solr-env.sh')},
+ user=params.logsearch_solr_user
+ )
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ Execute (format('{solr_bindir}/solr stop -all >> {logsearch_solr_log}'),
+ environment = {'SOLR_INCLUDE': format('{logsearch_solr_conf}/logsearch-solr-env.sh')},
+ user=params.logsearch_solr_user,
+ only_if = format("test -f {logsearch_solr_pidfile}")
+ )
+ File(params.logsearch_solr_pidfile,
+ action="delete"
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.logsearch_solr_pidfile)
+
+if __name__ == "__main__":
+ Solr().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
new file mode 100644
index 0000000..3a293fd
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+logsearch_solr_port = config['configurations']['logsearch-solr-env']['logsearch_solr_port']
+logsearch_solr_piddir = config['configurations']['logsearch-solr-env']['logsearch_solr_pid_dir']
+logsearch_solr_pidfile = format("{logsearch_solr_piddir}/solr-{logsearch_solr_port}.pid")
+
+#logsearch pid file
+logsearch_pid_dir = config['configurations']['logsearch-env']['logsearch_pid_dir']
+logsearch_pid_file = format("{logsearch_pid_dir}/logsearch.pid")
+
+#logfeeder pid file
+logfeeder_pid_dir = config['configurations']['logfeeder-env']['logfeeder_pid_dir']
+logfeeder_pid_file = format("{logfeeder_pid_dir}/logfeeder.pid")
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
new file mode 100644
index 0000000..f337527
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
@@ -0,0 +1,28 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "global":{
+ "add_fields":{
+ "cluster":"{{cluster_name}}"
+ },
+ "source":"file",
+ "tail":"true",
+ "gen_event_md5":"true",
+ "start_position":"beginning"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
new file mode 100644
index 0000000..c85a06a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2
@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Updated JAVACLASS to be same as JAVAFILE. Because if class doesn't have package, then it doesn't work.
+JAVACLASS (?:[A-Za-z$0-9_. -]+)
+#JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
+#JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
+
+#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
+JAVAFILE (?:[A-Za-z0-9_. -]+)
+#Allow special <init> or <clinit> method
+JAVAMETHOD (?:(<init>)|(<clinit>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
+#Line number is optional in special cases 'Native method' or 'Unknown source'
+JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
+# Java Logs
+JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
+
+JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
+JAVALOGMESSAGE (.*)
+# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
+CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
+# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800
+TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}
+CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}
+# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...
+TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}
+
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME}
+EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
+EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
+HTTPDUSER %{EMAILADDRESS}|%{USER}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5
]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
+IP (?:%{IPV6}|%{IPV4})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+IPORHOST (?:%{IP}|%{HOSTNAME})
+HOSTPORT %{IPORHOST}:%{POSINT}
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|รค)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG [\x21-\x5a\x5c\x5e-\x7e]+
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
+HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
+
+
+# Log Levels
+LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
+
+
+# Custom
+USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2
new file mode 100644
index 0000000..a6c7ab8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2
@@ -0,0 +1,105 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"accumulo_gc",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/gc_*.log"
+ },
+ {
+ "type":"accumulo_master",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/master_*.log"
+ },
+ {
+ "type":"accumulo_monitor",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/monitor_*.log"
+ },
+ {
+ "type":"accumulo_tracer",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/tracer_*.log"
+ },
+ {
+ "type":"accumulo_tserver",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/tserver_*.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "accumulo_master"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "comment":"This one has one extra space after LEVEL",
+ "conditions":{
+ "fields":{
+ "type":[
+ "accumulo_gc",
+ "accumulo_monitor",
+ "accumulo_tracer",
+ "accumulo_tserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
new file mode 100644
index 0000000..273b880
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
@@ -0,0 +1,93 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"ambari_agent",
+ "rowtype":"service",
+ "path":"{{ambari_agent_log_dir}}/ambari-agent.log"
+ },
+ {
+ "type":"ambari_server",
+ "rowtype":"service",
+ "path":"{{ambari_server_log_dir}}/ambari-server.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_agent"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_server"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
new file mode 100644
index 0000000..d2f3f1a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2
@@ -0,0 +1,92 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"ams_hbase_master",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log"
+ },
+ {
+ "type":"ams_hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log"
+ },
+ {
+ "type":"ams_collector",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_collector"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %p %c: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_hbase_master",
+ "ams_hbase_regionserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2
new file mode 100644
index 0000000..84bc84c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2
@@ -0,0 +1,55 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"atlas_app",
+ "rowtype":"service",
+ "path":"{{atlas_log_dir}}/application.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "atlas_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2
new file mode 100644
index 0000000..2e24e3f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2
@@ -0,0 +1,55 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"falcon_app",
+ "rowtype":"service",
+ "path":"{{falcon_log_dir}}/falcon.application.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "falcon_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
new file mode 100644
index 0000000..fe28f89
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
@@ -0,0 +1,62 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hbase_master",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/hbase-hbase-master-*.log"
+ },
+ {
+ "type":"hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hbase_master",
+ "hbase_regionserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
new file mode 100644
index 0000000..5697012
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2
@@ -0,0 +1,246 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hdfs_datanode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log"
+ },
+ {
+ "type":"hdfs_namenode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log"
+ },
+ {
+ "type":"hdfs_journalnode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log"
+ },
+ {
+ "type":"hdfs_secondarynamenode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log"
+ },
+ {
+ "type":"hdfs_zkfc",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log"
+ },
+ {
+ "type":"hdfs_audit",
+ "rowtype":"audit",
+ "is_enabled":"true",
+ "add_fields":{
+ "logType":"HDFSAudit",
+ "enforcer":"hadoop-acl",
+ "repoType":"1",
+ "repo":"hdfs"
+ },
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_datanode",
+ "hdfs_journalnode",
+ "hdfs_secondarynamenode",
+ "hdfs_namenode",
+ "hdfs_zkfc"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "evtTime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"keyvalue",
+ "sort_order":1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "source_field":"log_message",
+ "value_split":"=",
+ "field_split":"\t",
+ "post_map_values":{
+ "src":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+
+ },
+ "ip":{
+ "map_fieldname":{
+ "new_fieldname":"cliIP"
+ }
+
+ },
+ "allowed":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"true",
+ "post_value":"1"
+ }
+
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"false",
+ "post_value":"0"
+ }
+
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+
+ }
+
+ ],
+ "cmd":{
+ "map_fieldname":{
+ "new_fieldname":"action"
+ }
+
+ },
+ "proto":{
+ "map_fieldname":{
+ "new_fieldname":"cliType"
+ }
+
+ },
+ "callerContext":{
+ "map_fieldname":{
+ "new_fieldname":"req_caller_id"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "sort_order":2,
+ "source_field":"ugi",
+ "remove_source_field":"false",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+ "post_map_values":{
+ "user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "x_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "p_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "k_user":{
+ "map_fieldname":{
+ "new_fieldname":"proxyUsers"
+ }
+
+ },
+ "p_authType":{
+ "map_fieldname":{
+ "new_fieldname":"authType"
+ }
+
+ },
+ "k_authType":{
+ "map_fieldname":{
+ "new_fieldname":"proxyAuthType"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
new file mode 100644
index 0000000..1b8d9a1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2
@@ -0,0 +1,62 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hive_hiveserver2",
+ "rowtype":"service",
+ "path":"{{hive_log_dir}}/hiveserver2.log"
+ },
+ {
+ "type":"hive_metastore",
+ "rowtype":"service",
+ "path":"{{hive_log_dir}}/hivemetastore.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hive_hiveserver2",
+ "hive_metastore"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file