You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/04/09 12:15:42 UTC
[46/61] [abbrv] [partial] ambari git commit: AMBARI-15679. Initial
commit for LogSearch service definition (oleewre)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java
new file mode 100644
index 0000000..a0380e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink.timeline.configuration;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class Configuration {
+ public final Log LOG = LogFactory.getLog(this.getClass());
+ private final Properties properties;
+
+ public Configuration(String configFile) {
+ properties = new Properties();
+
+ //Get property file stream from classpath
+ InputStream inputStream = Configuration.class.getResourceAsStream(configFile);
+
+ if (inputStream == null) {
+ throw new IllegalArgumentException(configFile + " not found in classpath");
+ }
+
+ // load the properties
+ try {
+ properties.load(inputStream);
+ inputStream.close();
+ } catch (FileNotFoundException fnf) {
+ LOG.info("No configuration file " + configFile + " found in classpath.", fnf);
+ } catch (IOException ie) {
+ throw new IllegalArgumentException("Can't read configuration file " +
+ configFile, ie);
+ }
+ }
+
+ public String getProperty(String key) {
+ return properties.getProperty(key);
+ }
+
+ public String getProperty(String key, String defaultValue) {
+ return properties.getProperty(key, defaultValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java
new file mode 100644
index 0000000..b3dc46f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.sink.util;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Helpers to handle server addresses
+ */
+public class Servers {
+ /**
+ * This class is not intended to be instantiated
+ */
+ private Servers() {}
+
+ /**
+ * Parses a space and/or comma separated sequence of server specifications
+ * of the form <i>hostname</i> or <i>hostname:port</i>. If
+ * the specs string is null, defaults to localhost:defaultPort.
+ *
+ * @param specs server specs (see description)
+ * @param defaultPort the default port if not specified
+ * @return a list of InetSocketAddress objects.
+ */
+ public static List<InetSocketAddress> parse(String specs, int defaultPort) {
+ List<InetSocketAddress> result = new ArrayList<InetSocketAddress>();
+ if (specs == null) {
+ result.add(new InetSocketAddress("localhost", defaultPort));
+ } else {
+ String[] specStrings = specs.split("[ ,]+");
+ for (String specString : specStrings) {
+ result.add(createSocketAddr(specString, defaultPort));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * @param host
+ * @param port
+ * @return a InetSocketAddress created with the specified host and port
+ */
+ private static InetSocketAddress createSocketAddr(String target, int defaultPort) {
+ String helpText = "";
+ if (target == null) {
+ throw new IllegalArgumentException("Target address cannot be null." + helpText);
+ }
+ boolean hasScheme = target.contains("://");
+ URI uri = null;
+ try {
+ uri = hasScheme ? URI.create(target) : URI.create("dummyscheme://" + target);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Does not contain a valid host:port authority: " + target + helpText);
+ }
+
+ String host = uri.getHost();
+ int port = uri.getPort();
+ if (port == -1) {
+ port = defaultPort;
+ }
+ String path = uri.getPath();
+
+ if ((host == null) || (port < 0) || (!hasScheme && path != null && !path.isEmpty())) {
+ throw new IllegalArgumentException("Does not contain a valid host:port authority: " + target + helpText);
+ }
+ return createSocketAddrForHost(host, port);
+ }
+
+ /**
+ * @param host
+ * @param port
+ * @return a InetSocketAddress created with the specified host and port
+ */
+ private static InetSocketAddress createSocketAddrForHost(String host, int port) {
+ InetSocketAddress addr;
+ try {
+ InetAddress iaddr = InetAddress.getByName(host);
+ iaddr = InetAddress.getByAddress(host, iaddr.getAddress());
+ addr = new InetSocketAddress(iaddr, port);
+ } catch (UnknownHostException e) {
+ addr = InetSocketAddress.createUnresolved(host, port);
+ }
+ return addr;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
new file mode 100644
index 0000000..40cd855
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+Package: [[artifactId]]
+Version: [[package-version]]-[[package-release]]
+Section: [[deb.section]]
+Priority: [[deb.priority]]
+Depends: [[deb.dependency.list]]
+Architecture: [[deb.architecture]]
+Description: [[description]]
+Maintainer: [[deb.publisher]]
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
new file mode 100644
index 0000000..d8a239a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
@@ -0,0 +1,42 @@
+{
+ "input": {
+ "file": {
+ "klass": "org.apache.ambari.logfeeder.input.InputFile"
+ }
+
+ },
+ "filter": {
+ "json": {
+ "klass": "org.apache.ambari.logfeeder.filter.JSONFilterCode"
+ },
+ "keyvalue": {
+ "klass": "org.apache.ambari.logfeeder.filter.FilterKeyValue"
+ },
+ "grok": {
+ "klass": "org.apache.ambari.logfeeder.filter.FilterGrok"
+ }
+ },
+
+ "mapper": {
+ "map_date": {
+ "klass": "org.apache.ambari.logfeeder.mapper.MapperDate"
+ },
+ "map_fieldname": {
+ "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName"
+ },
+ "map_fieldvalue": {
+ "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldValue"
+ }
+ },
+ "output": {
+ "solr": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputSolr"
+ },
+ "file": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputFile"
+ },
+ "kafka": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputKafka"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
new file mode 100644
index 0000000..163ee2b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
@@ -0,0 +1,995 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "global":{
+ "add_fields":{
+ "cluster":"{{cluster_name}}"
+ },
+ "source":"file",
+ "tail":"true",
+ "gen_event_md5":"true",
+ "start_position":"beginning"
+ },
+ "input":[
+ {
+ "type":"accumulo_gc",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/gc_*.log"
+ },
+ {
+ "type":"accumulo_master",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/master_*.log"
+ },
+ {
+ "type":"accumulo_monitor",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/monitor_*.log"
+ },
+ {
+ "type":"accumulo_tracer",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/tracer_*.log"
+ },
+ {
+ "type":"accumulo_tserver",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/tserver_*.log"
+ },
+ {
+ "type":"atlas_app",
+ "rowtype":"service",
+ "path":"{{atlas_log_dir}}/application.log"
+ },
+ {
+ "type":"ambari_agent",
+ "rowtype":"service",
+ "path":"{{ambari_agent_log_dir}}/ambari-agent.log"
+ },
+ {
+ "type":"ambari_server",
+ "rowtype":"service",
+ "path":"{{ambari_server_log_dir}}/ambari-server.log"
+ },
+ {
+ "type":"ams_hbase_master",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log"
+ },
+ {
+ "type":"ams_hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log"
+ },
+ {
+ "type":"ams_collector",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
+ },
+ {
+ "type":"falcon_app",
+ "rowtype":"service",
+ "path":"{{falcon_log_dir}}/falcon.application.log"
+ },
+ {
+ "type":"hbase_master",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/hbase-hbase-master-*.log"
+ },
+ {
+ "type":"hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+ },
+ {
+ "type":"hdfs_datanode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log"
+ },
+ {
+ "type":"hdfs_namenode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log"
+ },
+ {
+ "type":"hdfs_journalnode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log"
+ },
+ {
+ "type":"hdfs_secondarynamenode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log"
+ },
+ {
+ "type":"hdfs_zkfc",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log"
+ },
+ {
+ "type":"hive_hiveserver2",
+ "rowtype":"service",
+ "path":"{{hive_log_dir}}/hiveserver2.log"
+ },
+ {
+ "type":"hive_metastore",
+ "rowtype":"service",
+ "path":"{{hive_log_dir}}/hivemetastore.log"
+ },
+ {
+ "type":"kafka_controller",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/controller.log"
+ },
+ {
+ "type":"kafka_request",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/kafka-request.log"
+ },
+ {
+ "type":"kafka_logcleaner",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/log-cleaner.log"
+ },
+ {
+ "type":"kafka_server",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/server.log"
+ },
+ {
+ "type":"kafka_statechange",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/state-change.log"
+ },
+ {
+ "type":"knox_gateway",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/gateway.log"
+ },
+ {
+ "type":"knox_cli",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/knoxcli.log"
+ },
+ {
+ "type":"knox_ldap",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/ldap.log"
+ },
+ {
+ "type":"mapred_historyserver",
+ "rowtype":"service",
+ "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log"
+ },
+ {
+ "type":"logsearch_app",
+ "rowtype":"service",
+ "path":"{{logsearch_log_dir}}/logsearch.log"
+ },
+ {
+ "type":"logsearch_feeder",
+ "rowtype":"service",
+ "path":"{{logfeeder_log_dir}}/logfeeder.log"
+ },
+ {
+ "type":"logsearch_perf",
+ "rowtype":"service",
+ "path":"{{logsearch_log_dir}}/logsearch-performance.log"
+ },
+ {
+ "type":"ranger_admin",
+ "rowtype":"service",
+ "path":"{{ranger_admin_log_dir}}/xa_portal.log"
+ },
+ {
+ "type":"ranger_dbpatch",
+ "is_enabled":"true",
+ "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log"
+ },
+ {
+ "type":"ranger_kms",
+ "rowtype":"service",
+ "path":"{{ranger_kms_log_dir}}/kms.log"
+ },
+ {
+ "type":"ranger_usersync",
+ "rowtype":"service",
+ "path":"{{ranger_usersync_log_dir}}/usersync.log"
+ },
+ {
+ "type":"oozie_app",
+ "rowtype":"service",
+ "path":"{{oozie_log_dir}}/oozie.log"
+ },
+ {
+ "type":"yarn_nodemanager",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log"
+ },
+ {
+ "type":"yarn_resourcemanager",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log"
+ },
+ {
+ "type":"yarn_timelineserver",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log"
+ },
+ {
+ "type":"yarn_historyserver",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log"
+ },
+ {
+ "type":"yarn_jobsummary",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log"
+ },
+ {
+ "type":"storm_drpc",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/drpc.log"
+ },
+ {
+ "type":"storm_logviewer",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/logviewer.log"
+ },
+ {
+ "type":"storm_nimbus",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/nimbus.log"
+ },
+ {
+ "type":"storm_supervisor",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/supervisor.log"
+ },
+ {
+ "type":"storm_ui",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/ui.log"
+ },
+ {
+ "type":"storm_worker",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/*worker*.log"
+ },
+ {
+ "type":"zookeeper",
+ "rowtype":"service",
+ "path":"{{zk_log_dir}}/zookeeper/zookeeper*.out"
+ },
+ {
+ "type":"hdfs_audit",
+ "rowtype":"audit",
+ "is_enabled":"true",
+ "add_fields":{
+ "logType":"HDFSAudit",
+ "enforcer":"hadoop-acl",
+ "repoType":"1",
+ "repo":"hdfs"
+ },
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "accumulo_master"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "comment":"This one has one extra space after LEVEL",
+ "conditions":{
+ "fields":{
+ "type":[
+ "accumulo_gc",
+ "accumulo_monitor",
+ "accumulo_tracer",
+ "accumulo_tserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "atlas_app",
+ "falcon_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_collector"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %p %c: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_hbase_master",
+ "ams_hbase_regionserver",
+ "hbase_master",
+ "hbase_regionserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_agent"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_server"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_datanode",
+ "hdfs_journalnode",
+ "hdfs_secondarynamenode",
+ "hdfs_namenode",
+ "hdfs_zkfc",
+ "knox_gateway",
+ "knox_cli",
+ "knox_ldap",
+ "mapred_historyserver",
+ "yarn_historyserver",
+ "yarn_jobsummary",
+ "yarn_nodemanager",
+ "yarn_resourcemanager",
+ "yarn_timelineserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hive_hiveserver2",
+ "hive_metastore"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_controller",
+ "kafka_request",
+ "kafka_logcleaner"
+ ]
+
+ }
+
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_server",
+ "kafka_statechange"
+ ]
+
+ }
+
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "oozie_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "logsearch_app",
+ "logsearch_feeder",
+ "logsearch_perf",
+ "ranger_admin",
+ "ranger_dbpatch"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_kms"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_usersync"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_drpc",
+ "storm_logviewer",
+ "storm_nimbus",
+ "storm_supervisor",
+ "storm_ui",
+ "storm_worker"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "zookeeper"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "evtTime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"keyvalue",
+ "sort_order":1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "source_field":"log_message",
+ "value_split":"=",
+ "field_split":"\t",
+ "post_map_values":{
+ "src":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+
+ },
+ "ip":{
+ "map_fieldname":{
+ "new_fieldname":"cliIP"
+ }
+
+ },
+ "allowed":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"true",
+ "post_value":"1"
+ }
+
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"false",
+ "post_value":"0"
+ }
+
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+
+ }
+
+ ],
+ "cmd":{
+ "map_fieldname":{
+ "new_fieldname":"action"
+ }
+
+ },
+ "proto":{
+ "map_fieldname":{
+ "new_fieldname":"cliType"
+ }
+
+ },
+ "callerContext":{
+ "map_fieldname":{
+ "new_fieldname":"req_caller_id"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "sort_order":2,
+ "source_field":"ugi",
+ "remove_source_field":"false",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+ "post_map_values":{
+ "user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "x_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "p_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "k_user":{
+ "map_fieldname":{
+ "new_fieldname":"proxyUsers"
+ }
+
+ },
+ "p_authType":{
+ "map_fieldname":{
+ "new_fieldname":"authType"
+ }
+
+ },
+ "k_authType":{
+ "map_fieldname":{
+ "new_fieldname":"proxyAuthType"
+ }
+
+ }
+
+ }
+
+ }
+
+ ],
+ "output":[
+ {
+ "is_enabled":"{{solr_service_logs_enable}}",
+ "comment":"Output to solr for service logs",
+ "destination":"solr",
+ "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+ "collection":"{{solr_collection_service_logs}}",
+ "number_of_shards": "{{logsearch_numshards}}",
+ "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "service"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "comment":"Output to solr for audit records",
+ "is_enabled":"{{solr_audit_logs_enable}}",
+ "destination":"solr",
+ "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+ "collection":"{{solr_collection_audit_logs}}",
+ "number_of_shards": "{{logsearch_numshards}}",
+ "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "audit"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "is_enabled":"{{kafka_service_logs_enable}}",
+ "destination":"kafka",
+ "broker_list":"{{kafka_broker_list}}",
+ "topic":"{{kafka_topic_service_logs}}",
+ "kafka.security.protocol":"{{kafka_security_protocol}}",
+ "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "service"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "is_enabled":"{{kafka_topic_service_logs}}",
+ "destination":"kafka",
+ "broker_list":"{{kafka_broker_list}}",
+ "topic":"{{kafka_topic_audit_logs}}",
+ "kafka.security.protocol":"{{kafka_security_protocol}}",
+ "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "audit"
+ ]
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json
new file mode 100644
index 0000000..9493c6c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json
@@ -0,0 +1,626 @@
+{
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "accumulo_master"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "comment":"This one has one extra space after LEVEL",
+ "conditions":{
+ "fields":{
+ "type":[
+ "accumulo_gc",
+ "accumulo_monitor",
+ "accumulo_tracer",
+ "accumulo_tserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "atlas_app",
+ "falcon_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_collector"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %p %c: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ams_hbase_master",
+ "ams_hbase_regionserver",
+ "hbase_master",
+ "hbase_regionserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_agent"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ },
+ "level":{
+ "map_fieldvalue":{
+ "pre_value":"WARNING",
+ "post_value":"WARN"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_server"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_datanode",
+ "hdfs_journalnode",
+ "hdfs_secondarynamenode",
+ "hdfs_namenode",
+ "hdfs_zkfc",
+ "knox_gateway",
+ "knox_cli",
+ "knox_ldap",
+ "mapred_historyserver",
+ "yarn_historyserver",
+ "yarn_jobsummary",
+ "yarn_nodemanager",
+ "yarn_resourcemanager",
+ "yarn_timelineserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hive_hiveserver2",
+ "hive_metastore"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_controller",
+ "kafka_request",
+ "kafka_logcleaner"
+ ]
+
+ }
+
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_server",
+ "kafka_statechange"
+ ]
+
+ }
+
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "oozie_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "logsearch_app",
+ "logsearch_feeder",
+ "logsearch_perf",
+ "ranger_admin",
+ "ranger_dbpatch"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_kms"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_usersync"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_drpc",
+ "storm_logviewer",
+ "storm_nimbus",
+ "storm_supervisor",
+ "storm_ui",
+ "storm_worker"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "zookeeper"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "evtTime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"keyvalue",
+ "sort_order":1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "source_field":"log_message",
+ "value_split":"=",
+ "field_split":"\t",
+ "post_map_values":{
+ "src":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+
+ },
+ "ip":{
+ "map_fieldname":{
+ "new_fieldname":"cliIP"
+ }
+
+ },
+ "allowed":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"true",
+ "post_value":"1"
+ }
+
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"false",
+ "post_value":"0"
+ }
+
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+
+ }
+
+ ],
+ "cmd":{
+ "map_fieldname":{
+ "new_fieldname":"action"
+ }
+
+ },
+ "proto":{
+ "map_fieldname":{
+ "new_fieldname":"cliType"
+ }
+
+ },
+ "callerContext":{
+ "map_fieldname":{
+ "new_fieldname":"req_caller_id"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "sort_order":2,
+ "source_field":"ugi",
+ "remove_source_field":"false",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+ "post_map_values":{
+ "user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "x_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "p_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "k_user":{
+ "map_fieldname":{
+ "new_fieldname":"proxyUsers"
+ }
+
+ },
+ "p_authType":{
+ "map_fieldname":{
+ "new_fieldname":"authType"
+ }
+
+ },
+ "k_authType":{
+ "map_fieldname":{
+ "new_fieldname":"proxyAuthType"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2
new file mode 100644
index 0000000..cd51118
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2
@@ -0,0 +1,28 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "global":{
+ "add_fields":{
+ "cluster":"{{cluster_name}}"
+ },
+ "source":"file",
+ "tail":"true",
+ "gen_event_md5":"true",
+ "start_position":"beginning"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns
new file mode 100644
index 0000000..d25a78b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns
@@ -0,0 +1,145 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Updated JAVACLASS to be same as JAVAFILE. Because if class doesn't have package, then it doesn't work.
+JAVACLASS (?:[A-Za-z$0-9_. -]+)
+#JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
+#JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
+
+#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
+JAVAFILE (?:[A-Za-z0-9_. -]+)
+#Allow special <init> or <clinit> method
+JAVAMETHOD (?:(<init>)|(<clinit>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
+#Line number is optional in special cases 'Native method' or 'Unknown source'
+JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
+# Java Logs
+JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
+
+JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
+JAVALOGMESSAGE (.*)
+# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
+CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
+# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800
+TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}
+CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}
+# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...
+TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}
+
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME}
+EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
+EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
+HTTPDUSER %{EMAILADDRESS}|%{USER}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5
]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
+IP (?:%{IPV6}|%{IPV4})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+IPORHOST (?:%{IP}|%{HOSTNAME})
+HOSTPORT %{IPORHOST}:%{POSINT}
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|รค)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG [\x21-\x5a\x5c\x5e-\x7e]+
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
+HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
+
+
+# Log Levels
+LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
+
+
+# Custom
+USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2
new file mode 100644
index 0000000..bc48503
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2
@@ -0,0 +1,284 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"accumulo_gc",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/gc_*.log"
+ },
+ {
+ "type":"accumulo_master",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/master_*.log"
+ },
+ {
+ "type":"accumulo_monitor",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/monitor_*.log"
+ },
+ {
+ "type":"accumulo_tracer",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/tracer_*.log"
+ },
+ {
+ "type":"accumulo_tserver",
+ "rowtype":"service",
+ "path":"{{accumulo_log_dir}}/tserver_*.log"
+ },
+ {
+ "type":"atlas_app",
+ "rowtype":"service",
+ "path":"{{atlas_log_dir}}/application.log"
+ },
+ {
+ "type":"ambari_agent",
+ "rowtype":"service",
+ "path":"{{ambari_agent_log_dir}}/ambari-agent.log"
+ },
+ {
+ "type":"ambari_server",
+ "rowtype":"service",
+ "path":"{{ambari_server_log_dir}}/ambari-server.log"
+ },
+ {
+ "type":"ams_hbase_master",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log"
+ },
+ {
+ "type":"ams_hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log"
+ },
+ {
+ "type":"ams_collector",
+ "rowtype":"service",
+ "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
+ },
+ {
+ "type":"falcon_app",
+ "rowtype":"service",
+ "path":"{{falcon_log_dir}}/falcon.application.log"
+ },
+ {
+ "type":"hbase_master",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/hbase-hbase-master-*.log"
+ },
+ {
+ "type":"hbase_regionserver",
+ "rowtype":"service",
+ "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+ },
+ {
+ "type":"hdfs_datanode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log"
+ },
+ {
+ "type":"hdfs_namenode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log"
+ },
+ {
+ "type":"hdfs_journalnode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log"
+ },
+ {
+ "type":"hdfs_secondarynamenode",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log"
+ },
+ {
+ "type":"hdfs_zkfc",
+ "rowtype":"service",
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log"
+ },
+ {
+ "type":"hive_hiveserver2",
+ "rowtype":"service",
+ "path":"{{hive_log_dir}}/hiveserver2.log"
+ },
+ {
+ "type":"hive_metastore",
+ "rowtype":"service",
+ "path":"{{hive_log_dir}}/hivemetastore.log"
+ },
+ {
+ "type":"kafka_controller",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/controller.log"
+ },
+ {
+ "type":"kafka_request",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/kafka-request.log"
+ },
+ {
+ "type":"kafka_logcleaner",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/log-cleaner.log"
+ },
+ {
+ "type":"kafka_server",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/server.log"
+ },
+ {
+ "type":"kafka_statechange",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/state-change.log"
+ },
+ {
+ "type":"knox_gateway",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/gateway.log"
+ },
+ {
+ "type":"knox_cli",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/knoxcli.log"
+ },
+ {
+ "type":"knox_ldap",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/ldap.log"
+ },
+ {
+ "type":"mapred_historyserver",
+ "rowtype":"service",
+ "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log"
+ },
+ {
+ "type":"logsearch_app",
+ "rowtype":"service",
+ "path":"{{logsearch_log_dir}}/logsearch.log"
+ },
+ {
+ "type":"logsearch_feeder",
+ "rowtype":"service",
+ "path":"{{logfeeder_log_dir}}/logfeeder.log"
+ },
+ {
+ "type":"logsearch_perf",
+ "rowtype":"service",
+ "path":"{{logsearch_log_dir}}/logsearch-performance.log"
+ },
+ {
+ "type":"ranger_admin",
+ "rowtype":"service",
+ "path":"{{ranger_admin_log_dir}}/xa_portal.log"
+ },
+ {
+ "type":"ranger_dbpatch",
+ "is_enabled":"true",
+ "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log"
+ },
+ {
+ "type":"ranger_kms",
+ "rowtype":"service",
+ "path":"{{ranger_kms_log_dir}}/kms.log"
+ },
+ {
+ "type":"ranger_usersync",
+ "rowtype":"service",
+ "path":"{{ranger_usersync_log_dir}}/usersync.log"
+ },
+ {
+ "type":"oozie_app",
+ "rowtype":"service",
+ "path":"{{oozie_log_dir}}/oozie.log"
+ },
+ {
+ "type":"yarn_nodemanager",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log"
+ },
+ {
+ "type":"yarn_resourcemanager",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log"
+ },
+ {
+ "type":"yarn_timelineserver",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log"
+ },
+ {
+ "type":"yarn_historyserver",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log"
+ },
+ {
+ "type":"yarn_jobsummary",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log"
+ },
+ {
+ "type":"storm_drpc",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/drpc.log"
+ },
+ {
+ "type":"storm_logviewer",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/logviewer.log"
+ },
+ {
+ "type":"storm_nimbus",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/nimbus.log"
+ },
+ {
+ "type":"storm_supervisor",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/supervisor.log"
+ },
+ {
+ "type":"storm_ui",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/ui.log"
+ },
+ {
+ "type":"storm_worker",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/*worker*.log"
+ },
+ {
+ "type":"zookeeper",
+ "rowtype":"service",
+ "path":"{{zk_log_dir}}/zookeeper/zookeeper*.out"
+ },
+ {
+ "type":"hdfs_audit",
+ "rowtype":"audit",
+ "is_enabled":"true",
+ "add_fields":{
+ "logType":"HDFSAudit",
+ "enforcer":"hadoop-acl",
+ "repoType":"1",
+ "repo":"hdfs"
+ },
+ "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log"
+ }
+
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
new file mode 100644
index 0000000..0717477
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
+ <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> -->
+ </layout>
+ </appender>
+
+ <appender name="daily_rolling_file" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="file" value="logs/logsearch-logfeeder.log" />
+ <param name="datePattern" value="'.'yyyy-MM-dd" />
+ <param name="append" value="true" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Logs to suppress BEGIN -->
+ <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
+ <priority value="error" />
+ <appender-ref ref="rolling_file" />
+ </category>
+
+ <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false">
+ <priority value="fatal" />
+ <appender-ref ref="rolling_file" />
+ </category>
+ <!-- Logs to suppress END -->
+
+ <category name="org.apache.ambari.logfeeder" additivity="false">
+ <priority value="info" />
+ <appender-ref ref="console" />
+ <!-- <appender-ref ref="daily_rolling_file" /> -->
+ </category>
+
+ <root>
+ <priority value="warn" />
+ <!-- <appender-ref ref="console" /> -->
+ <!-- <appender-ref ref="daily_rolling_file" /> -->
+ </root>
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2
new file mode 100644
index 0000000..4338ee3
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
+ <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> -->
+ </layout>
+ </appender>
+
+ <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender">
+ <param name="file" value="{{logfeeder_log_dir}}/logfeeder.log" />
+ <param name="append" value="true" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Logs to suppress BEGIN -->
+ <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
+ <priority value="error" />
+ <appender-ref ref="rolling_file" />
+ </category>
+
+ <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false">
+ <priority value="fatal" />
+ <appender-ref ref="rolling_file" />
+ </category>
+
+ <!-- Logs to suppress END -->
+
+ <category name="org.apache.ambari.logfeeder" additivity="false">
+ <priority value="{{logfeeder_log_level}}" />
+ <appender-ref ref="rolling_file" />
+ </category>
+
+ <root>
+ <level value="warn" />
+ <!-- <appender-ref ref="console" /> -->
+ <appender-ref ref="rolling_file" />
+ </root>
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
new file mode 100644
index 0000000..22f3b78
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logfeeder.checkpoint.folder=
+metrics.collector.hosts=
+
+#filter config
+logfeeder.solr.url=
+logfeeder.solr.zkhosts=
+logfeeder.solr.core.history=history
+logfeeder.log.filter.enable=true
+#Internal to fetch filter config from solr in sec
+logfeeder.solr.config.internal=5
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2
new file mode 100644
index 0000000..d0aea47
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2
@@ -0,0 +1,97 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "output":[
+ {
+ "is_enabled":"{{solr_service_logs_enable}}",
+ "comment":"Output to solr for service logs",
+ "destination":"solr",
+ "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+ "collection":"{{solr_collection_service_logs}}",
+ "number_of_shards": "{{logsearch_numshards}}",
+ "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "service"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "comment":"Output to solr for audit records",
+ "is_enabled":"{{solr_audit_logs_enable}}",
+ "destination":"solr",
+ "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+ "collection":"{{solr_collection_audit_logs}}",
+ "number_of_shards": "{{logsearch_numshards}}",
+ "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "audit"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "is_enabled":"{{kafka_service_logs_enable}}",
+ "destination":"kafka",
+ "broker_list":"{{kafka_broker_list}}",
+ "topic":"{{kafka_topic_service_logs}}",
+ "kafka.security.protocol":"{{kafka_security_protocol}}",
+ "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "service"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "is_enabled":"{{kafka_topic_service_logs}}",
+ "destination":"kafka",
+ "broker_list":"{{kafka_broker_list}}",
+ "topic":"{{kafka_topic_audit_logs}}",
+ "kafka.security.protocol":"{{kafka_security_protocol}}",
+ "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "audit"
+ ]
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
new file mode 100644
index 0000000..1e7185a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+curr_dir=`pwd`
+cd `dirname $0`; script_dir=`pwd`; cd $curr_dir
+
+foreground=0
+if [ "$1" = "-foreground" ]; then
+ foreground=1
+ shift
+fi
+
+if [ ! -z "$LOGFEEDER_INCLUDE" ]; then
+ source $LOGFEEDER_INCLUDE
+fi
+
+JAVA=java
+if [ -x $JAVA_HOME/bin/java ]; then
+ JAVA=$JAVA_HOME/bin/java
+fi
+
+if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
+ LOGFEEDER_JAVA_MEM="-Xmx512m"
+fi
+
+if [ "$LOGFILE" = "" ]; then
+ LOGFILE="/var/log/logfeeder/logfeeder.out"
+fi
+
+if [ "$PID_FILE" = "" ]; then
+ LOGFEEDER_PID_DIR=$HOME
+ PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid
+fi
+
+if [ "$LOGFEEDER_CONF_DIR" = "" ]; then
+ LOGFEEDER_CONF_DIR="/etc/logfeeder/conf"
+fi
+
+LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log
+LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE"
+
+#LOGFEEDER_JAVA_OPTS=
+#JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098"
+
+if [ $foreground -eq 0 ]; then
+ if [ -f ${PID_FILE} ]; then
+ PID=`cat ${PID_FILE}`
+ if kill -0 $PID 2>/dev/null; then
+ echo "logfeeder already running (${PID}) killing..."
+ kill $PID 2>/dev/null
+ sleep 5
+ if kill -0 $PID 2>/dev/null; then
+ echo "logfeeder still running. Will kill process forcefully in another 10 seconds..."
+ sleep 10
+ kill -9 $PID 2>/dev/null
+ sleep 2
+ fi
+ fi
+
+ if kill -0 $PID 2>/dev/null; then
+ echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again."
+ exit 1
+ fi
+ fi
+
+ echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE"
+ #LOGFEEDER_CLI_CLASSPATH=
+ #set -x
+ nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes:$script_dir/LogProcessor.jar" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* > $LOGFILE 2>&1 &
+ echo $! > $PID_FILE
+else
+ $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes:$script_dir/LogProcessor.jar" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $*
+fi
+