You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/05/04 10:40:37 UTC
[ambari] branch trunk updated: AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense… (#1174)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8dfe69e AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense… (#1174)
8dfe69e is described below
commit 8dfe69e9b43ec0d6cf4ad39e1143c120e1402380
Author: kasakrisz <33...@users.noreply.github.com>
AuthorDate: Fri May 4 12:40:32 2018 +0200
AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense… (#1174)
* AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense etc.
* AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense etc.
revert input.config-hdfs.json
---
ambari-logsearch/ambari-logsearch-it/log4j.dtd | 230 ++++++++++++++++++++
ambari-logsearch/ambari-logsearch-it/pom.xml | 63 +++++-
.../patterns/AmbariInfraSolrLogPatternIT.java | 39 ++++
.../logsearch/patterns/AmbariLogPatternIT.java | 235 +++++++++++++++++++++
.../logsearch/patterns/AtlasLogPatternIT.java | 52 +++++
.../logsearch/patterns/HBaseLogPatternIT.java | 55 +++++
.../logsearch/patterns/HDFSLogPatternIT.java | 52 +++++
.../logsearch/patterns/HdfsAuditLogPatternIT.java | 89 ++++++++
.../logsearch/patterns/HiveLogPatterntIT.java | 45 ++++
.../ambari/logsearch/patterns/JinjaFunctions.java | 27 +++
.../logsearch/patterns/KafkaLogPatternIT.java | 100 +++++++++
.../logsearch/patterns/KnoxLogPatternIT.java | 69 ++++++
.../ambari/logsearch/patterns/LayoutQuery.java | 23 ++
.../ambari/logsearch/patterns/ListAppender.java | 59 ++++++
.../ambari/logsearch/patterns/Log4jContent.java | 25 +++
.../ambari/logsearch/patterns/Log4jProperties.java | 65 ++++++
.../apache/ambari/logsearch/patterns/Log4jXml.java | 74 +++++++
.../logsearch/patterns/Log4jXmlProperties.java | 45 ++++
.../logsearch/patterns/MetricsLogPatternIT.java | 77 +++++++
.../ambari/logsearch/patterns/PatternITBase.java | 147 +++++++++++++
.../logsearch/patterns/RangerLogPatternIT.java | 86 ++++++++
.../logsearch/patterns/Spark2LogPatternIT.java | 71 +++++++
.../ambari/logsearch/patterns/StackDefContent.java | 61 ++++++
.../logsearch/patterns/StormLogPatternIT.java | 100 +++++++++
.../logsearch/patterns/YarnLogPatternIT.java | 169 +++++++++++++++
.../logsearch/patterns/ZeppelinLogPatternIT.java | 55 +++++
.../logsearch/patterns/ZookeeperLogPatternIT.java | 53 +++++
.../ambari/logfeeder/common/ConfigHandler.java | 8 +-
.../logfeeder/common/LogEntryParseTester.java | 3 +-
ambari-server/conf/unix/log4j.properties | 2 +-
.../package/templates/input.config-kafka.json.j2 | 2 +-
.../0.5.0/properties/input.config-ambari.json.j2 | 8 +-
.../package/templates/input.config-storm.json.j2 | 79 ++++++-
.../templates/input.config-mapreduce2.json.j2 | 18 +-
34 files changed, 2260 insertions(+), 26 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-it/log4j.dtd b/ambari-logsearch/ambari-logsearch-it/log4j.dtd
new file mode 100644
index 0000000..8918101
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/log4j.dtd
@@ -0,0 +1,230 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+ <!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!ELEMENT log4j:configuration (renderer*, throwableRenderer?,
+ appender*,plugin*, (category|logger)*,root?,
+ (categoryFactory|loggerFactory)?)>
+
+ <!-- The "threshold" attribute takes a level value below which -->
+ <!-- all logging statements are disabled. -->
+
+ <!-- Setting the "debug" enable the printing of internal log4j logging -->
+ <!-- statements. -->
+
+ <!-- By default, debug attribute is "null", meaning that we not do touch -->
+ <!-- internal log4j logging settings. The "null" value for the threshold -->
+ <!-- attribute can be misleading. The threshold field of a repository -->
+ <!-- cannot be set to null. The "null" value for the threshold attribute -->
+ <!-- simply means don't touch the threshold field, the threshold field -->
+ <!-- keeps its old value. -->
+
+ <!ATTLIST log4j:configuration
+ xmlns:log4j CDATA #FIXED "http://jakarta.apache.org/log4j/"
+ threshold (all|trace|debug|info|warn|error|fatal|off|null) "null"
+ debug (true|false|null) "null"
+ reset (true|false) "false"
+ >
+
+ <!-- renderer elements allow the user to customize the conversion of -->
+ <!-- message objects to String. -->
+
+ <!ELEMENT renderer EMPTY>
+ <!ATTLIST renderer
+ renderedClass CDATA #REQUIRED
+ renderingClass CDATA #REQUIRED
+ >
+
+ <!-- throwableRenderer allows the user to customize the conversion
+ of exceptions to a string representation. -->
+ <!ELEMENT throwableRenderer (param*)>
+ <!ATTLIST throwableRenderer
+ class CDATA #REQUIRED
+ >
+
+
+ <!-- Appenders must have a name and a class. -->
+ <!-- Appenders may contain an error handler, a layout, optional parameters -->
+ <!-- and filters. They may also reference (or include) other appenders. -->
+ <!ELEMENT appender (errorHandler?, param*,
+ rollingPolicy?, triggeringPolicy?, connectionSource?,
+ layout?, filter*, appender-ref*)>
+ <!ATTLIST appender
+ name CDATA #REQUIRED
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT layout (param*)>
+ <!ATTLIST layout
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT filter (param*)>
+ <!ATTLIST filter
+ class CDATA #REQUIRED
+ >
+
+ <!-- ErrorHandlers can be of any class. They can admit any number of -->
+ <!-- parameters. -->
+
+ <!ELEMENT errorHandler (param*, root-ref?, logger-ref*, appender-ref?)>
+ <!ATTLIST errorHandler
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT root-ref EMPTY>
+
+ <!ELEMENT logger-ref EMPTY>
+ <!ATTLIST logger-ref
+ ref CDATA #REQUIRED
+ >
+
+ <!ELEMENT param EMPTY>
+ <!ATTLIST param
+ name CDATA #REQUIRED
+ value CDATA #REQUIRED
+ >
+
+
+ <!-- The priority class is org.apache.log4j.Level by default -->
+ <!ELEMENT priority (param*)>
+ <!ATTLIST priority
+ class CDATA #IMPLIED
+ value CDATA #REQUIRED
+ >
+
+ <!-- The level class is org.apache.log4j.Level by default -->
+ <!ELEMENT level (param*)>
+ <!ATTLIST level
+ class CDATA #IMPLIED
+ value CDATA #REQUIRED
+ >
+
+
+ <!-- If no level element is specified, then the configurator MUST not -->
+ <!-- touch the level of the named category. -->
+ <!ELEMENT category (param*,(priority|level)?,appender-ref*)>
+ <!ATTLIST category
+ class CDATA #IMPLIED
+ name CDATA #REQUIRED
+ additivity (true|false) "true"
+ >
+
+ <!-- If no level element is specified, then the configurator MUST not -->
+ <!-- touch the level of the named logger. -->
+ <!ELEMENT logger (param*,level?,appender-ref*)>
+ <!ATTLIST logger
+ class CDATA #IMPLIED
+ name CDATA #REQUIRED
+ additivity (true|false) "true"
+ >
+
+
+ <!ELEMENT categoryFactory (param*)>
+ <!ATTLIST categoryFactory
+ class CDATA #REQUIRED>
+
+ <!ELEMENT loggerFactory (param*)>
+ <!ATTLIST loggerFactory
+ class CDATA #REQUIRED>
+
+ <!ELEMENT appender-ref EMPTY>
+ <!ATTLIST appender-ref
+ ref CDATA #REQUIRED
+ >
+
+ <!-- plugins must have a name and class and can have optional parameters -->
+ <!ELEMENT plugin (param*, connectionSource?)>
+ <!ATTLIST plugin
+ name CDATA #REQUIRED
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT connectionSource (dataSource?, param*)>
+ <!ATTLIST connectionSource
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT dataSource (param*)>
+ <!ATTLIST dataSource
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT triggeringPolicy ((param|filter)*)>
+ <!ATTLIST triggeringPolicy
+ name CDATA #IMPLIED
+ class CDATA #REQUIRED
+ >
+
+ <!ELEMENT rollingPolicy (param*)>
+ <!ATTLIST rollingPolicy
+ name CDATA #IMPLIED
+ class CDATA #REQUIRED
+ >
+
+
+ <!-- If no priority element is specified, then the configurator MUST not -->
+ <!-- touch the priority of root. -->
+ <!-- The root category always exists and cannot be subclassed. -->
+ <!ELEMENT root (param*, (priority|level)?, appender-ref*)>
+
+
+ <!-- ==================================================================== -->
+ <!-- A logging event -->
+ <!-- ==================================================================== -->
+ <!ELEMENT log4j:eventSet (log4j:event*)>
+ <!ATTLIST log4j:eventSet
+ xmlns:log4j CDATA #FIXED "http://jakarta.apache.org/log4j/"
+ version (1.1|1.2) "1.2"
+ includesLocationInfo (true|false) "true"
+ >
+
+
+
+ <!ELEMENT log4j:event (log4j:message, log4j:NDC?, log4j:throwable?,
+ log4j:locationInfo?, log4j:properties?) >
+
+ <!-- The timestamp format is application dependent. -->
+ <!ATTLIST log4j:event
+ logger CDATA #REQUIRED
+ level CDATA #REQUIRED
+ thread CDATA #REQUIRED
+ timestamp CDATA #REQUIRED
+ time CDATA #IMPLIED
+ >
+
+ <!ELEMENT log4j:message (#PCDATA)>
+ <!ELEMENT log4j:NDC (#PCDATA)>
+
+ <!ELEMENT log4j:throwable (#PCDATA)>
+
+ <!ELEMENT log4j:locationInfo EMPTY>
+ <!ATTLIST log4j:locationInfo
+ class CDATA #REQUIRED
+ method CDATA #REQUIRED
+ file CDATA #REQUIRED
+ line CDATA #REQUIRED
+ >
+
+ <!ELEMENT log4j:properties (log4j:data*)>
+
+ <!ELEMENT log4j:data EMPTY>
+ <!ATTLIST log4j:data
+ name CDATA #REQUIRED
+ value CDATA #REQUIRED
+ >
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-it/pom.xml b/ambari-logsearch/ambari-logsearch-it/pom.xml
index db3e09f..6a2fdf1 100644
--- a/ambari-logsearch/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-it/pom.xml
@@ -117,7 +117,25 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>11.0.1</version>
+ <version>16.0.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.hubspot.jinjava</groupId>
+ <artifactId>jinjava</artifactId>
+ <version>2.2.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
</dependency>
</dependencies>
@@ -224,6 +242,49 @@
</build>
</profile>
<profile>
+ <id>input-config-tests</id>
+ <activation>
+ <property>
+ <name>input-config-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${failsafe-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>run-integration-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>**/*IT.java</include>
+ </includes>
+ <systemPropertyVariables>
+ <log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration>
+ <docker.host>${docker.host}</docker.host>
+ <backend.stories.location>${backend.stories.location}</backend.stories.location>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ <execution>
+ <id>verify-integration-tests</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>all-tests</id>
<activation>
<property>
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariInfraSolrLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariInfraSolrLogPatternIT.java
new file mode 100644
index 0000000..7b55716
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariInfraSolrLogPatternIT.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.File;
+
+import org.junit.Test;
+
+public class AmbariInfraSolrLogPatternIT extends PatternITBase {
+ // TODO: use hdp_ambari_definitions
+ @Test
+ public void testAmbariInfraSolrLogLayout() {
+ String layout = Log4jProperties.loadFrom(new File(AMBARI_STACK_DEFINITIONS, "AMBARI_INFRA_SOLR/0.1.0/properties/solr-log4j.properties.j2")).getLayout("file");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testAmbariInfraSolrGrokPatter() throws Exception {
+ String layout = Log4jProperties.loadFrom(new File(AMBARI_STACK_DEFINITIONS, "AMBARI_INFRA_SOLR/0.1.0/properties/solr-log4j.properties.j2")).getLayout("file");
+ testServiceLog("infra_solr", layout, inputConfigTemplate(
+ new File(AMBARI_STACK_DEFINITIONS, "AMBARI_INFRA_SOLR/0.1.0/package/templates/input.config-ambari-infra.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java
new file mode 100644
index 0000000..34f17c7
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Date;
+import java.util.Map;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class AmbariLogPatternIT extends PatternITBase {
+
+ public static File AMBARI_CONF;
+
+ @BeforeClass
+ public static void setupAmbariConfig() throws Exception {
+ setupGlobal();
+ AMBARI_CONF = new File(AMBARI_FOLDER, Paths.get("ambari-server", "conf", "unix").toString());
+ }
+
+ @Test
+ public void testAmbariAgentLogEntry() throws Exception {
+ // given
+ String logEntry = "INFO 2018-05-02 09:29:12,359 DataCleaner.py:39 - Data cleanup thread started";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "ambari_agent", ambariInputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("ambari_agent"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Data cleanup thread started"));
+ assertThat(result.get("file"), is("DataCleaner.py"));
+ assertThat(result.get("line_number"), is("39"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 29, 12, 359000000)));
+ }
+
+ @Test
+ public void testAmbariAgentMultilineLogEntry() throws Exception {
+ // given
+ String logEntry = "INFO 2018-05-02 09:31:52,227 RecoveryManager.py:572 - RecoverConfig = {u'components': u'',\n" +
+ " u'maxCount': u'6',\n" +
+ " u'maxLifetimeCount': u'1024',\n" +
+ " u'retryGap': u'5',\n" +
+ " u'type': u'AUTO_START',\n" +
+ " u'windowInMinutes': u'60'}";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "ambari_agent", ambariInputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("ambari_agent"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("RecoverConfig = {u'components': u'',\n" +
+ " u'maxCount': u'6',\n" +
+ " u'maxLifetimeCount': u'1024',\n" +
+ " u'retryGap': u'5',\n" +
+ " u'type': u'AUTO_START',\n" +
+ " u'windowInMinutes': u'60'}"));
+ assertThat(result.get("file"), is("RecoveryManager.py"));
+ assertThat(result.get("line_number"), is("572"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 31, 52, 227000000)));
+ }
+
+ @Test
+ public void testAmbariServerLogLayout() {
+ testAmbariServerLogLayout("file");
+ }
+
+ @Test
+ public void testAmbariAlertsLogLayout() {
+ testAmbariServerLogLayout("alerts");
+ }
+
+ @Test
+ public void testAmbariConfigChangesLogLayout() {
+ testAmbariServerLogLayout("configchange");
+ }
+
+ @Test
+ public void testAmbariDbCheckLogLayout() {
+ testAmbariServerLogLayout("dbcheckhelper");
+ }
+
+ public void testAmbariServerLogLayout(String appenderName) {
+ String layout = Log4jProperties.loadFrom(new File(AMBARI_CONF, "log4j.properties")).getLayout(appenderName);
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testAmbariServerLog() throws Exception {
+ testAmbariServerLog("file", "ambari_server");
+ }
+
+ @Test
+ public void testAmbariConfigChangesLog() throws Exception {
+ testAmbariServerLog("configchange", "ambari_config_changes");
+ }
+
+ @Test
+ public void testAmbariDBCheckLog() throws Exception {
+ testAmbariServerLog("dbcheckhelper", "ambari_server_check_database");
+ }
+
+ public void testAmbariServerLog(String appenderName, String logId) throws Exception {
+ String layout = Log4jProperties.loadFrom(new File(AMBARI_CONF, "log4j.properties")).getLayout(appenderName);
+ testServiceLog(logId, layout, ambariInputConfigTemplate());
+ }
+
+ private String ambariInputConfigTemplate() throws IOException {
+ return inputConfigTemplate(
+ new File(AMBARI_STACK_DEFINITIONS, "LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2"));
+ }
+
+ @Test
+ public void testAmbariAlertsLog() throws Exception {
+ // given
+ String layout = Log4jProperties.loadFrom(new File(AMBARI_CONF, "log4j.properties")).getLayout("alerts");
+ String logEntry = generateLogEntry(layout);
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "ambari_alerts", ambariInputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("ambari_alerts"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message").toString().contains("This is a test message"), is(true));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime.toLocalDate(), is(LocalDate.now()));
+ }
+
+ @Test
+ public void testAmbariEclipseLinkEntry() throws Exception {
+ // given
+ String logEntry = "[EL Info]: 2018-05-02 09:27:17.79--ServerSession(1657512321)-- EclipseLink, version: Eclipse Persistence Services - 2.6.2.v20151217-774c696";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "ambari_eclipselink", ambariInputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("ambari_eclipselink"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("--ServerSession(1657512321)-- EclipseLink, version: Eclipse Persistence Services - 2.6.2.v20151217-774c696"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 27, 17, 79000000)));
+
+ }
+
+ @Test
+ public void testAmbariAuditLogEntry() throws Exception {
+ // given
+ String logEntry = "2018-05-02T09:28:10.302Z, User(null), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" +
+ "), Status(Failed), Reason(Authentication required), Consecutive failures(UNKNOWN USER)\n" +
+ "2018-05-02T09:28:10.346Z, User(admin), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" +
+ " Ambari: Ambari Administrator\n" +
+ "), Status(Success)";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "ambari_audit", ambariInputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("ambari_audit"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("User(null), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" +
+ "), Status(Failed), Reason(Authentication required), Consecutive failures(UNKNOWN USER)\n" +
+ "2018-05-02T09:28:10.346Z, User(admin), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" +
+ " Ambari: Ambari Administrator\n" +
+ "), Status(Success)"));
+ Date logTime = (Date) result.get("evtTime");
+ ZonedDateTime localDateTime = ZonedDateTime.ofInstant(logTime.toInstant(), ZoneId.of("Z"));
+ assertThat(localDateTime, is(ZonedDateTime.parse("2018-05-02T09:28:10.302Z")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java
new file mode 100644
index 0000000..7aca100
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class AtlasLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testAtlasLogLayout() {
+ String layout = Log4jXml.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "ATLAS", "configuration", "atlas-log4j.xml").toString())).getLayout("FILE");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testAtlas() throws Exception {
+ String layout = Log4jXml.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "ATLAS", "configuration", "atlas-log4j.xml").toString())).getLayout("FILE");
+
+ testServiceLog("atlas_app", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER,"ATLAS/package/templates/input.config-atlas.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java
new file mode 100644
index 0000000..d8bee9e
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class HBaseLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testHBaseLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "HBASE", "configuration", "hbase-log4j.xml").toString())).getLayout("RFA");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testHBase() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "HBASE", "configuration", "hbase-log4j.xml").toString())).getLayout("RFA");
+
+ testServiceLog("hbase_master", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HBASE/package/templates/input.config-hbase.json.j2")));
+ testServiceLog("hbase_regionserver", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HBASE/package/templates/input.config-hbase.json.j2")));
+// testServiceLog("hbase_phoenix_server", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HBASE/package/templates/input.config-hbase.json.j2")));
+ }
+
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java
new file mode 100644
index 0000000..981b0e2
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class HDFSLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testHDFSLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "HDFS", "configuration", "hdfs-log4j.xml").toString())).getLayout("RFA");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testHDFS() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "HDFS", "configuration", "hdfs-log4j.xml").toString())).getLayout("RFA");
+
+ testServiceLog("hdfs_namenode", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HDFS/package/templates/input.config-hdfs.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java
new file mode 100644
index 0000000..66efebe
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.log4j.PatternLayout;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HdfsAuditLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testHDFSAudit() throws Exception {
+ // given
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "HDFS", "configuration", "hdfs-log4j.xml").toString())).getLayout("RFAS");
+ listAppender.setLayout(new PatternLayout(layout));
+ listAppender.activateOptions();
+
+ // when
+ LOG.info("allowed=true\tugi=hdfs (auth:SIMPLE)\tip=/192.168.73.101\tcmd=getfileinfo\tsrc=/user\tdst=null\tperm=null\tproto=rpc");
+
+ // then
+ String logEntry = listAppender.getLogList().get(0);
+ Map<String, Object> result = testLogEntry(logEntry, "hdfs_audit", inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "HDFS/package/templates/input.config-hdfs.json.j2")));
+
+ assertAuditLog(result);
+ }
+
+ private void assertAuditLog(Map<String, Object> resultEntry) {
+ assertThat(resultEntry.isEmpty(), is(false));
+ assertThat(resultEntry.get("logType"), is("HDFSAudit"));
+ assertThat(resultEntry.get("cluster"), is(CLUSTER));
+ assertThat(resultEntry.get("dst"), is("null"));
+ assertThat(resultEntry.get("perm"), is("null"));
+ assertThat(resultEntry.get("event_count"), is(1));
+ assertThat(resultEntry.get("repo"), is("hdfs"));
+ assertThat(resultEntry.get("reqUser"), is("hdfs"));
+ assertThat(resultEntry.get("type"), is("hdfs_audit"));
+ assertThat(resultEntry.get("level"), is("INFO"));
+ assertThat(resultEntry.containsKey("seq_num"), is(true));
+ assertThat(LOG.getName().contains(resultEntry.get("logger_name").toString()), is(true));
+ assertThat(resultEntry.containsKey("id"), is(true));
+ assertThat(resultEntry.get("authType"), is("SIMPLE"));
+ assertThat(resultEntry.get("action"), is("getfileinfo"));
+ assertThat(resultEntry.containsKey("message_md5"), is(true));
+ assertThat(resultEntry.containsKey("event_md5"), is(true));
+ assertThat(resultEntry.containsKey("ip"), is(true));
+ assertThat(resultEntry.containsKey("host"), is(true));
+ Date logTime = (Date) resultEntry.get("evtTime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime.toLocalDate(), is(LocalDate.now()));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
new file mode 100644
index 0000000..7793485
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.File;
+
+import org.junit.Test;
+
+public class HiveLogPatterntIT extends PatternITBase {
+// TODO: use hdp_ambari_definitions
+ @Test
+ public void testHiveLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testHiveServer2() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA");
+ testServiceLog("hive_hiveserver2", layout, inputConfigTemplate(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2")));
+ }
+
+ @Test
+ public void testHiveMetastore() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA");
+ testServiceLog("hive_metastore", layout, inputConfigTemplate(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2")));
+ }
+}
+
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/JinjaFunctions.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/JinjaFunctions.java
new file mode 100644
index 0000000..db000b6
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/JinjaFunctions.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+public class JinjaFunctions {
+ public static Object defaultFunc(Object value, Object defaultValue) {
+ if (value == null)
+ return defaultValue;
+ return value;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java
new file mode 100644
index 0000000..aaf4e46
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class KafkaLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testKafkaRequestAppenderLayout() {
+ testKafkaAppenderLayout("requestAppender");
+ }
+
+ @Test
+ public void testKafkaControllerAppenderLayout() {
+ testKafkaAppenderLayout("controllerAppender");
+ }
+
+ @Test
+ public void testKafkaLogCleanerAppenderLayout() {
+ testKafkaAppenderLayout("cleanerAppender");
+ }
+
+ @Test
+ public void testKafkaStateChangeAppenderLayout() {
+ testKafkaAppenderLayout("stateChangeAppender");
+ }
+
+ @Test
+ public void testKafkaServerAppenderLayout() {
+ testKafkaAppenderLayout("kafkaAppender");
+ }
+
+ private void testKafkaAppenderLayout(String appender) {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "KAFKA", "configuration", "kafka-log4j.xml").toString())).getLayout(appender);
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testKafkaRequestAppender() throws Exception {
+ testKafka("requestAppender", "kafka_request");
+ }
+
+ @Test
+ public void testKafkaControllerAppender() throws Exception {
+ testKafka("controllerAppender", "kafka_controller");
+ }
+
+ @Test
+ public void testKafkaLogCleanerAppender() throws Exception {
+ testKafka("cleanerAppender", "kafka_logcleaner");
+ }
+
+ @Test
+ public void testKafkaStateChangeAppender() throws Exception {
+ testKafka("stateChangeAppender", "kafka_statechange");
+ }
+
+ @Test
+ public void testKafkaServerAppender() throws Exception {
+ testKafka("kafkaAppender", "kafka_server");
+ }
+
+ private void testKafka(String appender, String logId) throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "KAFKA", "configuration", "kafka-log4j.xml").toString())).getLayout(appender);
+
+ testServiceLog(logId, layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "KAFKA/package/templates/input.config-kafka.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java
new file mode 100644
index 0000000..baad36f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class KnoxLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testKnoxGatewayAppenderLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "KNOX", "configuration", "gateway-log4j.xml").toString())).getLayout("drfa");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testKnoxLdapAppenderLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "KNOX", "configuration", "ldap-log4j.xml").toString())).getLayout("drfa");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testKnoxGateway() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "KNOX", "configuration", "gateway-log4j.xml").toString())).getLayout("drfa");
+
+ testServiceLog("knox_gateway", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "KNOX/package/templates/input.config-knox.json.j2")));
+ }
+
+ @Test
+ public void testKnoxLdap() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "KNOX", "configuration", "ldap-log4j.xml").toString())).getLayout("drfa");
+
+ testServiceLog("knox_ldap", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "KNOX/package/templates/input.config-knox.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/LayoutQuery.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/LayoutQuery.java
new file mode 100644
index 0000000..0400a60
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/LayoutQuery.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+public interface LayoutQuery {
+ String query(String parameterName);
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ListAppender.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ListAppender.java
new file mode 100644
index 0000000..658b8f6
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ListAppender.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Layout;
+import org.apache.log4j.WriterAppender;
+import org.apache.log4j.spi.LoggingEvent;
+
+public class ListAppender extends AppenderSkeleton {
+
+ private final List<String> logList;
+
+ public ListAppender() {
+ logList = new ArrayList<>();
+ }
+
+ @Override
+ protected void append(LoggingEvent event) {
+ StringWriter stringWriter = new StringWriter();
+ WriterAppender writerAppender = new WriterAppender(layout, stringWriter);
+ writerAppender.append(event);
+ logList.add(stringWriter.toString());
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public boolean requiresLayout() {
+ return true;
+ }
+
+ public List<String> getLogList() {
+ return logList;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jContent.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jContent.java
new file mode 100644
index 0000000..fabe11e
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jContent.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.File;
+
+public interface Log4jContent {
+ String loadContent();
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jProperties.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jProperties.java
new file mode 100644
index 0000000..d031f8d
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jProperties.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.UncheckedIOException;
+import java.nio.charset.Charset;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+
+public class Log4jProperties {
+ public static Log4jProperties loadFrom(File file) {
+ return new Log4jProperties(() -> {
+ try {
+ return FileUtils.readFileToString(file, Charset.defaultCharset());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ public static Log4jProperties unwrapFrom(File file) {
+ return new Log4jProperties(new StackDefContent(file, "content"));
+ }
+
+ public static Log4jProperties unwrapFrom(File file, String propertyName) {
+ return new Log4jProperties(new StackDefContent(file, propertyName));
+ }
+
+ private final Log4jContent content;
+
+ public Log4jProperties(Log4jContent content) {
+ this.content = content;
+ }
+
+ public String getLayout(String appenderName) {
+ Properties properties = new Properties();
+ try (StringReader reader = new StringReader(content.loadContent())) {
+ properties.load(reader);
+ return properties.getProperty("log4j.appender." + appenderName + ".layout.ConversionPattern");
+ }
+ catch (IOException ex) {
+ throw new UncheckedIOException(ex);
+ }
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXml.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXml.java
new file mode 100644
index 0000000..4b0b9e2
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXml.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.apache.ambari.logsearch.patterns.StackDefContent.DOCUMENT_BUILDER_FACTORY;
+import static org.apache.ambari.logsearch.patterns.StackDefContent.X_PATH_FACTORY;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpression;
+
+import org.w3c.dom.Document;
+
+public class Log4jXml {
+ public static Log4jXml unwrapFrom(File file) {
+ return unwrapFrom(file, "content");
+ }
+
+ public static Log4jXml unwrapFrom(File file, String propertyName) {
+ return new Log4jXml(
+ new StackDefContent(file, propertyName),
+ (appenderName) -> "/configuration/appender[@name='" + appenderName + "']/layout/param[@name='ConversionPattern']/@value");
+ }
+
+ private final Log4jContent content;
+ private final LayoutQuery layoutQuery;
+
+ public Log4jXml(Log4jContent content, LayoutQuery layoutQuery) {
+ this.content = content;
+ this.layoutQuery = layoutQuery;
+ }
+
+ public String getLayout(String appenderName) {
+ return getLayout(content, layoutQuery, appenderName);
+ }
+
+ public static String getLayout(Log4jContent content, LayoutQuery layoutQuery, String parameterName) {
+ try {
+ DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder();
+ Document doc;
+ try (InputStream stringReader = new ByteArrayInputStream(content.loadContent().getBytes(Charset.defaultCharset()))) {
+ doc = builder.parse(stringReader);
+ }
+ XPath xpath = X_PATH_FACTORY.newXPath();
+ XPathExpression expr = xpath.compile(layoutQuery.query(parameterName));
+ return (String) expr.evaluate(doc, XPathConstants.STRING);
+ }
+ catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXmlProperties.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXmlProperties.java
new file mode 100644
index 0000000..ada5f2a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXmlProperties.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.File;
+
+public class Log4jXmlProperties {
+ public static Log4jXmlProperties unwrapFrom(File file) {
+ return unwrapFrom(file, "content");
+ }
+
+ public static Log4jXmlProperties unwrapFrom(File file, String contentPropertyName) {
+ return new Log4jXmlProperties(
+ new StackDefContent(file, contentPropertyName),
+ (xmlPropertyName) -> "/configuration/properties/property[@name='" + xmlPropertyName + "']/text()");
+ }
+
+ public Log4jXmlProperties(Log4jContent content, LayoutQuery layoutQuery) {
+ this.content = content;
+ this.layoutQuery = layoutQuery;
+ }
+
+ private final Log4jContent content;
+ private final LayoutQuery layoutQuery;
+
+ public String getLayout(String propertyName) {
+ return Log4jXml.getLayout(content, layoutQuery, propertyName);
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java
new file mode 100644
index 0000000..80483b1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class MetricsLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testMetricsLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "AMBARI_METRICS", "configuration", "ams-log4j.xml").toString())).getLayout("file");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testMetrics() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "AMBARI_METRICS", "configuration", "ams-log4j.xml").toString())).getLayout("file");
+
+ testServiceLog("ams_collector", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "AMBARI_METRICS/package/templates/input.config-ambari-metrics.json.j2")));
+// testServiceLog("ams_monitor", layout, Paths.get("AMBARI_METRICS", "package", "templates", "input.config-ambari-metrics.json.j2"));
+ }
+
+// @Test
+// public void testMetricsGrafana() throws Exception {
+// testServiceLog("ams_grafana", "%d{ISO8601} %-5p [%t] %c{2}: %m%n", Paths.get("AMBARI_METRICS", "package", "templates", "input.config-ambari-metrics.json.j2"));
+// }
+
+ @Test
+ public void testMetricsHBaseLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "AMBARI_METRICS", "configuration", "ams-hbase-log4j.xml").toString())).getLayout("DRFA");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testMetricsHBase() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "AMBARI_METRICS", "configuration", "ams-hbase-log4j.xml").toString())).getLayout("DRFA");
+
+ testServiceLog("ams_hbase_master", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "AMBARI_METRICS/package/templates/input.config-ambari-metrics.json.j2")));
+ testServiceLog("ams_hbase_regionserver", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "AMBARI_METRICS/package/templates/input.config-ambari-metrics.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
new file mode 100644
index 0000000..9558ed9
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.logfeeder.common.LogEntryParseTester;
+import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter;
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
+import com.hubspot.jinjava.Jinjava;
+import com.hubspot.jinjava.lib.fn.ELFunctionDefinition;
+
+public class PatternITBase {
+ protected final static Logger LOG = Logger.getLogger(PatternITBase.class);
+
+ public static final String HDP_AMBARI_DEFINITIONS_PATH = "/Users/kkasa/project/hdp_ambari_definitions/";
+ public static final File HDP_AMBARI_DEFINITIONS = new File(
+ isBlank(System.getProperty("hdp.ambari.definitions.path")) ? HDP_AMBARI_DEFINITIONS_PATH : System.getProperty("hdp.ambari.definitions.path"));
+ public static File AMBARI_STACK_DEFINITIONS;
+ public static File AMBARI_FOLDER;
+ public static final File HDP_SERVICES_FOLDER = new File(HDP_AMBARI_DEFINITIONS, Paths.get( "src", "main", "resources", "stacks", "HDP", "3.0", "services").toString());
+ public static final String CLUSTER = "cl1";
+ public static final String GLOBAL_CONFIG = "[\n" +
+ " {\n" +
+ " \"add_fields\": {\n" +
+ " \"cluster\": \""+ CLUSTER +"\"\n" +
+ " },\n" +
+ " \"source\": \"file\",\n" +
+ " \"tail\": \"true\",\n" +
+ " \"gen_event_md5\": \"true\"\n" +
+ " }\n" +
+ "]";
+
+ private Jinjava jinjava = new Jinjava();
+ protected ListAppender listAppender;
+
+
+ @BeforeClass
+ public static void setupGlobal() throws Exception {
+ URL location = PatternITBase.class.getProtectionDomain().getCodeSource().getLocation();
+
+ AMBARI_FOLDER = new File(new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent());
+ AMBARI_STACK_DEFINITIONS = new File(AMBARI_FOLDER, Paths.get("ambari-server", "src", "main", "resources", "common-services").toString());
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ JsonParser jsonParser = new JsonParser();
+ JsonElement globalConfigJsonElement = jsonParser.parse(GLOBAL_CONFIG);
+
+ InputAdapter.setGlobalConfigs(globalConfigJsonElement.getAsJsonArray());
+ jinjava.getGlobalContext().registerFunction(new ELFunctionDefinition("", "default", JinjaFunctions.class, "defaultFunc", Object.class, Object.class));
+
+ listAppender = new ListAppender();
+ LOG.addAppender(listAppender);
+ }
+
+ protected String inputConfigTemplate(File templateFile) throws IOException {
+ return FileUtils.readFileToString(templateFile, Charset.defaultCharset());
+ }
+
+ protected void testServiceLog(String logId, String log4jLayout, String inputConfigTemplate) throws Exception {
+ String logEntry = generateLogEntry(log4jLayout);
+ Map<String, Object> resultEntry = testLogEntry(logEntry, logId, inputConfigTemplate);
+ assertServiceLog(logId, resultEntry);
+ }
+
+ protected String generateLogEntry(String log4jLayout) {
+ return generateLogEntry(log4jLayout, "This is a test message");
+ }
+
+ protected String generateLogEntry(String log4jLayout, String message) {
+ listAppender.setLayout(new PatternLayout(log4jLayout));
+ listAppender.activateOptions();
+ LOG.error(message, new Exception("TEST"));
+ return listAppender.getLogList().get(0);
+ }
+
+ protected Map<String, Object> testLogEntry(String logEntry, String logId, String inputConfigTemplate) throws Exception {
+ String grokFilter = jinjava.render(inputConfigTemplate, new HashMap<>());
+
+ LogEntryParseTester tester = new LogEntryParseTester(logEntry, grokFilter, GLOBAL_CONFIG, logId);
+ return tester.parse();
+ }
+
+ private void assertServiceLog(String logId, Map<String, Object> resultEntry) {
+ assertThat(resultEntry.isEmpty(), is(false));
+ assertThat(resultEntry.get("cluster"), is(CLUSTER));
+ assertThat(resultEntry.get("level"), is("ERROR"));
+ assertThat(resultEntry.get("event_count"), is(1));
+ assertThat(resultEntry.get("type"), is(logId));
+ assertThat(resultEntry.containsKey("seq_num"), is(true));
+// assertThat(LOG.getName().contains(resultEntry.get("logger_name").toString()), is(true));
+ assertThat(resultEntry.containsKey("id"), is(true));
+ assertThat(resultEntry.containsKey("message_md5"), is(true));
+ assertThat(resultEntry.containsKey("event_md5"), is(true));
+ assertThat(resultEntry.containsKey("ip"), is(true));
+ assertThat(resultEntry.containsKey("host"), is(true));
+ assertThat(resultEntry.get("log_message").toString().contains("This is a test message"), is(true));
+ assertThat(resultEntry.get("log_message").toString().contains("java.lang.Exception: TEST"), is(true));
+ Date logTime = (Date) resultEntry.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime.toLocalDate(), is(LocalDate.now()));
+ }
+
+ protected void assertThatDateIsISO8601(String layout) {
+ assertThat(layout.toLowerCase().contains("%d{iso8601}"), is(true));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java
new file mode 100644
index 0000000..dc05611
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class RangerLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testRangerAdminLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "RANGER", "configuration", "admin-log4j.xml").toString())).getLayout("xa_log_appender");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testRangerUserSynchLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "RANGER", "configuration", "usersync-log4j.xml").toString())).getLayout("logFile");
+ assertThat(layout.contains("%d{dd MMM yyyy HH:mm:ss}"), is(true));
+ }
+
+ @Test
+ public void testRangerAdminLog() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "RANGER", "configuration", "admin-log4j.xml").toString())).getLayout("xa_log_appender");
+
+ testServiceLog("ranger_admin", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "RANGER/package/templates/input.config-ranger.json.j2")));
+ }
+
+ @Test
+ public void testRangerUserSynchLog() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "RANGER", "configuration", "usersync-log4j.xml").toString())).getLayout("logFile");
+
+ testServiceLog("ranger_usersync", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "RANGER/package/templates/input.config-ranger.json.j2")));
+ }
+
+ @Test
+ public void testRangerKMSLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "RANGER_KMS", "configuration", "kms-log4j.xml").toString())).getLayout("kms");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testRangerKMSLog() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "RANGER_KMS", "configuration", "kms-log4j.xml").toString())).getLayout("kms");
+
+ testServiceLog("ranger_kms", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "RANGER_KMS/package/templates/input.config-ranger-kms.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java
new file mode 100644
index 0000000..41b7ff9
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class Spark2LogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testSpark2LogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "SPARK2", "configuration", "spark2-log4j-properties.xml").toString())).getLayout("console");
+ assertThat(layout.contains("%d{yy/MM/dd HH:mm:ss}"), is(true));
+ }
+
+ @Test
+ public void testSpark2Livy2LogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "SPARK2", "configuration", "livy2-log4j-properties.xml").toString())).getLayout("console");
+ assertThat(layout.contains("%d{yy/MM/dd HH:mm:ss}"), is(true));
+ }
+
+ @Test
+ public void testSpark2Log() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "SPARK2", "configuration", "spark2-log4j-properties.xml").toString())).getLayout("console");
+
+ testServiceLog("spark2_jobhistory_server", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "SPARK2/package/templates/input.config-spark2.json.j2")));
+ }
+
+ @Test
+ public void testSpark2Livy2Log() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "SPARK2", "configuration", "livy2-log4j-properties.xml").toString())).getLayout("console");
+
+ testServiceLog("livy2_server", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "SPARK2/package/templates/input.config-spark2.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StackDefContent.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StackDefContent.java
new file mode 100644
index 0000000..5ae33f8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StackDefContent.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathFactory;
+
+import org.w3c.dom.Document;
+
+public class StackDefContent implements Log4jContent {
+ public static final XPathFactory X_PATH_FACTORY = XPathFactory.newInstance();
+ public static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance();
+
+ private final File file;
+ private final String propertyName;
+
+ public StackDefContent(File file, String propertyName) {
+ this.file = file;
+ this.propertyName = propertyName;
+ }
+
+ @Override
+ public String loadContent() {
+ try {
+ DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder();
+ Document doc;
+ try (FileInputStream fileInputStream = new FileInputStream(file)) {
+ doc = builder.parse(fileInputStream);
+ }
+ XPath xpath = X_PATH_FACTORY.newXPath();
+ XPathExpression expr = xpath.compile("/configuration/property[name/text()='" + propertyName + "']/value/text()");
+ return expr.evaluate(doc);
+ }
+ catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java
new file mode 100644
index 0000000..bf1b392
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class StormLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testStormClusterLogLayout() {
+ String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "STORM", "configuration", "storm-cluster-log4j.xml").toString())).getLayout("pattern");
+ assertThat(layout.contains("yyyy-MM-dd HH:mm:ss.SSS"), is(true));
+ }
+
+ @Test
+ public void testStormWorkerLogLayout() {
+ String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "STORM", "configuration", "storm-worker-log4j.xml").toString())).getLayout("pattern");
+ assertThat(layout.contains("yyyy-MM-dd HH:mm:ss.SSS"), is(true));
+ }
+
+ @Test
+ public void testStormLog() throws Exception {
+ String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "STORM", "configuration", "storm-cluster-log4j.xml").toString())).getLayout("pattern");
+
+ testServiceLog("storm_drpc", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "STORM/package/templates/input.config-storm.json.j2")));
+ }
+
+ @Test
+ public void testStormWorkerLog() throws Exception {
+ String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "STORM", "configuration", "storm-worker-log4j.xml").toString())).getLayout("pattern");
+
+ testServiceLog("storm_worker", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "STORM/package/templates/input.config-storm.json.j2")));
+ }
+
+ @Test
+ public void testStormWorkerLogEntry() throws Exception {
+ String logEntry = "2018-05-04 05:10:00.120 o.a.s.d.executor main [INFO] Loaded executor tasks count:[5 5]";
+ Map<String, Object> result = testLogEntry(logEntry, "storm_worker", inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "STORM/package/templates/input.config-storm.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("storm_worker"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Loaded executor tasks count:[5 5]"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime.toLocalDate(), is(LocalDate.now()));
+
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
new file mode 100644
index 0000000..a2d1069
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class YarnLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testYarnJobSummaryLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "YARN", "configuration", "yarn-log4j.xml").toString())).getLayout("RMSUMMARY");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testYarnJobSummaryLog() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "YARN", "configuration", "yarn-log4j.xml").toString())).getLayout("RMSUMMARY");
+
+ testServiceLog("yarn_jobsummary", layout,
+ inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2")));
+ }
+
+ @Test
+ public void testYarnNodemanagerLogEntry() throws Exception {
+ Map<String, Object> result = testLogEntry("2018-05-02 09:43:46,898 INFO zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:zookeeper.version=3.4.6-1173--1,\n" +
+ " built on 04/10/2018 11:42 GMT",
+ "yarn_nodemanager",
+ inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("yarn_nodemanager"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("logger_name"), is("zookeeper.ZooKeeper "));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Client environment:zookeeper.version=3.4.6-1173--1,\n built on 04/10/2018 11:42 GMT"));
+ assertThat(result.get("line_number"), is("100"));
+ assertThat(result.get("file"), is("Environment.java"));
+ assertThat(result.get("method"), is("logEnv"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 43, 46, 898000000)));
+ }
+
+ @Test
+ public void testYarnResourcemanagerLogEntry() throws Exception {
+ Map<String, Object> result = testLogEntry("2018-05-02 09:41:43,917 INFO placement.UserGroupMappingPlacementRule (UserGroupMappingPlacementRule.java:get(232)) - Initialized queue mappings, override: false",
+ "yarn_resourcemanager",
+ inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("yarn_resourcemanager"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("logger_name"), is("placement.UserGroupMappingPlacementRule "));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Initialized queue mappings, override: false"));
+ assertThat(result.get("line_number"), is("232"));
+ assertThat(result.get("file"), is("UserGroupMappingPlacementRule.java"));
+ assertThat(result.get("method"), is("get"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 41, 43, 917000000)));
+ }
+
+ @Test
+ public void testYarnTimelineServerLogEntry() throws Exception {
+ Map<String, Object> result = testLogEntry("2018-05-02 10:36:27,868 INFO timeline.RollingLevelDB (RollingLevelDB.java:evictOldDBs(345)) - Evicting entity-ldb DBs scheduled for eviction",
+ "yarn_timelineserver",
+ inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("yarn_timelineserver"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("logger_name"), is("timeline.RollingLevelDB "));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Evicting entity-ldb DBs scheduled for eviction"));
+ assertThat(result.get("line_number"), is("345"));
+ assertThat(result.get("file"), is("RollingLevelDB.java"));
+ assertThat(result.get("method"), is("evictOldDBs"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 36, 27, 868000000)));
+ }
+
+ @Test
+ public void testYarnHistoryServerLogEntry() throws Exception {
+ Map<String, Object> result = testLogEntry("2018-05-02 10:02:54,215 INFO webapp.View (HsJobsBlock.java:render(74)) - Getting list of all Jobs.",
+ "yarn_historyserver",
+ inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("yarn_historyserver"));
+ assertThat(result.get("logger_name"), is("webapp.View "));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Getting list of all Jobs."));
+ assertThat(result.get("line_number"), is("74"));
+ assertThat(result.get("file"), is("HsJobsBlock.java"));
+ assertThat(result.get("method"), is("render"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 02, 54, 215000000)));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java
new file mode 100644
index 0000000..6db6e7e
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class ZeppelinLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testZeppelinLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "ZEPPELIN", "configuration", "zeppelin-log4j-properties.xml").toString()),
+ "log4j_properties_content").getLayout("dailyfile");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testZeppelinLog() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "ZEPPELIN", "configuration", "zeppelin-log4j-properties.xml").toString()),
+ "log4j_properties_content").getLayout("dailyfile");
+
+ testServiceLog("zeppelin", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "ZEPPELIN/package/templates/input.config-zeppelin.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java
new file mode 100644
index 0000000..1e00a0f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class ZookeeperLogPatternIT extends PatternITBase {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(HDP_SERVICES_FOLDER.exists());
+ }
+
+ @Test
+ public void testZookeeperLogLayout() {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "ZOOKEEPER", "configuration", "zookeeper-log4j.xml").toString())).getLayout("ROLLINGFILE");
+ assertThatDateIsISO8601(layout);
+ }
+
+ @Test
+ public void testZookeeperLog() throws Exception {
+ String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
+ "ZOOKEEPER", "configuration", "zookeeper-log4j.xml").toString())).getLayout("ROLLINGFILE");
+
+ testServiceLog("zookeeper", layout, inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "ZOOKEEPER/package/templates/input.config-zookeeper.json.j2")));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
index b4c4f22..2a23cd7 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
@@ -187,10 +187,10 @@ public class ConfigHandler implements InputConfigMonitor {
}
for (FilterDescriptor filterDescriptor : inputConfig.getFilter()) {
- if ("grok".equals(filterDescriptor.getFilter())) {
- // Thus ensure that the log entry passed will be parsed immediately
- ((FilterGrokDescriptor)filterDescriptor).setMultilinePattern(null);
- }
+// if ("grok".equals(filterDescriptor.getFilter())) {
+// // Thus ensure that the log entry passed will be parsed immediately
+// ((FilterGrokDescriptor)filterDescriptor).setMultilinePattern(null);
+// }
filterConfigList.add(filterDescriptor);
}
loadInputs("test");
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
index 14b6f83..9e4edcc 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
@@ -137,7 +137,8 @@ public class LogEntryParseTester {
}
});
input.outputLine(logEntry, new InputFileMarker(input, null, 0));
-
+ input.outputLine(logEntry, new InputFileMarker(input, null, 0));
+
return result.isEmpty() ?
ImmutableMap.of("errorMessage", (Object)"Could not parse test log entry") :
result;
diff --git a/ambari-server/conf/unix/log4j.properties b/ambari-server/conf/unix/log4j.properties
index a5b58bc..dec1334 100644
--- a/ambari-server/conf/unix/log4j.properties
+++ b/ambari-server/conf/unix/log4j.properties
@@ -35,7 +35,7 @@ log4j.appender.file.File=${ambari.log.dir}/${ambari.log.file}
log4j.appender.file.MaxFileSize=80MB
log4j.appender.file.MaxBackupIndex=60
log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] %c{1}:%L - %m%n
# Log config changes
log4j.logger.configchange=INFO,configchange
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2
index 5b8f896..2b424b1 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2
@@ -57,7 +57,7 @@
},
"log4j_format":"[%d] %p %m (%c)%n",
"multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
- "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
"post_map_values":{
"logtime":{
"map_date":{
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
index d948ea4..9ec651c 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
@@ -108,13 +108,13 @@
}
},
- "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
- "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
- "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "log4j_format":"%d{ISO8601} %5p [%t] %c{1}:%L - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
"post_map_values":{
"logtime":{
"map_date":{
- "target_date_pattern":"dd MMM yyyy HH:mm:ss"
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
}
}
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2
index a2a4841..1767c28 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2
@@ -45,19 +45,40 @@
{
"type":"storm_worker",
"rowtype":"service",
- "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log"
+ "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/*/*/worker.log",
+ "init_default_fields":"true",
+ "cache_enabled":"true",
+ "cache_size":"100",
+ "cache_dedup_interval":"1000",
+ "cache_key_field":"log_message",
+ "cache_last_dedup_enabled":"true"
+ },
+ {
+ "type":"storm_worker_event",
+ "rowtype":"service",
+ "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/*/*/events.log",
+ "init_default_fields":"true",
+ "cache_enabled":"true",
+ "cache_size":"100",
+ "cache_dedup_interval":"1000",
+ "cache_key_field":"log_message",
+ "cache_last_dedup_enabled":"true",
+ "add_fields":{
+ "level":"INFO"
+ }
}
],
"filter":[
{
"filter":"grok",
+ "sort_order":1,
"conditions":{
"fields":{
"type":[
- "storm_drpc",
- "storm_logviewer",
"storm_nimbus",
"storm_supervisor",
+ "storm_logviewer",
+ "storm_drpc",
"storm_ui",
"storm_worker"
]
@@ -65,7 +86,7 @@
},
"log4j_format":"",
"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}\\s%{GREEDYDATA:thread_name}\\s\\[%{LOGLEVEL:level}\\]\\s%{GREEDYDATA:log_message}",
"post_map_values":{
"logtime":{
"map_date":{
@@ -73,6 +94,54 @@
}
}
}
+ },
+ {
+ "filter":"grok",
+ "sort_order":5,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker_event"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "message_pattern":"^%{TIMESTAMP_ISO8601:logtime}(!_DELIM_!<STREAMLINE_EVENT>!_DELIM_!%{DATA:sdi_streamline_component_name}!_DELIM_!%{DATA:sdi_streamline_event_id}!_DELIM_!%{DATA:sdi_streamline_root_ids}!_DELIM_!%{DATA:sdi_streamline_parent_ids}!_DELIM_!%{DATA:sdi_streamline_event_fields_and_values}!_DELIM_!%{DATA:sdi_streamline_event_headers}!_DELIM_!%{DATA:sdi_streamline_event_aux_fields_and_values})|(%{GREEDYDATA})",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "sort_order":6,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker_event"
+ ]
+ }
+ },
+ "source_field":"path",
+ "remove_source_field":"false",
+ "message_pattern":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/%{DATA:sdi_storm_topology_id}/%{DATA:sdi_storm_worker_port}/events\\.log"
+ },
+ {
+ "filter":"grok",
+ "sort_order":7,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker_event"
+ ]
+ }
+ },
+ "source_field":"sdi_storm_topology_id",
+ "remove_source_field":"false",
+ "message_pattern":"(streamline\\-%{DATA:sdi_streamline_topology_id}\\-%{DATA:sdi_streamline_topology_name}\\-[0-9]+\\-[0-9]+)|(%{DATA:sdi_storm_topology_id})"
}
]
-}
\ No newline at end of file
+}
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2
index 8034843..1bab72b 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2
@@ -15,14 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
-{
+ {
"input":[
{
"type":"mapred_historyserver",
"rowtype":"service",
"path":"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log"
}
- ],
+ ],
"filter":[
{
"filter":"grok",
@@ -31,8 +31,8 @@
"type":[
"mapred_historyserver"
]
- }
- },
+ }
+ },
"log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
@@ -41,8 +41,8 @@
"map_date":{
"target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
}
- }
- }
- }
- ]
-}
\ No newline at end of file
+ }
+ }
+ }
+ ]
+ }
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.