You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/05/11 16:32:26 UTC
[ambari] branch trunk updated: AMBARI-23825 - Log feeder fails to
parse date (#1248)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1988821 AMBARI-23825 - Log feeder fails to parse date (#1248)
1988821 is described below
commit 1988821d27c3050e24ad952c024acf8d6327c298
Author: kasakrisz <33...@users.noreply.github.com>
AuthorDate: Fri May 11 18:32:24 2018 +0200
AMBARI-23825 - Log feeder fails to parse date (#1248)
---
.../logsearch/patterns/AtlasLogPatternIT.java | 10 --
.../logsearch/patterns/HBaseLogPatternIT.java | 10 --
.../logsearch/patterns/HDFSLogPatternIT.java | 10 --
.../logsearch/patterns/HdfsAuditLogPatternIT.java | 9 --
.../logsearch/patterns/HiveLogPatterntIT.java | 91 +++++++++++--
.../logsearch/patterns/KafkaLogPatternIT.java | 10 --
.../logsearch/patterns/KnoxLogPatternIT.java | 10 --
.../logsearch/patterns/MetricsLogPatternIT.java | 10 --
.../ambari/logsearch/patterns/PatternITBase.java | 16 ++-
.../logsearch/patterns/RangerLogPatternIT.java | 9 --
.../logsearch/patterns/SmartSenseLogPatternIT.java | 145 +++++++++++++++++++++
.../logsearch/patterns/Spark2LogPatternIT.java | 9 --
.../logsearch/patterns/StormLogPatternIT.java | 14 +-
.../logsearch/patterns/YarnLogPatternIT.java | 12 +-
.../logsearch/patterns/ZeppelinLogPatternIT.java | 10 --
.../logsearch/patterns/ZookeeperLogPatternIT.java | 10 --
.../input.config-smartsense.json.j2 | 85 ++++++++++++
17 files changed, 325 insertions(+), 145 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java
index 7aca100..c1027d5 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class AtlasLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testAtlasLogLayout() {
String layout = Log4jXml.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java
index d8bee9e..27119fe 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class HBaseLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testHBaseLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java
index 981b0e2..9d17da0 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class HDFSLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testHDFSLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java
index 66efebe..3c3cd9f 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java
@@ -20,7 +20,6 @@ package org.apache.ambari.logsearch.patterns;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.nio.file.Paths;
@@ -31,18 +30,10 @@ import java.util.Date;
import java.util.Map;
import org.apache.log4j.PatternLayout;
-import org.junit.Before;
import org.junit.Test;
public class HdfsAuditLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testHDFSAudit() throws Exception {
// given
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
index 7793485..988c758 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
@@ -18,28 +18,99 @@
*/
package org.apache.ambari.logsearch.patterns;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
import java.io.File;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.Map;
+import org.hamcrest.MatcherAssert;
import org.junit.Test;
public class HiveLogPatterntIT extends PatternITBase {
-// TODO: use hdp_ambari_definitions
+
@Test
- public void testHiveLogLayout() {
- String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA");
- assertThatDateIsISO8601(layout);
+ public void testHiveServer2LogEntry() throws Exception {
+ String logEntry = "2018-05-11T07:46:01,087 WARN [main]: metastore.HiveMetaStoreClient (:()) - Failed to connect to the MetaStore Server...";
+ Map<String, Object> result = testLogEntry(logEntry,"hive_hiveserver2", inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "HIVE/package/templates/input.config-hive.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("WARN"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("hive_hiveserver2"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Failed to connect to the MetaStore Server..."));
+ assertThat(result.get("logger_name"), is("metastore.HiveMetaStoreClient "));
+ assertThat(result.get("host"), is("HW13201.local"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ MatcherAssert.assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 11, 7, 46, 1, 87000000)));
}
@Test
- public void testHiveServer2() throws Exception {
- String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA");
- testServiceLog("hive_hiveserver2", layout, inputConfigTemplate(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2")));
+ public void testHiveServer2InteractiveLogEntry() throws Exception {
+ String logEntry = "2018-05-11T08:48:02,973 WARN [main]: conf.HiveConf (HiveConf.java:initialize(5193)) - HiveConf of name hive.hook.proto.base-directory does not exist";
+ Map<String, Object> result = testLogEntry(logEntry,"hive_hiveserver2", inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "HIVE/package/templates/input.config-hive.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("WARN"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("hive_hiveserver2"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("HiveConf of name hive.hook.proto.base-directory does not exist"));
+ assertThat(result.get("logger_name"), is("conf.HiveConf "));
+ assertThat(result.get("host"), is("HW13201.local"));
+ assertThat(result.get("file"), is("HiveConf.java"));
+ assertThat(result.get("method"), is("initialize"));
+ assertThat(result.get("line_number"), is("5193"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ MatcherAssert.assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 11, 8, 48, 2, 973000000)));
}
@Test
- public void testHiveMetastore() throws Exception {
- String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA");
- testServiceLog("hive_metastore", layout, inputConfigTemplate(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2")));
+ public void testHiveMetastoreLogEntry() throws Exception {
+ String logEntry = "2018-05-11T09:13:14,706 INFO [pool-7-thread-6]: txn.TxnHandler (TxnHandler.java:performWriteSetGC(1588)) - Deleted 0 obsolete rows from WRTIE_SET";
+ Map<String, Object> result = testLogEntry(logEntry,"hive_metastore", inputConfigTemplate(
+ new File(HDP_SERVICES_FOLDER, "HIVE/package/templates/input.config-hive.json.j2")));
+
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("hive_metastore"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Deleted 0 obsolete rows from WRTIE_SET"));
+ assertThat(result.get("logger_name"), is("txn.TxnHandler "));
+ assertThat(result.get("host"), is("HW13201.local"));
+ assertThat(result.get("line_number"), is("1588"));
+ assertThat(result.get("file"), is("TxnHandler.java"));
+ assertThat(result.get("method"), is("performWriteSetGC"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ MatcherAssert.assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 11, 9, 13, 14, 706000000)));
}
}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java
index aaf4e46..bdba473 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class KafkaLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testKafkaRequestAppenderLayout() {
testKafkaAppenderLayout("requestAppender");
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java
index baad36f..c1c2cef 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class KnoxLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testKnoxGatewayAppenderLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java
index 80483b1..18b35ca 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class MetricsLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testMetricsLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
index 9558ed9..a4cc2c2 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
@@ -18,7 +18,7 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
import static org.junit.Assume.assumeTrue;
@@ -51,12 +51,10 @@ import com.hubspot.jinjava.lib.fn.ELFunctionDefinition;
public class PatternITBase {
protected final static Logger LOG = Logger.getLogger(PatternITBase.class);
- public static final String HDP_AMBARI_DEFINITIONS_PATH = "/Users/kkasa/project/hdp_ambari_definitions/";
- public static final File HDP_AMBARI_DEFINITIONS = new File(
- isBlank(System.getProperty("hdp.ambari.definitions.path")) ? HDP_AMBARI_DEFINITIONS_PATH : System.getProperty("hdp.ambari.definitions.path"));
+ public static File HDP_AMBARI_DEFINITIONS;
public static File AMBARI_STACK_DEFINITIONS;
public static File AMBARI_FOLDER;
- public static final File HDP_SERVICES_FOLDER = new File(HDP_AMBARI_DEFINITIONS, Paths.get( "src", "main", "resources", "stacks", "HDP", "3.0", "services").toString());
+ public static File HDP_SERVICES_FOLDER;
public static final String CLUSTER = "cl1";
public static final String GLOBAL_CONFIG = "[\n" +
" {\n" +
@@ -75,6 +73,14 @@ public class PatternITBase {
@BeforeClass
public static void setupGlobal() throws Exception {
+ String hdpAmbariDefinitionsPath = System.getProperty("hdp.ambari.definitions.path");
+ if (isNotBlank(hdpAmbariDefinitionsPath)) {
+ HDP_AMBARI_DEFINITIONS = new File(hdpAmbariDefinitionsPath);
+ HDP_SERVICES_FOLDER = new File(HDP_AMBARI_DEFINITIONS, Paths.get( "src", "main", "resources", "stacks", "HDP", "3.0", "services").toString());
+ }
+
+ assumeTrue(HDP_SERVICES_FOLDER != null && HDP_SERVICES_FOLDER.exists());
+
URL location = PatternITBase.class.getProtectionDomain().getCodeSource().getLocation();
AMBARI_FOLDER = new File(new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent());
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java
index dc05611..11d3a2a 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java
@@ -20,23 +20,14 @@ package org.apache.ambari.logsearch.patterns;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class RangerLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testRangerAdminLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/SmartSenseLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/SmartSenseLogPatternIT.java
new file mode 100644
index 0000000..8ca1294
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/SmartSenseLogPatternIT.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.patterns;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+public class SmartSenseLogPatternIT extends PatternITBase {
+
+ // TODO: read input config from hdp-ambari-definitions when available
+
+ @Test
+ public void testHSTServerLogEntry() throws Exception {
+ //given
+ String logEntry = "2018-05-02 09:40:14,740 INFO [main] SupportToolServer:143 - Starting HST Server.";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "hst_server", inputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("hst_server"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("logger_name"), is("SupportToolServer"));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Starting HST Server."));
+ assertThat(result.get("line_number"), is("143"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 40, 14, 740000000)));
+ }
+
+ private String inputConfigTemplate() throws IOException {
+ return IOUtils.toString(getClass().getClassLoader().getResourceAsStream("test-input-config/input.config-smartsense.json.j2"), Charset.defaultCharset());
+ }
+
+ @Test
+ public void testHSTAgentLogEntry() throws Exception {
+ // given
+ String logEntry = "INFO 2018-05-02 09:32:47,197 security.py:177 - Server certificate not exists, downloading";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "hst_agent", inputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("hst_agent"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("file"), is("security.py"));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Server certificate not exists, downloading"));
+ assertThat(result.get("line_number"), is("177"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 32, 47, 197000000)));
+ }
+
+ @Test
+ public void testActivityAnalyserLogEntry() throws Exception {
+ // given
+ String logEntry = "2018-05-02 10:23:49,592 INFO [main] ActivityUtil:410 - Could not find valid SmartSense ID. Will recheck every 5 minutes for next 5 minutes.";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "activity_analyser", inputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("activity_analyser"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("logger_name"), is("ActivityUtil"));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Could not find valid SmartSense ID. Will recheck every 5 minutes for next 5 minutes."));
+ assertThat(result.get("line_number"), is("410"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 23, 49, 592000000)));
+ }
+
+ @Test
+ public void testActivityExplorerLogEntry() throws Exception {
+ // given
+ String logEntry = "2018-05-02 09:44:26,883 INFO [main] FileSystemConfigStorage:74 - Creating filesystem: org.apache.hadoop.fs.RawLocalFileSystem";
+ // when
+ Map<String, Object> result = testLogEntry(logEntry, "activity_explorer", inputConfigTemplate());
+ // then
+ assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("cluster"), is(CLUSTER));
+ assertThat(result.get("level"), is("INFO"));
+ assertThat(result.get("event_count"), is(1));
+ assertThat(result.get("type"), is("activity_explorer"));
+ assertThat(result.containsKey("seq_num"), is(true));
+ assertThat(result.get("logger_name"), is("FileSystemConfigStorage"));
+ assertThat(result.containsKey("id"), is(true));
+ assertThat(result.containsKey("message_md5"), is(true));
+ assertThat(result.containsKey("event_md5"), is(true));
+ assertThat(result.containsKey("ip"), is(true));
+ assertThat(result.containsKey("host"), is(true));
+ assertThat(result.get("log_message"), is("Creating filesystem: org.apache.hadoop.fs.RawLocalFileSystem"));
+ assertThat(result.get("line_number"), is("74"));
+ Date logTime = (Date) result.get("logtime");
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 44, 26, 883000000)));
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java
index 41b7ff9..99fcc84 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java
@@ -20,23 +20,14 @@ package org.apache.ambari.logsearch.patterns;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class Spark2LogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testSpark2LogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java
index bf1b392..f1e65df 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java
@@ -20,28 +20,19 @@ package org.apache.ambari.logsearch.patterns;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.nio.file.Paths;
-import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Date;
import java.util.Map;
-import org.junit.Before;
+import org.hamcrest.MatcherAssert;
import org.junit.Test;
public class StormLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testStormClusterLogLayout() {
String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
@@ -94,7 +85,6 @@ public class StormLogPatternIT extends PatternITBase {
assertThat(result.get("log_message"), is("Loaded executor tasks count:[5 5]"));
Date logTime = (Date) result.get("logtime");
LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
- assertThat(localDateTime.toLocalDate(), is(LocalDate.now()));
-
+ MatcherAssert.assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 4, 5, 10, 0, 120000000)));
}
}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
index a2d1069..4fafd88 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
@@ -20,28 +20,18 @@ package org.apache.ambari.logsearch.patterns;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.nio.file.Paths;
-import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Date;
import java.util.Map;
-import org.junit.Before;
import org.junit.Test;
public class YarnLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testYarnJobSummaryLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
@@ -164,6 +154,6 @@ public class YarnLogPatternIT extends PatternITBase {
assertThat(result.get("method"), is("render"));
Date logTime = (Date) result.get("logtime");
LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault());
- assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 02, 54, 215000000)));
+ assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 2, 54, 215000000)));
}
}
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java
index 6db6e7e..d924ef9 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class ZeppelinLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testZeppelinLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java
index 1e00a0f..a68b7c4 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java
@@ -18,23 +18,13 @@
*/
package org.apache.ambari.logsearch.patterns;
-import static org.junit.Assume.assumeTrue;
-
import java.io.File;
import java.nio.file.Paths;
-import org.junit.Before;
import org.junit.Test;
public class ZookeeperLogPatternIT extends PatternITBase {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- assumeTrue(HDP_SERVICES_FOLDER.exists());
- }
-
@Test
public void testZookeeperLogLayout() {
String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get(
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-input-config/input.config-smartsense.json.j2 b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-input-config/input.config-smartsense.json.j2
new file mode 100644
index 0000000..d2037a5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-input-config/input.config-smartsense.json.j2
@@ -0,0 +1,85 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"hst_agent",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hst-log4j/hst.log.dir', '/var/log/hst')}}/hst-agent.log"
+ },
+ {
+ "type":"hst_server",
+ "rowtype":"service",
+ "path":"{{default('/configurations/hst-log4j/hst.log.dir', '/var/log/hst')}}/hst-server.log"
+ },
+ {
+ "type":"activity_analyser",
+ "rowtype":"service",
+ "path":"{{default('/configurations/activity-log4j/activity.log.dir', '/var/log/smartsense-activity')}}/activity-analyzer.log"
+ },
+ {
+ "type":"activity_explorer",
+ "rowtype":"service",
+ "path":"{{default('/configurations/activity-log4j/activity.log.dir', '/var/log/smartsense-activity')}}/activity-explorer.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hst_server",
+ "activity_analyser",
+ "activity_explorer"
+ ]
+ }
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hst_agent"
+ ]
+ }
+ },
+ "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+ "multiline_pattern":"^(%{LOGLEVEL:level}%{SPACE}%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{LOGLEVEL:level}%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.