You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by vy...@apache.org on 2020/06/24 06:08:41 UTC
[logging-log4j2] branch master updated (4f841bc -> 18750c1)
This is an automated email from the ASF dual-hosted git repository.
vy pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git.
from 4f841bc LOG4J2-2877 - Determine the container id to obtain container and image information
new c846b21 #335 Add fallbackKey to MessageResolver.
new 631c660 #335 Fix broken ITs due to recent ECS layout changes.
new 18750c1 #335 Allow non-String values in MapResolver.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../layout/json/template/resolver/MapResolver.java | 32 ++++--
.../json/template/resolver/MessageResolver.java | 116 +++++++++++++--------
.../log4j/layout/json/template/EcsLayoutTest.java | 2 +-
.../json/template/JsonTemplateLayoutTest.java | 91 ++++++++++++++++
.../log4j/layout/json/template/LogstashIT.java | 8 ++
src/site/asciidoc/manual/json-template-layout.adoc | 32 +++++-
6 files changed, 222 insertions(+), 59 deletions(-)
[logging-log4j2] 02/03: #335 Fix broken ITs due to recent ECS
layout changes.
Posted by vy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit 631c660d1b55e3f754d747e1baef772d432ca8cf
Author: Volkan Yazıcı <vo...@gmail.com>
AuthorDate: Wed Jun 24 08:04:54 2020 +0200
#335 Fix broken ITs due to recent ECS layout changes.
---
.../apache/logging/log4j/layout/json/template/EcsLayoutTest.java | 2 +-
.../org/apache/logging/log4j/layout/json/template/LogstashIT.java | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/EcsLayoutTest.java b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/EcsLayoutTest.java
index d6cbd30..8f5593d 100644
--- a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/EcsLayoutTest.java
+++ b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/EcsLayoutTest.java
@@ -20,7 +20,7 @@ public class EcsLayoutTest {
private static final String SERVICE_NAME = "test";
- private static final String EVENT_DATASET = "test.log";
+ private static final String EVENT_DATASET = SERVICE_NAME + ".log";
private static final JsonTemplateLayout JSON_TEMPLATE_LAYOUT = JsonTemplateLayout
.newBuilder()
diff --git a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/LogstashIT.java b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/LogstashIT.java
index 9c30be8..311b0df 100644
--- a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/LogstashIT.java
+++ b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/LogstashIT.java
@@ -63,6 +63,8 @@ public class LogstashIT {
private static final String SERVICE_NAME = "LogstashIT";
+ private static final String EVENT_DATASET = SERVICE_NAME + ".log";
+
private static final GelfLayout GELF_LAYOUT = GelfLayout
.newBuilder()
.setConfiguration(CONFIGURATION)
@@ -97,6 +99,7 @@ public class LogstashIT {
.setConfiguration(CONFIGURATION)
.setCharset(CHARSET)
.setServiceName(SERVICE_NAME)
+ .setEventDataset(EVENT_DATASET)
.build();
private static final JsonTemplateLayout JSON_TEMPLATE_ECS_LAYOUT = JsonTemplateLayout
@@ -114,6 +117,11 @@ public class LogstashIT {
.newBuilder()
.setKey("service.name")
.setValue(SERVICE_NAME)
+ .build(),
+ EventTemplateAdditionalField
+ .newBuilder()
+ .setKey("event.dataset")
+ .setValue(EVENT_DATASET)
.build()
})
.build())
[logging-log4j2] 01/03: #335 Add fallbackKey to MessageResolver.
Posted by vy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit c846b21dcbf9c98d2a96526c3be253a038e3a79e
Author: Volkan Yazıcı <vo...@gmail.com>
AuthorDate: Fri Jun 19 10:24:08 2020 +0200
#335 Add fallbackKey to MessageResolver.
---
.../json/template/resolver/MessageResolver.java | 116 +++++++++++++--------
.../json/template/JsonTemplateLayoutTest.java | 50 +++++++++
src/site/asciidoc/manual/json-template-layout.adoc | 23 +++-
3 files changed, 142 insertions(+), 47 deletions(-)
diff --git a/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MessageResolver.java b/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MessageResolver.java
index 54daefe..53dc7d9 100644
--- a/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MessageResolver.java
+++ b/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MessageResolver.java
@@ -18,6 +18,7 @@ package org.apache.logging.log4j.layout.json.template.resolver;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.layout.json.template.util.JsonWriter;
+import org.apache.logging.log4j.message.MapMessage;
import org.apache.logging.log4j.message.Message;
import org.apache.logging.log4j.message.MultiformatMessage;
import org.apache.logging.log4j.message.ObjectMessage;
@@ -30,8 +31,9 @@ import org.apache.logging.log4j.util.StringBuilderFormattable;
* <h3>Configuration</h3>
*
* <pre>
- * config = [ stringified ]
+ * config = [ stringified ] , [ fallbackKey ]
* stringified = "stringified" -> boolean
+ * fallbackKey = "fallbackKey" -> string
* </pre>
*
* <h3>Examples</h3>
@@ -54,6 +56,28 @@ import org.apache.logging.log4j.util.StringBuilderFormattable;
* "$resolver": "message"
* }
* </pre>
+ *
+ * Given the above configuration, a {@link SimpleMessage} will generate a
+ * <tt>"sample log message"</tt>, whereas a {@link MapMessage} will generate a
+ * <tt>{"action": "login", "sessionId": "87asd97a"}</tt>. Certain indexed log
+ * storage systems (e.g., <a
+ * href="https://www.elastic.co/elasticsearch/">Elasticsearch</a>) will not
+ * allow both values to coexist due to type mismatch: one is a <tt>string</tt>
+ * while the other is an <tt>object</tt>. Here one can use a
+ * <tt>fallbackKey</tt> to work around the problem:
+ *
+ * <pre>
+ * {
+ * "$resolver": "message",
+ * "fallbackKey": "formattedMessage"
+ * }
+ * </pre>
+ *
+ * Using this configuration, a {@link SimpleMessage} will generate a
+ * <tt>{"formattedMessage": "sample log message"}</tt> and a {@link MapMessage}
+ * will generate a <tt>{"action": "login", "sessionId": "87asd97a"}</tt>. Note
+ * that both emitted JSONs are of type <tt>object</tt> and have no
+ * type-conflicting fields.
*/
final class MessageResolver implements EventResolver {
@@ -72,9 +96,14 @@ final class MessageResolver implements EventResolver {
private static EventResolver createInternalResolver(
final TemplateResolverConfig config) {
final boolean stringified = config.getBoolean("stringified", false);
+ final String fallbackKey = config.getString("fallbackKey");
+ if (stringified && fallbackKey != null) {
+ throw new IllegalArgumentException(
+ "fallbackKey is not allowed when stringified is enable: " + config);
+ }
return stringified
- ? MessageResolver::resolveString
- : MessageResolver::resolveObject;
+ ? createStringResolver(fallbackKey)
+ : createObjectResolver(fallbackKey);
}
@Override
@@ -84,16 +113,27 @@ final class MessageResolver implements EventResolver {
internalResolver.resolve(logEvent, jsonWriter);
}
+ private static EventResolver createStringResolver(final String fallbackKey) {
+ return (final LogEvent logEvent, final JsonWriter jsonWriter) ->
+ resolveString(fallbackKey, logEvent, jsonWriter);
+ }
+
private static void resolveString(
+ final String fallbackKey,
final LogEvent logEvent,
final JsonWriter jsonWriter) {
final Message message = logEvent.getMessage();
- resolveString(message, jsonWriter);
+ resolveString(fallbackKey, message, jsonWriter);
}
private static void resolveString(
+ final String fallbackKey,
final Message message,
final JsonWriter jsonWriter) {
+ if (fallbackKey != null) {
+ jsonWriter.writeObjectStart();
+ jsonWriter.writeObjectKey(fallbackKey);
+ }
if (message instanceof StringBuilderFormattable) {
final StringBuilderFormattable formattable =
(StringBuilderFormattable) message;
@@ -102,48 +142,35 @@ final class MessageResolver implements EventResolver {
final String formattedMessage = message.getFormattedMessage();
jsonWriter.writeString(formattedMessage);
}
- }
-
- private static void resolveObject(
- final LogEvent logEvent,
- final JsonWriter jsonWriter) {
-
- // Try SimpleMessage serializer.
- final Message message = logEvent.getMessage();
- if (writeSimpleMessage(jsonWriter, message)) {
- return;
- }
-
- // Try MultiformatMessage serializer.
- if (writeMultiformatMessage(jsonWriter, message)) {
- return;
+ if (fallbackKey != null) {
+ jsonWriter.writeObjectEnd();
}
+ }
- // Try ObjectMessage serializer.
- if (writeObjectMessage(jsonWriter, message)) {
- return;
- }
+ private static EventResolver createObjectResolver(final String fallbackKey) {
+ return (final LogEvent logEvent, final JsonWriter jsonWriter) -> {
- // Fallback to plain Object write.
- resolveString(logEvent, jsonWriter);
+ // Skip custom serializers for SimpleMessage.
+ final Message message = logEvent.getMessage();
+ final boolean simple = message instanceof SimpleMessage;
+ if (!simple) {
- }
+ // Try MultiformatMessage serializer.
+ if (writeMultiformatMessage(jsonWriter, message)) {
+ return;
+ }
- private static boolean writeSimpleMessage(
- final JsonWriter jsonWriter,
- final Message message) {
+ // Try ObjectMessage serializer.
+ if (writeObjectMessage(jsonWriter, message)) {
+ return;
+ }
- // Check type.
- if (!(message instanceof SimpleMessage)) {
- return false;
- }
- final SimpleMessage simpleMessage = (SimpleMessage) message;
+ }
- // Write message.
- final String formattedMessage = simpleMessage.getFormattedMessage();
- jsonWriter.writeString(formattedMessage);
- return true;
+ // Fallback to plain String serializer.
+ resolveString(fallbackKey, logEvent, jsonWriter);
+ };
}
private static boolean writeMultiformatMessage(
@@ -165,16 +192,13 @@ final class MessageResolver implements EventResolver {
break;
}
}
-
- // Write the formatted JSON, if supported.
- if (jsonSupported) {
- final String messageJson = multiformatMessage.getFormattedMessage(FORMATS);
- jsonWriter.writeRawString(messageJson);
- return true;
+ if (!jsonSupported) {
+ return false;
}
- // Fallback to the default message formatter.
- resolveString((LogEvent) message, jsonWriter);
+ // Write the formatted JSON.
+ final String messageJson = multiformatMessage.getFormattedMessage(FORMATS);
+ jsonWriter.writeRawString(messageJson);
return true;
}
diff --git a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java
index 38ef58f..4419b58 100644
--- a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java
+++ b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java
@@ -38,6 +38,7 @@ import org.apache.logging.log4j.layout.json.template.JsonTemplateLayout.EventTem
import org.apache.logging.log4j.layout.json.template.util.JsonReader;
import org.apache.logging.log4j.layout.json.template.util.JsonWriter;
import org.apache.logging.log4j.layout.json.template.util.MapAccessor;
+import org.apache.logging.log4j.message.Message;
import org.apache.logging.log4j.message.ObjectMessage;
import org.apache.logging.log4j.message.SimpleMessage;
import org.apache.logging.log4j.message.StringMapMessage;
@@ -321,6 +322,55 @@ public class JsonTemplateLayoutTest {
}
@Test
+ public void test_message_fallbackKey() {
+
+ // Create the event template.
+ final String eventTemplate = writeJson(Map(
+ "message", Map(
+ "$resolver", "message",
+ "fallbackKey", "formattedMessage")));
+
+ // Create the layout.
+ final JsonTemplateLayout layout = JsonTemplateLayout
+ .newBuilder()
+ .setConfiguration(CONFIGURATION)
+ .setEventTemplate(eventTemplate)
+ .build();
+
+ // Create a log event with a MapMessage.
+ final Message mapMessage = new StringMapMessage()
+ .with("key1", "val1");
+ final LogEvent mapMessageLogEvent = Log4jLogEvent
+ .newBuilder()
+ .setLoggerName(LOGGER_NAME)
+ .setLevel(Level.INFO)
+ .setMessage(mapMessage)
+ .setTimeMillis(System.currentTimeMillis())
+ .build();
+
+ // Check the serialized MapMessage.
+ usingSerializedLogEventAccessor(layout, mapMessageLogEvent, accessor ->
+ assertThat(accessor.getString(new String[]{"message", "key1"}))
+ .isEqualTo("val1"));
+
+ // Create a log event with a SimpleMessage.
+ final Message simpleMessage = new SimpleMessage("simple");
+ final LogEvent simpleMessageLogEvent = Log4jLogEvent
+ .newBuilder()
+ .setLoggerName(LOGGER_NAME)
+ .setLevel(Level.INFO)
+ .setMessage(simpleMessage)
+ .setTimeMillis(System.currentTimeMillis())
+ .build();
+
+ // Check the serialized MapMessage.
+ usingSerializedLogEventAccessor(layout, simpleMessageLogEvent, accessor ->
+ assertThat(accessor.getString(new String[]{"message", "formattedMessage"}))
+ .isEqualTo("simple"));
+
+ }
+
+ @Test
public void test_property_injection() {
// Create the log event.
diff --git a/src/site/asciidoc/manual/json-template-layout.adoc b/src/site/asciidoc/manual/json-template-layout.adoc
index f3e22db..9b819d1 100644
--- a/src/site/asciidoc/manual/json-template-layout.adoc
+++ b/src/site/asciidoc/manual/json-template-layout.adoc
@@ -733,8 +733,9 @@ parent such that keys are prefixed with `_`:
a|
[source]
----
-config = [ stringified ]
+config = [ stringified ] , [ fallbackKey ]
stringified = "stringified" -> boolean
+fallbackKey = "fallbackKey" -> string
----
a| `logEvent.getMessage()`
| For simple string messages, the resolution is performed without allocations.
@@ -761,6 +762,26 @@ will be retained:
}
----
+Given the above configuration, a `SimpleMessage` will generate a `"sample log
+message"`, whereas a `MapMessage` will generate a `{"action": "login",
+"sessionId": "87asd97a"}`. Certain indexed log storage systems (e.g.,
+https://www.elastic.co/elasticsearch/[Elasticsearch]) will not allow both values
+to coexist due to type mismatch: one is a `string` while the other is an `object`.
+Here one can use a `fallbackKey` to work around the problem:
+
+[source,json]
+----
+{
+ "$resolver": "message",
+ "fallbackKey": "formattedMessage"
+}
+----
+
+Using this configuration, a `SimpleMessage` will generate a
+`{"formattedMessage": "sample log message"}` and a `MapMessage` will generate a
+`{"action": "login", "sessionId": "87asd97a"}`. Note that both emitted JSONs are
+of type `object` and have no type-conflicting fields.
+
| ndc
a|
[source]
[logging-log4j2] 03/03: #335 Allow non-String values in MapResolver.
Posted by vy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit 18750c1647421a83b6256b5df28447286c772e8b
Author: Volkan Yazıcı <vo...@gmail.com>
AuthorDate: Wed Jun 24 08:08:16 2020 +0200
#335 Allow non-String values in MapResolver.
---
.../layout/json/template/resolver/MapResolver.java | 32 ++++++++++++-----
.../json/template/JsonTemplateLayoutTest.java | 41 ++++++++++++++++++++++
src/site/asciidoc/manual/json-template-layout.adoc | 9 +++--
3 files changed, 71 insertions(+), 11 deletions(-)
diff --git a/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MapResolver.java b/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MapResolver.java
index 622d8f3..21d125c 100644
--- a/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MapResolver.java
+++ b/log4j-layout-json-template/src/main/java/org/apache/logging/log4j/layout/json/template/resolver/MapResolver.java
@@ -17,9 +17,10 @@
package org.apache.logging.log4j.layout.json.template.resolver;
import org.apache.logging.log4j.core.LogEvent;
-import org.apache.logging.log4j.core.lookup.MapLookup;
import org.apache.logging.log4j.layout.json.template.util.JsonWriter;
import org.apache.logging.log4j.message.MapMessage;
+import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.util.IndexedReadOnlyStringMap;
/**
* {@link MapMessage} field resolver.
@@ -27,7 +28,9 @@ import org.apache.logging.log4j.message.MapMessage;
* <h3>Configuration</h3>
*
* <pre>
- * config = "key" -> string
+ * config = key , [ stringified ]
+ * key = "key" -> string
+ * stringified = "stringified" -> boolean
* </pre>
*
* <h3>Examples</h3>
@@ -43,20 +46,20 @@ import org.apache.logging.log4j.message.MapMessage;
*/
final class MapResolver implements EventResolver {
- private static final MapLookup MAP_LOOKUP = new MapLookup();
-
private final String key;
+ private final boolean stringified;
+
static String getName() {
return "map";
}
MapResolver(final TemplateResolverConfig config) {
- final String key = config.getString("key");
+ this.key = config.getString("key");
+ this.stringified = config.getBoolean("stringified", false);
if (key == null) {
throw new IllegalArgumentException("missing key: " + config);
}
- this.key = key;
}
@Override
@@ -68,8 +71,21 @@ final class MapResolver implements EventResolver {
public void resolve(
final LogEvent logEvent,
final JsonWriter jsonWriter) {
- final String resolvedValue = MAP_LOOKUP.lookup(logEvent, key);
- jsonWriter.writeString(resolvedValue);
+ final Message message = logEvent.getMessage();
+ if (!(message instanceof MapMessage)) {
+ jsonWriter.writeNull();
+ } else {
+ @SuppressWarnings("unchecked")
+ MapMessage<?, Object> mapMessage = (MapMessage<?, Object>) message;
+ final IndexedReadOnlyStringMap map = mapMessage.getIndexedReadOnlyStringMap();
+ final Object value = map.getValue(key);
+ if (stringified) {
+ final String stringifiedValue = String.valueOf(value);
+ jsonWriter.writeString(stringifiedValue);
+ } else {
+ jsonWriter.writeValue(value);
+ }
+ }
}
}
diff --git a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java
index 4419b58..80cd2b9 100644
--- a/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java
+++ b/log4j-layout-json-template/src/test/java/org/apache/logging/log4j/layout/json/template/JsonTemplateLayoutTest.java
@@ -322,6 +322,47 @@ public class JsonTemplateLayoutTest {
}
@Test
+ public void test_MapMessage_keyed_access() {
+
+ // Create the event template.
+ final String key = "list";
+ final String eventTemplate = writeJson(Map(
+ "typedValue", Map(
+ "$resolver", "map",
+ "key", key),
+ "stringifiedValue", Map(
+ "$resolver", "map",
+ "key", key,
+ "stringified", true)));
+
+ // Create the layout.
+ final JsonTemplateLayout layout = JsonTemplateLayout
+ .newBuilder()
+ .setConfiguration(CONFIGURATION)
+ .setEventTemplate(eventTemplate)
+ .build();
+
+ // Create the log event with a MapMessage.
+ final List<Integer> value = Arrays.asList(1, 2);
+ final StringMapMessage mapMessage = new StringMapMessage()
+ .with(key, value);
+ final LogEvent logEvent = Log4jLogEvent
+ .newBuilder()
+ .setLoggerName(LOGGER_NAME)
+ .setLevel(Level.INFO)
+ .setMessage(mapMessage)
+ .setTimeMillis(System.currentTimeMillis())
+ .build();
+
+ // Check the serialized event.
+ usingSerializedLogEventAccessor(layout, logEvent, accessor -> {
+ assertThat(accessor.getObject("typedValue")).isEqualTo(value);
+ assertThat(accessor.getString("stringifiedValue")).isEqualTo(String.valueOf(value));
+ });
+
+ }
+
+ @Test
public void test_message_fallbackKey() {
// Create the event template.
diff --git a/src/site/asciidoc/manual/json-template-layout.adoc b/src/site/asciidoc/manual/json-template-layout.adoc
index 9b819d1..97a8898 100644
--- a/src/site/asciidoc/manual/json-template-layout.adoc
+++ b/src/site/asciidoc/manual/json-template-layout.adoc
@@ -593,10 +593,13 @@ Resolve the argument coming right after `--userId`:
a|
[source]
----
-config = "key" -> string
+config = key , [ stringified ]
+key = "key" -> string
+stringified = "stringified" -> boolean
----
-| performs link:lookups.html#MapLookup[Map Lookup] for the given `key`
-| none
+| resolves the given `key` of ``MapMessage``s
+| `stringified` flag translates to `String.valueOf(value)`, hence mind
+ not-`String`-typed values.
a|
Resolve the `userRole` field of the message: