You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by pk...@apache.org on 2024/08/23 13:01:12 UTC
(logging-log4j2) 01/01: Add support for concatenated plugin caches
This is an automated email from the ASF dual-hosted git repository.
pkarwasz pushed a commit to branch feature/2.x/plugin-cache-additivity
in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit 575d2cf5ed70b020d434e4e2b4a1c499614f1b7e
Author: Piotr P. Karwasz <pi...@karwasz.org>
AuthorDate: Thu Aug 8 22:49:18 2024 +0200
Add support for concatenated plugin caches
`Log4j2Plugins.dat` descriptors cause problems while shading
an application that uses Log4j Core, since they need a
specialized resource transformer to be merged.
We maintain such a resource transformer for the Maven Shade Plugin
and Gradle Up Shadow also contains a similar tool.
Both of them have open issues.
The remaining shading build tools, such as SBT Assembly,
however, don't have any resource transformer capable of dealing
with the `Log4j2Plugins.dat` format.
This PR extends the `Log4j2Plugins.dat` format to allow
it to contain `\n`-separated lists of serialized `PluginCache`
objects.
This will allow users to merge plugin caches using more standard
tools.
---
.../config/plugins/processor/PluginCacheTest.java | 70 ++++++++++++++++------
.../core/config/plugins/processor/PluginCache.java | 28 +++++++--
.../core/config/plugins/processor/PluginEntry.java | 18 ++++++
log4j-flume-ng/pom.xml | 5 --
src/changelog/.2.x.x/plugin-cache-additivity.xml | 7 +++
src/site/antora/modules/ROOT/pages/faq.adoc | 17 ++----
.../antora/modules/ROOT/pages/manual/plugins.adoc | 45 ++++++++++++++
7 files changed, 151 insertions(+), 39 deletions(-)
diff --git a/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java b/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java
index ffd89430c8..9fe8b8326c 100644
--- a/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java
+++ b/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java
@@ -16,29 +16,23 @@
*/
package org.apache.logging.log4j.core.config.plugins.processor;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
-import java.util.List;
import java.util.Map;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import java.util.TreeMap;
+import org.junit.jupiter.api.Test;
-@RunWith(JUnit4.class)
-public class PluginCacheTest {
+class PluginCacheTest {
@Test
- public void testOutputIsReproducibleWhenInputOrderingChanges() throws IOException {
- final PluginCache cacheA = new PluginCache();
- createCategory(cacheA, "one", Arrays.asList("bravo", "alpha", "charlie"));
- createCategory(cacheA, "two", Arrays.asList("alpha", "charlie", "bravo"));
- assertEquals(cacheA.getAllCategories().size(), 2);
- assertEquals(cacheA.getAllCategories().get("one").size(), 3);
- assertEquals(cacheA.getAllCategories().get("two").size(), 3);
+ void testOutputIsReproducibleWhenInputOrderingChanges() throws IOException {
+ final PluginCache cacheA = createSampleCache();
final PluginCache cacheB = new PluginCache();
createCategory(cacheB, "two", Arrays.asList("bravo", "alpha", "charlie"));
createCategory(cacheB, "one", Arrays.asList("alpha", "charlie", "bravo"));
@@ -48,9 +42,51 @@ public class PluginCacheTest {
assertArrayEquals(cacheData(cacheA), cacheData(cacheB));
}
- private void createCategory(final PluginCache cache, final String categoryName, final List<String> entryNames) {
+ @Test
+ void testDeserialize() throws IOException {
+ final PluginCache expected = createSampleCache();
+
+ final Map<String, Map<String, PluginEntry>> actual = new TreeMap<>();
+ PluginCache.loadInputStream(new ByteArrayInputStream(cacheData(expected)), actual);
+
+ assertThat(actual).as("Deserialized plugin cache").isEqualTo(expected.getAllCategories());
+ }
+
+ @Test
+ void testConcatenationOfCaches() throws IOException {
+ final ByteArrayOutputStream output = new ByteArrayOutputStream();
+ boolean first = true;
+ for (final String categoryName : Arrays.asList("one", "two")) {
+ final PluginCache cache = new PluginCache();
+ createCategory(cache, categoryName, Arrays.asList("alpha", "bravo", "charlie"));
+ cache.writeCache(output);
+ if (first) {
+ output.write('\n');
+ first = false;
+ }
+ }
+
+ final Map<String, Map<String, PluginEntry>> actual = new TreeMap<>();
+ PluginCache.loadInputStream(new ByteArrayInputStream(output.toByteArray()), actual);
+
+ assertThat(actual)
+ .as("Deserialized plugin cache")
+ .isEqualTo(createSampleCache().getAllCategories());
+ }
+
+ private PluginCache createSampleCache() {
+ final PluginCache cacheA = new PluginCache();
+ createCategory(cacheA, "one", Arrays.asList("alpha", "bravo", "charlie"));
+ createCategory(cacheA, "two", Arrays.asList("alpha", "bravo", "charlie"));
+ assertEquals(cacheA.getAllCategories().size(), 2);
+ assertEquals(cacheA.getAllCategories().get("one").size(), 3);
+ assertEquals(cacheA.getAllCategories().get("two").size(), 3);
+ return cacheA;
+ }
+
+ private void createCategory(final PluginCache cache, final String categoryName, final Iterable<String> entryNames) {
final Map<String, PluginEntry> category = cache.getCategory(categoryName);
- for (String entryName : entryNames) {
+ for (final String entryName : entryNames) {
final PluginEntry entry = new PluginEntry();
entry.setKey(entryName);
entry.setClassName("com.example.Plugin");
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java
index ab3f84a491..e1e72ceb86 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java
@@ -23,6 +23,7 @@ import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.Enumeration;
@@ -52,8 +53,12 @@ public class PluginCache {
* @return plugin mapping of names to plugin entries.
*/
public Map<String, PluginEntry> getCategory(final String category) {
- final String key = toRootLowerCase(category);
- return categories.computeIfAbsent(key, ignored -> new TreeMap<>());
+ return getCategory(category, categories);
+ }
+
+ private static Map<String, PluginEntry> getCategory(
+ final String category, final Map<String, Map<String, PluginEntry>> categories) {
+ return categories.computeIfAbsent(toRootLowerCase(category), ignored -> new TreeMap<>());
}
/**
@@ -94,11 +99,26 @@ public class PluginCache {
categories.clear();
while (resources.hasMoreElements()) {
final URL url = resources.nextElement();
- try (final DataInputStream in = new DataInputStream(new BufferedInputStream(url.openStream()))) {
+ loadInputStream(url.openStream(), categories);
+ }
+ }
+
+ static void loadInputStream(final InputStream input, final Map<String, Map<String, PluginEntry>> categories)
+ throws IOException {
+ try (final DataInputStream in = new DataInputStream(new BufferedInputStream(input))) {
+ // Allow concatenated plugin files separated by `\n`
+ boolean first = true;
+ while (in.available() > 0) {
+ if (first) {
+ first = false;
+ } else {
+ // Read the `\n` character
+ in.readByte();
+ }
final int count = in.readInt();
for (int i = 0; i < count; i++) {
final String category = in.readUTF();
- final Map<String, PluginEntry> m = getCategory(category);
+ final Map<String, PluginEntry> m = getCategory(category, categories);
final int entries = in.readInt();
for (int j = 0; j < entries; j++) {
// Must always read all parts of the entry, even if not adding, so that the stream progresses
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java
index 7f2b89d1a3..3d148945aa 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java
@@ -17,6 +17,7 @@
package org.apache.logging.log4j.core.config.plugins.processor;
import java.io.Serializable;
+import java.util.Objects;
/**
* Memento object for storing a plugin entry to a cache file.
@@ -79,6 +80,23 @@ public class PluginEntry implements Serializable {
this.category = category;
}
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ final PluginEntry that = (PluginEntry) o;
+ return printable == that.printable
+ && defer == that.defer
+ && Objects.equals(key, that.key)
+ && Objects.equals(className, that.className)
+ && Objects.equals(name, that.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, className, name, printable, defer);
+ }
+
@Override
public String toString() {
return "PluginEntry [key=" + key + ", className=" + className + ", name=" + name + ", printable=" + printable
diff --git a/log4j-flume-ng/pom.xml b/log4j-flume-ng/pom.xml
index 505f5969a6..d59e05a900 100644
--- a/log4j-flume-ng/pom.xml
+++ b/log4j-flume-ng/pom.xml
@@ -54,11 +54,6 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- <optional>true</optional>
- </dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-embedded-agent</artifactId>
diff --git a/src/changelog/.2.x.x/plugin-cache-additivity.xml b/src/changelog/.2.x.x/plugin-cache-additivity.xml
new file mode 100644
index 0000000000..f2daa46e94
--- /dev/null
+++ b/src/changelog/.2.x.x/plugin-cache-additivity.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<entry xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="https://logging.apache.org/xml/ns"
+ xsi:schemaLocation="https://logging.apache.org/xml/ns https://logging.apache.org/xml/ns/log4j-changelog-0.xsd"
+ type="added">
+ <description format="asciidoc">Add support for concatenated `Log4j2Plugins.dat` files.</description>
+</entry>
diff --git a/src/site/antora/modules/ROOT/pages/faq.adoc b/src/site/antora/modules/ROOT/pages/faq.adoc
index cb4fbd97e9..917bc58020 100644
--- a/src/site/antora/modules/ROOT/pages/faq.adoc
+++ b/src/site/antora/modules/ROOT/pages/faq.adoc
@@ -313,17 +313,8 @@ https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html[`ServiceL
You need to properly merge them by concatenating conflicting files.
`META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat`::
-These files contain Log4j plugin descriptors and need to be properly merged using the appropriate resource transformer for your shading plugin:
-
-https://maven.apache.org/plugins/maven-assembly-plugin/[Maven Assembly Plugin]:::
-https://github.com/sbt/sbt-assembly[SBT Assembly Plugin]:::
-We are not aware of any resource transformers capable of merging Log4j plugin descriptors.
-
-https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]:::
-You need to use the
-https://logging.staged.apache.org/log4j/transform/log4j-transform-maven-shade-plugin-extensions.html#log4j-plugin-cache-transformer[Log4j Plugin Descriptor Transformer].
-
-https://gradleup.com/shadow/[Gradle Shadow Plugin]:::
-You need to use the
-https://github.com/GradleUp/shadow/blob/main/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.groovy[`Log4j2PluginsCacheFileTransformer`].
+These files contain
+xref:manual/plugins.adoc[Log4j Plugin]
+descriptors.
+See xref:manual/plugins.adoc#plugin-merge[Merging plugin descriptors] for more details.
====
diff --git a/src/site/antora/modules/ROOT/pages/manual/plugins.adoc b/src/site/antora/modules/ROOT/pages/manual/plugins.adoc
index 1c33e97f32..bdf17d8bf4 100644
--- a/src/site/antora/modules/ROOT/pages/manual/plugins.adoc
+++ b/src/site/antora/modules/ROOT/pages/manual/plugins.adoc
@@ -281,3 +281,48 @@ See `@PluginElement("EventTemplateAdditionalField")` usage in {project-github-ur
link:../javadoc/log4j-core/org/apache/logging/log4j/core/config/plugins/util/PluginUtil.html[`PluginUtil`],
which is a convenient wrapper around <<#plugin-discovery,`PluginManager`>>, to discover and load plugins.
See {project-github-url}/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/TemplateResolverFactories.java[`TemplateResolverFactories.java`] for example usages.
+
+[#plugin-merge]
+== Merging plugin descriptors
+
+The plugin descriptor is located in a JAR file at the following fixed location:
+----
+META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat
+----
+
+Some deployment techniques like
+https://softwareengineering.stackexchange.com/questions/297276/what-is-a-shaded-java-dependency[shading]
+require multiple plugin descriptors to be merged into one.
+This can be done in multiple ways:
+
+* By using a resource transformer specialized in `Log4j2Plugins.dat` files.
+We are aware of the existence of the following resource transformers for different Java build tools:
+
+https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]:::
+You can use the
+https://logging.staged.apache.org/log4j/transform/log4j-transform-maven-shade-plugin-extensions.html#log4j-plugin-cache-transformer[Log4j Plugin Descriptor Transformer].
+
+https://gradleup.com/shadow/[Gradle Shadow Plugin]:::
+You can use the
+https://github.com/GradleUp/shadow/blob/main/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.groovy[`Log4j2PluginsCacheFileTransformer`].
+
+* Since version `2.24.0` you can any resource transformer capable of concatenating files using `\n` as separator.
+The following resource transformers are known to exist:
+
+https://github.com/sbt/sbt-assembly[SBT Assembly Plugin]:::
+You can use `MergeStrategy.concat`.
+See
+https://github.com/sbt/sbt-assembly?tab=readme-ov-file#merge-strategy[SBT Assembly Merge Strategy]
+for details.
+
+https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]:::
+You can use the standard `AppendingTransformer`.
+See
+https://maven.apache.org/plugins/maven-shade-plugin/examples/resource-transformers.html#AppendingTransformer[Maven Shade Plugin Resource Transformers]
+for details.
+
+https://gradleup.com/shadow/[Gradle Shadow Plugin]:::
+You can use the standard `AppendingTransformer`.
+See
+https://gradleup.com/shadow/configuration/merging/#appending-text-files[Appending Text Files]
+in the Gradle Shadow Plugin documentation for details.
\ No newline at end of file