You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2021/12/09 19:42:10 UTC
[nifi] branch main updated: NIFI-9382: This closes #5584. Added system test that replicates issue in which a closed shared classloader causes issues when used again NIFI-9382: Fixed issue with SharedInstanceClassLoader where the classloader may get closed but then get used again. When the SharedInstanceClassLoader is closed, we will now ensure that we don't use anymore and instead create a new one.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 97198e3 NIFI-9382: This closes #5584. Added system test that replicates issue in which a closed shared classloader causes issues when used again NIFI-9382: Fixed issue with SharedInstanceClassLoader where the classloader may get closed but then get used again. When the SharedInstanceClassLoader is closed, we will now ensure that we don't use anymore and instead create a new one.
97198e3 is described below
commit 97198e35a04c12e66684d9545ff24156d16c60f6
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Wed Dec 8 11:14:23 2021 -0500
NIFI-9382: This closes #5584. Added system test that replicates issue in which a closed shared classloader causes issues when used again
NIFI-9382: Fixed issue with SharedInstanceClassLoader where the classloader may get closed but then get used again. When the SharedInstanceClassLoader is closed, we will now ensure that we don't use anymore and instead create a new one.
Signed-off-by: Joe Witt <jo...@apache.org>
---
.../processors/hadoop/AbstractHadoopProcessor.java | 2 +-
.../service/StandardControllerServiceNode.java | 5 +-
.../org/apache/nifi/nar/InstanceClassLoader.java | 4 --
.../nifi/nar/NarThreadContextClassLoader.java | 1 +
.../apache/nifi/nar/SharedInstanceClassLoader.java | 9 +++-
.../nar/StandardExtensionDiscoveringManager.java | 8 ++--
.../tests/system/WriteFlowFileCountToFile.java | 30 ++++++++++--
.../classloaders/ClassloaderIsolationKeyIT.java | 55 ++++++++++++++++++++++
.../resources/conf/clustered/node1/logback.xml | 41 +++++++++++++---
.../resources/conf/clustered/node2/logback.xml | 45 ++++++++++++++----
.../src/test/resources/conf/default/logback.xml | 43 +++++++++++++++--
11 files changed, 208 insertions(+), 35 deletions(-)
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
index 222effd..7c574a7 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
@@ -205,7 +205,7 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor implemen
@Override
public String getClassloaderIsolationKey(final PropertyContext context) {
- final String explicitKerberosPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).getValue();
+ final String explicitKerberosPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
if (explicitKerberosPrincipal != null) {
return explicitKerberosPrincipal;
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
index bdae2b1..8593201 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
@@ -338,8 +338,9 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
@Override
public void verifyCanEnable() {
- if (getState() != ControllerServiceState.DISABLED) {
- throw new IllegalStateException(getControllerServiceImplementation().getIdentifier() + " cannot be enabled because it is not disabled");
+ final ControllerServiceState state = getState();
+ if (state != ControllerServiceState.DISABLED) {
+ throw new IllegalStateException(getControllerServiceImplementation().getIdentifier() + " cannot be enabled because it is not disabled - it has a state of " + state);
}
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
index e7153d7..b024bb5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
@@ -69,10 +69,6 @@ public class InstanceClassLoader extends AbstractNativeLibHandlingClassLoader {
this.instanceType = type;
this.instanceUrls = instanceUrls == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(instanceUrls));
this.additionalResourceUrls = additionalResourceUrls == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(additionalResourceUrls));
-
- if (parent instanceof SharedInstanceClassLoader) {
- ((SharedInstanceClassLoader) parent).incrementReferenceCount();
- }
}
@Override
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
index 949d207..915afa5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
@@ -266,6 +266,7 @@ public class NarThreadContextClassLoader extends URLClassLoader {
private static ClassLoader createClassLoader(final String implementationClassName, final String instanceId, final Bundle bundle, final ExtensionManager extensionManager)
throws ClassNotFoundException {
+
final ClassLoader bundleClassLoader = bundle.getClassLoader();
final Class<?> rawClass = Class.forName(implementationClassName, true, bundleClassLoader);
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
index 5aabef6..5ec1327 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
@@ -24,6 +24,7 @@ import java.util.Set;
public class SharedInstanceClassLoader extends InstanceClassLoader {
private long referenceCount = 0L;
+ private boolean closed = false;
public SharedInstanceClassLoader(final String identifier, final String type, final Set<URL> instanceUrls, final Set<URL> additionalResourceUrls,
final Set<File> narNativeLibDirs, final ClassLoader parent) {
@@ -35,11 +36,17 @@ public class SharedInstanceClassLoader extends InstanceClassLoader {
referenceCount--;
if (referenceCount <= 0) {
+ closed = true;
super.close();
}
}
- public synchronized void incrementReferenceCount() {
+ public synchronized boolean incrementReferenceCount() {
+ if (closed) {
+ return false;
+ }
+
referenceCount++;
+ return true;
}
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
index baa29b6..f8e1806 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
@@ -88,7 +88,7 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering
private final Map<String, ConfigurableComponent> tempComponentLookup = new HashMap<>();
private final Map<String, InstanceClassLoader> instanceClassloaderLookup = new ConcurrentHashMap<>();
- private final ConcurrentMap<BaseClassLoaderKey, ClassLoader> sharedBaseClassloaders = new ConcurrentHashMap<>();
+ private final ConcurrentMap<BaseClassLoaderKey, SharedInstanceClassLoader> sharedBaseClassloaders = new ConcurrentHashMap<>();
public StandardExtensionDiscoveringManager() {
this(Collections.emptyList());
@@ -407,8 +407,8 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering
if (requiresInstanceClassLoading.cloneAncestorResources()) {
// Check to see if there's already a shared ClassLoader that can be used as the parent/base classloader
if (baseClassLoaderKey != null) {
- final ClassLoader sharedBaseClassloader = sharedBaseClassloaders.get(baseClassLoaderKey);
- if (sharedBaseClassloader != null) {
+ final SharedInstanceClassLoader sharedBaseClassloader = sharedBaseClassloaders.get(baseClassLoaderKey);
+ if (sharedBaseClassloader != null && sharedBaseClassloader.incrementReferenceCount()) {
resolvedSharedClassLoader = true;
ancestorClassLoader = sharedBaseClassloader;
logger.debug("Creating InstanceClassLoader for type {} using shared Base ClassLoader {} for component {}", type, sharedBaseClassloader, instanceIdentifier);
@@ -444,6 +444,8 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering
// Created a shared class loader that is everything we need except for the additional URLs, as the additional URLs are instance-specific.
final SharedInstanceClassLoader sharedClassLoader = new SharedInstanceClassLoader(instanceIdentifier, classType, instanceUrls,
Collections.emptySet(), narNativeLibDirs, ancestorClassLoader);
+ sharedClassLoader.incrementReferenceCount();
+
instanceClassLoader = new InstanceClassLoader(instanceIdentifier, classType, Collections.emptySet(), additionalUrls, Collections.emptySet(), sharedClassLoader);
logger.debug("Creating InstanceClassLoader for type {} using newly created shared Base ClassLoader {} for component {}", type, sharedClassLoader, instanceIdentifier);
diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
index 7e78f0a..ef6eb75 100644
--- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
+++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
@@ -34,7 +34,7 @@ import java.io.FileOutputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
-import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -64,18 +64,31 @@ public class WriteFlowFileCountToFile extends AbstractProcessor implements Class
.defaultValue("counts.txt")
.build();
+ static final PropertyDescriptor CLASS_TO_CREATE = new Builder()
+ .name("Class to Create")
+ .displayName("Class to Create")
+ .description("If specified, each iteration of #onTrigger will create an instance of this class in order to test ClassLoader behavior. If unable to create the object, the FlowFile will be " +
+ "routed to failure")
+ .required(false)
+ .addValidator(NON_EMPTY_VALIDATOR)
+ .build();
+
private final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.build();
+ private final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .autoTerminateDefault(true)
+ .build();
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- return Arrays.asList(ISOLATION_KEY, FILE_TO_WRITE);
+ return Arrays.asList(ISOLATION_KEY, FILE_TO_WRITE, CLASS_TO_CREATE);
}
@Override
public Set<Relationship> getRelationships() {
- return Collections.singleton(REL_SUCCESS);
+ return new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE));
}
@Override
@@ -90,6 +103,17 @@ public class WriteFlowFileCountToFile extends AbstractProcessor implements Class
return;
}
+ final String className = context.getProperty(CLASS_TO_CREATE).getValue();
+ if (className != null) {
+ try {
+ Class.forName(className, true, Thread.currentThread().getContextClassLoader());
+ } catch (final ClassNotFoundException e) {
+ getLogger().error("Failed to load class {} for {}; routing to failure", className, flowFile);
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+ }
+
final long counterValue = counter.incrementAndGet();
final byte[] fileContents = String.valueOf(counterValue).getBytes(StandardCharsets.UTF_8);
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
index 0665ae8..4c5fb82 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
@@ -19,6 +19,7 @@ package org.apache.nifi.tests.system.classloaders;
import org.apache.nifi.tests.system.NiFiSystemIT;
import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException;
+import org.apache.nifi.web.api.entity.ConnectionEntity;
import org.apache.nifi.web.api.entity.ProcessorEntity;
import org.junit.Test;
@@ -28,8 +29,62 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Collections;
+import static org.junit.Assert.assertEquals;
+
public class ClassloaderIsolationKeyIT extends NiFiSystemIT {
+ /**
+ * After creating 1+ processors with the same ClassLoader Isolation Key, and then removing them,
+ * the SharedInstanceClassLoader will be closed. If we then create a new processor with the same
+ * ClassLoader Isolation Key, we need to ensure that we are then able to load classes from the ClassLoader
+ * that were not loaded previously.
+ */
+ @Test
+ public void testRemoveAllInstancesThenCreateForSameIsolationKeyAllowsClassLoading() throws NiFiClientException, IOException, InterruptedException {
+ final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile");
+ final ProcessorEntity counter = getClientUtil().createProcessor("WriteFlowFileCountToFile");
+ final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile");
+
+ getClientUtil().updateProcessorProperties(counter, Collections.singletonMap("File to Write", "count1.txt"));
+ getClientUtil().updateProcessorProperties(counter, Collections.singletonMap("Isolation Key", "abc123"));
+
+ getClientUtil().createConnection(generate, counter, "success");
+ final ConnectionEntity counterToTerminate = getClientUtil().createConnection(counter, terminate, "success");
+
+ getClientUtil().waitForValidProcessor(counter.getId());
+
+ getClientUtil().startProcessor(generate);
+ getClientUtil().startProcessor(counter);
+
+ waitForQueueCount(counterToTerminate.getId(), 1);
+
+ // Stop components, purge FlowFiles, delete all components
+ destroyFlow();
+
+ final ProcessorEntity newGenerate = getClientUtil().createProcessor("GenerateFlowFile");
+ final ProcessorEntity newCounter = getClientUtil().createProcessor("WriteFlowFileCountToFile");
+ final ProcessorEntity terminateSuccess = getClientUtil().createProcessor("TerminateFlowFile");
+ final ProcessorEntity terminateFailure = getClientUtil().createProcessor("TerminateFlowFile");
+
+ final ConnectionEntity generateToCounter = getClientUtil().createConnection(newGenerate, newCounter, "success");
+ final ConnectionEntity counterSuccess = getClientUtil().createConnection(newCounter, terminateSuccess, "success");
+ final ConnectionEntity counterFailure = getClientUtil().createConnection(newCounter, terminateFailure, "failure");
+
+ getClientUtil().updateProcessorProperties(newCounter, Collections.singletonMap("Class to Create", "org.apache.nifi.processors.tests.system.CountEvents"));
+ getClientUtil().updateProcessorProperties(newCounter, Collections.singletonMap("File to Write", "count1.txt"));
+ getClientUtil().updateProcessorProperties(newCounter, Collections.singletonMap("Isolation Key", "abc123"));
+
+ getClientUtil().waitForValidProcessor(newCounter.getId());
+
+ getClientUtil().startProcessor(newGenerate);
+ getClientUtil().startProcessor(newCounter);
+
+ waitForQueueCount(generateToCounter.getId(), 0);
+ assertEquals(0, getConnectionQueueSize(counterFailure.getId()));
+ assertEquals(1, getConnectionQueueSize(counterSuccess.getId()));
+ }
+
+
@Test
public void testClassloaderChanges() throws NiFiClientException, IOException, InterruptedException {
final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile");
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
index 6c3f0bb..c9bc033 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
@@ -57,6 +57,17 @@
</encoder>
</appender>
+ <appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -82,21 +93,22 @@
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
- <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/>
- <logger name="org.apache.nifi.web.util.ClusterReplicationComponentLifecycle" level="DEBUG" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
+ <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
@@ -128,6 +140,17 @@
<!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />
+ <!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default -->
+ <logger name="com.amazonaws.services.kinesis" level="WARN" />
+
+ <!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default -->
+ <logger name="org.apache.atlas" level="WARN"/>
+
+ <!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
+ <logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
+ <appender-ref ref="APP_FILE"/>
+ </logger>
+
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
@@ -148,10 +171,17 @@
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
- <logger name="org.apache.nifi.web.api" level="DEBUG" additivity="false">
+ <logger name="org.springframework.security.saml.log" level="WARN" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.opensaml" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
+ <!-- Web Server Request Log -->
+ <logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false">
+ <appender-ref ref="REQUEST_FILE"/>
+ </logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
@@ -174,9 +204,8 @@
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
-
<root level="INFO">
- <appender-ref ref="APP_FILE"/>
+ <appender-ref ref="APP_FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
index a9cdb26..c9bc033 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
@@ -57,6 +57,17 @@
</encoder>
</appender>
+ <appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -82,25 +93,22 @@
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
- <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/>
- <logger name="org.apache.nifi.web.util.ClusterReplicationComponentLifecycle" level="DEBUG" />
- <logger name="org.apache.nifi.groups.AbstractComponentScheduler" level="DEBUG" />
- <logger name="org.apache.nifi.controller.XmlFlowSynchronizer" level="DEBUG" />
- <logger name="org.apache.nifi.controller.inheritance" level="DEBUG" />
- <logger name="org.apache.nifi.controller.serialization.VersionedFlowSynchronizer" level="DEBUG" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
+ <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
@@ -132,6 +140,17 @@
<!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />
+ <!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default -->
+ <logger name="com.amazonaws.services.kinesis" level="WARN" />
+
+ <!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default -->
+ <logger name="org.apache.atlas" level="WARN"/>
+
+ <!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
+ <logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
+ <appender-ref ref="APP_FILE"/>
+ </logger>
+
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
@@ -152,10 +171,17 @@
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
- <logger name="org.apache.nifi.web.api" level="DEBUG" additivity="false">
+ <logger name="org.springframework.security.saml.log" level="WARN" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.opensaml" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
+ <!-- Web Server Request Log -->
+ <logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false">
+ <appender-ref ref="REQUEST_FILE"/>
+ </logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
@@ -178,9 +204,8 @@
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
-
<root level="INFO">
- <appender-ref ref="APP_FILE"/>
+ <appender-ref ref="APP_FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
index c42b3be..c9bc033 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
@@ -57,6 +57,17 @@
</encoder>
</appender>
+ <appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -82,20 +93,22 @@
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
- <logger name="org.apache.nifi.web.util.LocalComponentLifecycle" level="DEBUG" />
+
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
- <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/>
+
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
+ <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
@@ -127,6 +140,17 @@
<!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />
+ <!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default -->
+ <logger name="com.amazonaws.services.kinesis" level="WARN" />
+
+ <!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default -->
+ <logger name="org.apache.atlas" level="WARN"/>
+
+ <!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
+ <logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
+ <appender-ref ref="APP_FILE"/>
+ </logger>
+
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
@@ -147,7 +171,17 @@
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
+ <logger name="org.springframework.security.saml.log" level="WARN" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.opensaml" level="WARN" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <!-- Web Server Request Log -->
+ <logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false">
+ <appender-ref ref="REQUEST_FILE"/>
+ </logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
@@ -170,9 +204,8 @@
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
-
<root level="INFO">
- <appender-ref ref="APP_FILE"/>
+ <appender-ref ref="APP_FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>