You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2021/04/07 12:40:28 UTC
[nifi] branch main updated: NIFI-7912 - Added properties to
configure DoSFilter timeout and whitelisted addresses
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9da3b1e NIFI-7912 - Added properties to configure DoSFilter timeout and whitelisted addresses
9da3b1e is described below
commit 9da3b1ec01555a5f36047f500b784edffd2f80b3
Author: Nathan Gough <th...@gmail.com>
AuthorDate: Thu Apr 1 17:16:11 2021 -0400
NIFI-7912 - Added properties to configure DoSFilter timeout and whitelisted addresses
- Added nifi.web.request.ip.whitelist property to set DoSFilter.ipWhitelist
- Added nifi.web.request.timeout property to set DoSFilter.maxRequestMs with default of 60 seconds
This closes #4972
Signed-off-by: David Handermann <ex...@apache.org>
---
.../java/org/apache/nifi/util/NiFiProperties.java | 11 +++++++
.../src/main/asciidoc/administration-guide.adoc | 2 ++
.../nifi-framework/nifi-resources/pom.xml | 2 ++
.../src/main/resources/conf/nifi.properties | 2 ++
.../org/apache/nifi/remote/StandardPublicPort.java | 4 +++
.../org/apache/nifi/web/server/JettyServer.java | 34 ++++++++++++++++++----
6 files changed, 49 insertions(+), 6 deletions(-)
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index 1bdf495..e9f91f9 100644
--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -215,6 +215,8 @@ public abstract class NiFiProperties {
public static final String WEB_PROXY_HOST = "nifi.web.proxy.host";
public static final String WEB_MAX_CONTENT_SIZE = "nifi.web.max.content.size";
public static final String WEB_MAX_REQUESTS_PER_SECOND = "nifi.web.max.requests.per.second";
+ public static final String WEB_REQUEST_TIMEOUT = "nifi.web.request.timeout";
+ public static final String WEB_REQUEST_IP_WHITELIST = "nifi.web.request.ip.whitelist";
public static final String WEB_SHOULD_SEND_SERVER_VERSION = "nifi.web.should.send.server.version";
// ui properties
@@ -303,6 +305,7 @@ public abstract class NiFiProperties {
public static final String DEFAULT_WEB_WORKING_DIR = "./work/jetty";
public static final String DEFAULT_WEB_MAX_CONTENT_SIZE = "20 MB";
public static final String DEFAULT_WEB_MAX_REQUESTS_PER_SECOND = "30000";
+ public static final String DEFAULT_WEB_REQUEST_TIMEOUT = "60 secs";
public static final String DEFAULT_NAR_WORKING_DIR = "./work/nar";
public static final String DEFAULT_COMPONENT_DOCS_DIRECTORY = "./work/docs/components";
public static final String DEFAULT_NAR_LIBRARY_DIR = "./lib";
@@ -672,6 +675,14 @@ public abstract class NiFiProperties {
return getProperty(WEB_MAX_REQUESTS_PER_SECOND, DEFAULT_WEB_MAX_REQUESTS_PER_SECOND);
}
+ public String getWebRequestTimeout() {
+ return getProperty(WEB_REQUEST_TIMEOUT, DEFAULT_WEB_REQUEST_TIMEOUT);
+ }
+
+ public String getWebRequestIpWhitelist() {
+ return getProperty(WEB_REQUEST_IP_WHITELIST);
+ }
+
public int getWebThreads() {
return getIntegerProperty(WEB_THREADS, DEFAULT_WEB_THREADS);
}
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 29285e6..6fc66a9 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -3481,6 +3481,8 @@ host[:port] that NiFi is bound to.
blank meaning all requests containing a proxy context path are rejected. Configuring this property would allow requests where the proxy path is contained in this listing.
|`nifi.web.max.content.size`|The maximum size (HTTP `Content-Length`) for PUT and POST requests. No default value is set for backward compatibility. Providing a value for this property enables the `Content-Length` filter on all incoming API requests (except Site-to-Site and cluster communications). A suggested value is `20 MB`.
|`nifi.web.max.requests.per.second`|The maximum number of requests from a connection per second. Requests in excess of this are first delayed, then throttled.
+|`nifi.web.request.ip.whitelist`|A comma separated list of IP addresses. Used to specify the IP addresses of clients which can exceed the maximum requests per second (`nifi.web.max.requests.per.second`). Does not apply to web request timeout.
+|`nifi.web.request.timeout`|The request timeout for web requests. Requests running longer than this time will be forced to end with a HTTP 503 Service Unavailable response. Default value is `60 secs`.
|====
[[security_properties]]
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
index 4ebbb31..0dc8dad 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
@@ -144,6 +144,8 @@
<nifi.web.proxy.host />
<nifi.web.max.content.size />
<nifi.web.max.requests.per.second>30000</nifi.web.max.requests.per.second>
+ <nifi.web.request.timeout>60 secs</nifi.web.request.timeout>
+ <nifi.web.request.ip.whitelist />
<nifi.web.should.send.server.version>true</nifi.web.should.send.server.version>
<!-- nifi.properties: security properties -->
<nifi.security.keystore />
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
index c28e5c9..1366daf 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
@@ -163,6 +163,8 @@ nifi.web.proxy.context.path=${nifi.web.proxy.context.path}
nifi.web.proxy.host=${nifi.web.proxy.host}
nifi.web.max.content.size=${nifi.web.max.content.size}
nifi.web.max.requests.per.second=${nifi.web.max.requests.per.second}
+nifi.web.request.timeout=${nifi.web.request.timeout}
+nifi.web.request.ip.whitelist=${nifi.web.request.ip.whitelist}
nifi.web.should.send.server.version=${nifi.web.should.send.server.version}
# security properties #
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java
index a74a99a..ec1b5b3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java
@@ -584,6 +584,10 @@ public class StandardPublicPort extends AbstractPort implements PublicPort {
} else {
throw new ProcessException(e);
}
+ } catch (final InterruptedException e) {
+ logger.error("The NiFi DoS filter has interrupted a long running session. If this is undesired, configure a longer web request timeout value in nifi.properties using '{}'",
+ NiFiProperties.WEB_REQUEST_TIMEOUT);
+ throw new ProcessException(e);
} catch (final Exception e) {
throw new ProcessException(e);
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
index d2c50dd..7dc1ffd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
@@ -691,8 +691,10 @@ public class JettyServer implements NiFiServer, ExtensionUiLoader {
*/
private static void addDenialOfServiceFilters(String path, WebAppContext webAppContext, NiFiProperties props) {
// Add the requests rate limiting filter to all requests
- int maxWebRequestsPerSecond = determineMaxWebRequestsPerSecond(props);
- addWebRequestRateLimitingFilter(path, webAppContext, maxWebRequestsPerSecond);
+ final int maxWebRequestsPerSecond = determineMaxWebRequestsPerSecond(props);
+ final long requestTimeoutInMilliseconds = determineRequestTimeoutInMilliseconds(props);
+ final String ipWhitelist = props.getWebRequestIpWhitelist();
+ addWebRequestLimitingFilter(path, webAppContext, maxWebRequestsPerSecond, ipWhitelist, requestTimeoutInMilliseconds);
// Only add the ContentLengthFilter if the property is explicitly set (empty by default)
int maxRequestSize = determineMaxRequestSize(props);
@@ -703,32 +705,52 @@ public class JettyServer implements NiFiServer, ExtensionUiLoader {
}
}
- private static int determineMaxWebRequestsPerSecond(NiFiProperties props) {
+ private static int determineMaxWebRequestsPerSecond(final NiFiProperties props) {
int defaultMaxRequestsPerSecond = Integer.parseInt(NiFiProperties.DEFAULT_WEB_MAX_REQUESTS_PER_SECOND);
int configuredMaxRequestsPerSecond = 0;
try {
configuredMaxRequestsPerSecond = Integer.parseInt(props.getMaxWebRequestsPerSecond());
} catch (final NumberFormatException e) {
- logger.warn("Exception parsing property " + NiFiProperties.WEB_MAX_REQUESTS_PER_SECOND + "; using default value: " + defaultMaxRequestsPerSecond);
+ logger.warn("Exception parsing property [{}]; using default value: [{}]", NiFiProperties.WEB_MAX_REQUESTS_PER_SECOND, defaultMaxRequestsPerSecond);
}
return configuredMaxRequestsPerSecond > 0 ? configuredMaxRequestsPerSecond : defaultMaxRequestsPerSecond;
}
+ private static long determineRequestTimeoutInMilliseconds(final NiFiProperties props) {
+ long defaultRequestTimeout = Math.round(FormatUtils.getPreciseTimeDuration(NiFiProperties.DEFAULT_WEB_REQUEST_TIMEOUT, TimeUnit.MILLISECONDS));
+ long configuredRequestTimeout = 0L;
+ try {
+ configuredRequestTimeout = Math.round(FormatUtils.getPreciseTimeDuration(props.getWebRequestTimeout(), TimeUnit.MILLISECONDS));
+ } catch (final NumberFormatException e) {
+ logger.warn("Exception parsing property [{}]; using default value: [{}]", NiFiProperties.WEB_REQUEST_TIMEOUT, defaultRequestTimeout);
+ }
+
+ return configuredRequestTimeout > 0 ? configuredRequestTimeout : defaultRequestTimeout;
+ }
+
/**
* Adds the {@link org.eclipse.jetty.servlets.DoSFilter} to the specified context and path. Limits incoming web requests to {@code maxWebRequestsPerSecond} per second.
+ * In order to allow clients to make more requests than the maximum rate, clients can be added to the {@code ipWhitelist}.
+ * The {@code requestTimeoutInMilliseconds} value limits requests to the given request timeout amount, and will close connections that run longer than this time.
*
* @param path the path to apply this filter
* @param webAppContext the context to apply this filter
* @param maxWebRequestsPerSecond the maximum number of allowed requests per second
+ * @param ipWhitelist a whitelist of IP addresses that should not be rate limited. Does not apply to request timeout
+ * @param requestTimeoutInMilliseconds the amount of time before a connection will be automatically closed
*/
- private static void addWebRequestRateLimitingFilter(String path, WebAppContext webAppContext, int maxWebRequestsPerSecond) {
+ private static void addWebRequestLimitingFilter(String path, WebAppContext webAppContext, int maxWebRequestsPerSecond, final String ipWhitelist, long requestTimeoutInMilliseconds) {
FilterHolder holder = new FilterHolder(DoSFilter.class);
holder.setInitParameters(new HashMap<String, String>() {{
put("maxRequestsPerSec", String.valueOf(maxWebRequestsPerSecond));
+ put("maxRequestMs", String.valueOf(requestTimeoutInMilliseconds));
+ put("ipWhitelist", String.valueOf(ipWhitelist));
}});
holder.setName(DoSFilter.class.getSimpleName());
- logger.debug("Adding DoSFilter to context at path: " + path + " with max req/sec: " + maxWebRequestsPerSecond);
+
+ logger.debug("Adding DoSFilter to context at path: [{}] with max req/sec: [{}], request timeout: [{}] ms. Whitelisted IPs not subject to filter: [{}]",
+ path, maxWebRequestsPerSecond, requestTimeoutInMilliseconds, StringUtils.defaultIfEmpty(ipWhitelist, "none"));
webAppContext.addFilter(holder, path, EnumSet.allOf(DispatcherType.class));
}