You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2022/06/16 16:22:03 UTC
[camel] 02/02: CAMEL-18200: Sanitized uri should hide more sensitive keys.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit f084a88b89c1d4baf9963a27340789f8d7afe677
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Thu Jun 16 18:21:19 2022 +0200
CAMEL-18200: Sanitized uri should hide more sensitive keys.
---
.../apache/camel/support/DefaultEndpointTest.java | 4 +-
.../camel/management/ManagedSanitizeTest.java | 4 +-
.../camel/support/ScheduledPollConsumer.java | 2 +-
.../java/org/apache/camel/util/URISupport.java | 187 ++++++++++++---------
.../java/org/apache/camel/util/URISupportTest.java | 6 +-
.../ROOT/pages/camel-3x-upgrade-guide-3_18.adoc | 5 +
6 files changed, 117 insertions(+), 91 deletions(-)
diff --git a/core/camel-core/src/test/java/org/apache/camel/support/DefaultEndpointTest.java b/core/camel-core/src/test/java/org/apache/camel/support/DefaultEndpointTest.java
index fff873da549..18dbe7c3102 100644
--- a/core/camel-core/src/test/java/org/apache/camel/support/DefaultEndpointTest.java
+++ b/core/camel-core/src/test/java/org/apache/camel/support/DefaultEndpointTest.java
@@ -35,10 +35,10 @@ public class DefaultEndpointTest extends ContextTestSupport {
assertSanitizedUriUnchanged("irc://irc.codehaus.org/camel");
assertSanitizedUriUnchanged("direct:foo?bar=123&cheese=yes");
assertSanitizedUriUnchanged("https://issues.apache.org/activemq/secure/AddComment!default.jspa?id=33239");
- assertEquals("ftp://host.mysite.com/records?passiveMode=true&user=someuser&password=xxxxxx",
+ assertEquals("ftp://host.mysite.com/records?passiveMode=true&user=xxxxxx&password=xxxxxx",
URISupport.sanitizeUri("ftp://host.mysite.com/records?passiveMode=true&user=someuser&password=superSecret"));
assertEquals(
- "sftp://host.mysite.com/records?user=someuser&privateKeyFile=key.file&privateKeyFilePassphrase=xxxxxx&knownHostsFile=hosts.list",
+ "sftp://host.mysite.com/records?user=xxxxxx&privateKeyFile=xxxxxx&privateKeyFilePassphrase=xxxxxx&knownHostsFile=hosts.list",
URISupport.sanitizeUri(
"sftp://host.mysite.com/records?user=someuser&privateKeyFile=key.file&privateKeyFilePassphrase=superSecret&knownHostsFile=hosts.list"));
}
diff --git a/core/camel-management/src/test/java/org/apache/camel/management/ManagedSanitizeTest.java b/core/camel-management/src/test/java/org/apache/camel/management/ManagedSanitizeTest.java
index c4ac80c6525..b22693f3ea0 100644
--- a/core/camel-management/src/test/java/org/apache/camel/management/ManagedSanitizeTest.java
+++ b/core/camel-management/src/test/java/org/apache/camel/management/ManagedSanitizeTest.java
@@ -43,10 +43,10 @@ public class ManagedSanitizeTest extends ManagementTestSupport {
public void testSanitize() throws Exception {
MBeanServer mbeanServer = getMBeanServer();
- ObjectName name = getCamelObjectName(TYPE_ENDPOINT, "stub://foo\\?password=xxxxxx&username=foo");
+ ObjectName name = getCamelObjectName(TYPE_ENDPOINT, "stub://foo\\?password=xxxxxx&username=xxxxxx");
assertTrue(mbeanServer.isRegistered(name), "Should be registered");
String uri = (String) mbeanServer.getAttribute(name, "EndpointUri");
- assertEquals("stub://foo?password=xxxxxx&username=foo", uri);
+ assertEquals("stub://foo?password=xxxxxx&username=xxxxxx", uri);
}
@Override
diff --git a/core/camel-support/src/main/java/org/apache/camel/support/ScheduledPollConsumer.java b/core/camel-support/src/main/java/org/apache/camel/support/ScheduledPollConsumer.java
index 62cfc68041d..c12bfc67158 100644
--- a/core/camel-support/src/main/java/org/apache/camel/support/ScheduledPollConsumer.java
+++ b/core/camel-support/src/main/java/org/apache/camel/support/ScheduledPollConsumer.java
@@ -249,7 +249,7 @@ public abstract class ScheduledPollConsumer extends DefaultConsumer
// let exception handler deal with the caused exception
// but suppress this during shutdown as the logs may get flooded with exceptions during shutdown/forced shutdown
try {
- getExceptionHandler().handleException("Consumer " + this + " failed polling endpoint: " + getEndpoint()
+ getExceptionHandler().handleException("Failed polling endpoint: " + getEndpoint()
+ ". Will try again at next poll",
cause);
} catch (Throwable e) {
diff --git a/core/camel-util/src/main/java/org/apache/camel/util/URISupport.java b/core/camel-util/src/main/java/org/apache/camel/util/URISupport.java
index b6467642dc5..c72f15b4e8b 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/URISupport.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/URISupport.java
@@ -26,6 +26,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Pattern;
import static org.apache.camel.util.CamelURIParser.URI_ALREADY_NORMALIZED;
@@ -36,15 +37,13 @@ import static org.apache.camel.util.CamelURIParser.URI_ALREADY_NORMALIZED;
public final class URISupport {
public static final String RAW_TOKEN_PREFIX = "RAW";
- public static final char[] RAW_TOKEN_START = { '(', '{' };
- public static final char[] RAW_TOKEN_END = { ')', '}' };
+ public static final char[] RAW_TOKEN_START = {'(', '{'};
+ public static final char[] RAW_TOKEN_END = {')', '}'};
// Match any key-value pair in the URI query string whose key contains
// "passphrase" or "password" or secret key (case-insensitive).
// First capture group is the key, second is the value.
- private static final Pattern SECRETS = Pattern.compile(
- "([?&][^=]*(?:passphrase|password|secretKey|accessToken|clientSecret|authorizationToken|saslJaasConfig)[^=]*)=(RAW(([{][^}]*[}])|([(][^)]*[)]))|[^&]*)",
- Pattern.CASE_INSENSITIVE);
+ private static final Pattern ALL_SECRETS = createSecretsPattern(SensitiveUtils.getSensitiveKeys());
// Match the user password in the URI as second capture group
// (applies to URI with authority component and userinfo token in the form
@@ -65,16 +64,16 @@ public final class URISupport {
/**
* Removes detected sensitive information (such as passwords) from the URI and returns the result.
*
- * @param uri The uri to sanitize.
- * @see #SECRETS and #USERINFO_PASSWORD for the matched pattern
- * @return Returns null if the uri is null, otherwise the URI with the passphrase, password or secretKey
- * sanitized.
+ * @param uri The uri to sanitize.
+ * @return Returns null if the uri is null, otherwise the URI with the passphrase, password or secretKey
+ * sanitized.
+ * @see #SECRETS and #USERINFO_PASSWORD for the matched pattern
*/
public static String sanitizeUri(String uri) {
// use xxxxx as replacement as that works well with JMX also
String sanitized = uri;
if (uri != null) {
- sanitized = SECRETS.matcher(sanitized).replaceAll("$1=xxxxxx");
+ sanitized = ALL_SECRETS.matcher(sanitized).replaceAll("$1=xxxxxx");
sanitized = USERINFO_PASSWORD.matcher(sanitized).replaceFirst("$1xxxxxx$3");
}
return sanitized;
@@ -84,8 +83,8 @@ public final class URISupport {
* Removes detected sensitive information (such as passwords) from the <em>path part</em> of an URI (that is, the
* part without the query parameters or component prefix) and returns the result.
*
- * @param path the URI path to sanitize
- * @return null if the path is null, otherwise the sanitized path
+ * @param path the URI path to sanitize
+ * @return null if the path is null, otherwise the sanitized path
*/
public static String sanitizePath(String path) {
String sanitized = path;
@@ -98,9 +97,9 @@ public final class URISupport {
/**
* Extracts the scheme specific path from the URI that is used as the remainder option when creating endpoints.
*
- * @param u the URI
- * @param useRaw whether to force using raw values
- * @return the remainder path
+ * @param u the URI
+ * @param useRaw whether to force using raw values
+ * @return the remainder path
*/
public static String extractRemainderPath(URI u, boolean useRaw) {
String path = useRaw ? u.getRawSchemeSpecificPart() : u.getSchemeSpecificPart();
@@ -120,8 +119,8 @@ public final class URISupport {
/**
* Extracts the query part of the given uri
*
- * @param uri the uri
- * @return the query parameters or <tt>null</tt> if the uri has no query
+ * @param uri the uri
+ * @return the query parameters or <tt>null</tt> if the uri has no query
*/
public static String extractQuery(String uri) {
if (uri == null) {
@@ -138,8 +137,8 @@ public final class URISupport {
/**
* Strips the query parameters from the uri
*
- * @param uri the uri
- * @return the uri without the query parameter
+ * @param uri the uri
+ * @return the uri without the query parameter
*/
public static String stripQuery(String uri) {
int idx = uri.indexOf('?');
@@ -156,12 +155,12 @@ public final class URISupport {
* <tt>key=RAW(value)</tt> which tells Camel to not encode the value, and use the value as is (eg key=value) and the
* value has <b>not</b> been encoded.
*
- * @param uri the uri
- * @return the parameters, or an empty map if no parameters (eg never null)
+ * @param uri the uri
+ * @return the parameters, or an empty map if no parameters (eg never null)
* @throws URISyntaxException is thrown if uri has invalid syntax.
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
public static Map<String, Object> parseQuery(String uri) throws URISyntaxException {
return parseQuery(uri, false);
@@ -174,13 +173,13 @@ public final class URISupport {
* <tt>key=RAW(value)</tt> which tells Camel to not encode the value, and use the value as is (eg key=value) and the
* value has <b>not</b> been encoded.
*
- * @param uri the uri
- * @param useRaw whether to force using raw values
- * @return the parameters, or an empty map if no parameters (eg never null)
+ * @param uri the uri
+ * @param useRaw whether to force using raw values
+ * @return the parameters, or an empty map if no parameters (eg never null)
* @throws URISyntaxException is thrown if uri has invalid syntax.
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
public static Map<String, Object> parseQuery(String uri, boolean useRaw) throws URISyntaxException {
return parseQuery(uri, useRaw, false);
@@ -193,15 +192,15 @@ public final class URISupport {
* <tt>key=RAW(value)</tt> which tells Camel to not encode the value, and use the value as is (eg key=value) and the
* value has <b>not</b> been encoded.
*
- * @param uri the uri
- * @param useRaw whether to force using raw values
- * @param lenient whether to parse lenient and ignore trailing & markers which has no key or value which
- * can happen when using HTTP components
- * @return the parameters, or an empty map if no parameters (eg never null)
+ * @param uri the uri
+ * @param useRaw whether to force using raw values
+ * @param lenient whether to parse lenient and ignore trailing & markers which has no key or value which
+ * can happen when using HTTP components
+ * @return the parameters, or an empty map if no parameters (eg never null)
* @throws URISyntaxException is thrown if uri has invalid syntax.
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
public static Map<String, Object> parseQuery(String uri, boolean useRaw, boolean lenient) throws URISyntaxException {
if (uri == null || uri.isEmpty()) {
@@ -226,12 +225,12 @@ public final class URISupport {
* This is a companion method with {@link #isRaw(int, List)} and the returned value is supposed to be used as the
* parameter of that method.
*
- * @param str the string to scan RAW tokens
- * @return the list of pair indexes which represent the start and end positions of a RAW token
- * @see #isRaw(int, List)
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @param str the string to scan RAW tokens
+ * @return the list of pair indexes which represent the start and end positions of a RAW token
+ * @see #isRaw(int, List)
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
public static List<Pair<Integer>> scanRaw(String str) {
return URIScanner.scanRaw(str);
@@ -244,13 +243,13 @@ public final class URISupport {
* This is a companion method with {@link #scanRaw(String)} and is supposed to consume the returned value of that
* method as the second parameter <tt>pairs</tt>.
*
- * @param index the index to be tested
- * @param pairs the list of pair indexes which represent the start and end positions of a RAW token
- * @return <tt>true</tt> if the index is within any pair of the indexes, <tt>false</tt> otherwise
- * @see #scanRaw(String)
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @param index the index to be tested
+ * @param pairs the list of pair indexes which represent the start and end positions of a RAW token
+ * @return <tt>true</tt> if the index is within any pair of the indexes, <tt>false</tt> otherwise
+ * @see #scanRaw(String)
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
public static boolean isRaw(int index, List<Pair<Integer>> pairs) {
if (pairs == null || pairs.isEmpty()) {
@@ -271,8 +270,8 @@ public final class URISupport {
/**
* Parses the query parameters of the uri (eg the query part).
*
- * @param uri the uri
- * @return the parameters, or an empty map if no parameters (eg never null)
+ * @param uri the uri
+ * @return the parameters, or an empty map if no parameters (eg never null)
* @throws URISyntaxException is thrown if uri has invalid syntax.
*/
public static Map<String, Object> parseParameters(URI uri) throws URISyntaxException {
@@ -307,10 +306,10 @@ public final class URISupport {
* just the value.
*
* @param parameters the uri parameters
- * @see #parseQuery(String)
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @see #parseQuery(String)
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
@SuppressWarnings("unchecked")
public static void resolveRawParameterValues(Map<String, Object> parameters) {
@@ -347,9 +346,9 @@ public final class URISupport {
/**
* Creates a URI with the given query
*
- * @param uri the uri
- * @param query the query to append to the uri
- * @return uri with the query appended
+ * @param uri the uri
+ * @param query the query to append to the uri
+ * @return uri with the query appended
* @throws URISyntaxException is thrown if uri has invalid syntax.
*/
public static URI createURIWithQuery(URI uri, String query) throws URISyntaxException {
@@ -380,9 +379,9 @@ public final class URISupport {
* <p/>
* Returns the value as-is if not starting with the prefix.
*
- * @param value the value
- * @param prefix the prefix to remove from value
- * @return the value without the prefix
+ * @param value the value
+ * @param prefix the prefix to remove from value
+ * @return the value without the prefix
*/
public static String stripPrefix(String value, String prefix) {
if (value == null || prefix == null) {
@@ -401,9 +400,9 @@ public final class URISupport {
* <p/>
* Returns the value as-is if not ending with the prefix.
*
- * @param value the value
- * @param suffix the suffix to remove from value
- * @return the value without the suffix
+ * @param value the value
+ * @param suffix the suffix to remove from value
+ * @return the value without the suffix
*/
public static String stripSuffix(final String value, final String suffix) {
if (value == null || suffix == null) {
@@ -420,9 +419,9 @@ public final class URISupport {
/**
* Assembles a query from the given map.
*
- * @param options the map with the options (eg key/value pairs)
- * @return a query string with <tt>key1=value&key2=value2&...</tt>, or an empty string if there
- * is no options.
+ * @param options the map with the options (eg key/value pairs)
+ * @return a query string with <tt>key1=value&key2=value2&...</tt>, or an empty string if there
+ * is no options.
* @throws URISyntaxException is thrown if uri has invalid syntax.
*/
@SuppressWarnings("unchecked")
@@ -433,10 +432,10 @@ public final class URISupport {
/**
* Assembles a query from the given map.
*
- * @param options the map with the options (eg key/value pairs)
- * @param encode whether to URL encode the query string
- * @return a query string with <tt>key1=value&key2=value2&...</tt>, or an empty string if there
- * is no options.
+ * @param options the map with the options (eg key/value pairs)
+ * @param encode whether to URL encode the query string
+ * @return a query string with <tt>key1=value&key2=value2&...</tt>, or an empty string if there
+ * is no options.
* @throws URISyntaxException is thrown if uri has invalid syntax.
*/
@SuppressWarnings("unchecked")
@@ -464,7 +463,7 @@ public final class URISupport {
// values
if (value instanceof List) {
List<String> list = (List<String>) value;
- for (Iterator<String> it = list.iterator(); it.hasNext();) {
+ for (Iterator<String> it = list.iterator(); it.hasNext(); ) {
String s = it.next();
appendQueryStringParameter(key, s, rc, encode);
// append & separator if there is more in the list
@@ -538,9 +537,9 @@ public final class URISupport {
* It keeps the original parameters and if a new parameter is already defined in {@code originalURI}, it will be
* replaced by its value in {@code newParameters}.
*
- * @param originalURI the original URI
- * @param newParameters the parameters to add
- * @return the URI with all the parameters
+ * @param originalURI the original URI
+ * @param newParameters the parameters to add
+ * @return the URI with all the parameters
* @throws URISyntaxException is thrown if the uri syntax is invalid
* @throws UnsupportedEncodingException is thrown if encoding error
*/
@@ -560,13 +559,13 @@ public final class URISupport {
* <tt>key=RAW(value)</tt> which tells Camel to not encode the value, and use the value as is (eg key=value) and the
* value has <b>not</b> been encoded.
*
- * @param uri the uri
- * @return the normalized uri
+ * @param uri the uri
+ * @return the normalized uri
* @throws URISyntaxException in thrown if the uri syntax is invalid
* @throws UnsupportedEncodingException is thrown if encoding error
- * @see #RAW_TOKEN_PREFIX
- * @see #RAW_TOKEN_START
- * @see #RAW_TOKEN_END
+ * @see #RAW_TOKEN_PREFIX
+ * @see #RAW_TOKEN_START
+ * @see #RAW_TOKEN_END
*/
public static String normalizeUri(String uri) throws URISyntaxException, UnsupportedEncodingException {
// try to parse using the simpler and faster Camel URI parser
@@ -726,7 +725,7 @@ public final class URISupport {
public static Map<String, Object> extractProperties(Map<String, Object> properties, String optionPrefix) {
Map<String, Object> rc = new LinkedHashMap<>(properties.size());
- for (Iterator<Map.Entry<String, Object>> it = properties.entrySet().iterator(); it.hasNext();) {
+ for (Iterator<Map.Entry<String, Object>> it = properties.entrySet().iterator(); it.hasNext(); ) {
Map.Entry<String, Object> entry = it.next();
String name = entry.getKey();
if (name.startsWith(optionPrefix)) {
@@ -787,4 +786,26 @@ public final class URISupport {
return joined.toString();
}
+
+ private static Pattern createSecretsPattern(Set<String> keywords) {
+ StringBuilder regex = createOneOfThemRegex(keywords);
+ regex.insert(0, "([?&][^=]*(?:");
+ regex.append(")[^=]*)=(RAW(([{][^}]*[}])|([(][^)]*[)]))|[^&]*)");
+ return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE);
+ }
+
+ private static StringBuilder createOneOfThemRegex(Set<String> keywords) {
+ // from DefaultMaskingFormatter
+ StringBuilder regex = new StringBuilder();
+ String[] strKeywords = keywords.toArray(new String[0]);
+ regex.append(Pattern.quote(strKeywords[0]));
+ if (strKeywords.length > 1) {
+ for (int i = 1; i < strKeywords.length; i++) {
+ regex.append('|');
+ regex.append(Pattern.quote(strKeywords[i]));
+ }
+ }
+ return regex;
+ }
+
}
diff --git a/core/camel-util/src/test/java/org/apache/camel/util/URISupportTest.java b/core/camel-util/src/test/java/org/apache/camel/util/URISupportTest.java
index 92bc58f73de..ff9f59cc486 100644
--- a/core/camel-util/src/test/java/org/apache/camel/util/URISupportTest.java
+++ b/core/camel-util/src/test/java/org/apache/camel/util/URISupportTest.java
@@ -301,7 +301,7 @@ public class URISupportTest {
public void testSanitizeUriWithRawPassword() {
String uri1 = "http://foo?username=me&password=RAW(me#@123)&foo=bar";
String uri2 = "http://foo?username=me&password=RAW{me#@123}&foo=bar";
- String expected = "http://foo?username=me&password=xxxxxx&foo=bar";
+ String expected = "http://foo?username=xxxxxx&password=xxxxxx&foo=bar";
assertEquals(expected, URISupport.sanitizeUri(uri1));
assertEquals(expected, URISupport.sanitizeUri(uri2));
}
@@ -310,7 +310,7 @@ public class URISupportTest {
public void testSanitizeUriRawUnsafePassword() {
String uri1 = "sftp://localhost/target?password=RAW(beforeAmp&afterAmp)&username=jrandom";
String uri2 = "sftp://localhost/target?password=RAW{beforeAmp&afterAmp}&username=jrandom";
- String expected = "sftp://localhost/target?password=xxxxxx&username=jrandom";
+ String expected = "sftp://localhost/target?password=xxxxxx&username=xxxxxx";
assertEquals(expected, URISupport.sanitizeUri(uri1));
assertEquals(expected, URISupport.sanitizeUri(uri2));
}
@@ -322,7 +322,7 @@ public class URISupportTest {
String uriCurly
= "http://foo?username=me&password=RAW{me#@123}&foo=bar&port=21&tempFileName=${file:name.noext}.tmp&anotherOption=true";
String expected
- = "http://foo?username=me&password=xxxxxx&foo=bar&port=21&tempFileName=${file:name.noext}.tmp&anotherOption=true";
+ = "http://foo?username=xxxxxx&password=xxxxxx&foo=bar&port=21&tempFileName=${file:name.noext}.tmp&anotherOption=true";
assertEquals(expected, URISupport.sanitizeUri(uriPlain));
assertEquals(expected, URISupport.sanitizeUri(uriCurly));
}
diff --git a/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_18.adoc b/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_18.adoc
index a2b598f1b60..97c28a76d1b 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_18.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_18.adoc
@@ -6,6 +6,11 @@ from both 3.0 to 3.1 and 3.1 to 3.2.
== Upgrading Camel 3.17 to 3.18
+=== camel-core
+
+Camel will now mask all known secret values when logging endpoint URIs to avoid leaking sensitive details
+such as from stacktraces. Previously only a sub set of known _secret_ keys was masked.
+
=== camel-console
The `AbstractDevConsole` has changed the method `doCall` into two separate methods `doCallText` and `doCallJson`