You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2021/09/02 10:35:04 UTC
[camel] branch main updated (3fd547e -> e2109c6)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git.
from 3fd547e Sync deps
new d577d58 CAMEL-16895: Log masking optimized to detect if input is xml,json or key value and mask accordingly. Added unit test for user reporting issue.
new 5ee388f CAMEL-16908: Added apipassword as known secret key.
new 350d10e CAMEL-16908: log mask should use all known sensitive keys instead of a few hardcoded values.
new e2109c6 CAMEL-16895: Fix log mask json to not stackoverflow in regexp parsing for some JSon inputs.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../DefaultMaskingFormatterLongJsonTest.java | 24 +++----
.../processor/DefaultMaskingFormatterTest.java | 33 ++++++---
.../camel/support/processor/long-json-masked.json | 84 ++++++++++++++++++++++
.../apache/camel/support/processor/long-json.json | 84 ++++++++++++++++++++++
.../support/processor/DefaultMaskingFormatter.java | 50 +++++++++----
.../java/org/apache/camel/util/SensitiveUtils.java | 18 ++++-
.../maven/packaging/UpdateSensitizeHelper.java | 9 ++-
7 files changed, 264 insertions(+), 38 deletions(-)
copy tooling/camel-tooling-model/src/test/java/org/apache/camel/tooling/model/ComponentOptionModelTest.java => core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java (59%)
create mode 100644 core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json
create mode 100644 core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
[camel] 04/04: CAMEL-16895: Fix log mask json to not stackoverflow
in regexp parsing for some JSon inputs.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit e2109c62cac1f9592c951942ff9c38c0fb143e0a
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Thu Sep 2 12:34:25 2021 +0200
CAMEL-16895: Fix log mask json to not stackoverflow in regexp parsing for some JSon inputs.
---
.../DefaultMaskingFormatterLongJsonTest.java | 2 --
.../processor/DefaultMaskingFormatterTest.java | 16 ++++++++--------
.../apache/camel/support/processor/long-json.json | 2 +-
.../support/processor/DefaultMaskingFormatter.java | 22 +++++++++++++++++++---
4 files changed, 28 insertions(+), 14 deletions(-)
diff --git a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
index 2564a1e..f537a58 100644
--- a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
+++ b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
@@ -18,13 +18,11 @@ package org.apache.camel.support.processor;
import org.apache.camel.util.IOHelper;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
public class DefaultMaskingFormatterLongJsonTest {
@Test
- @Disabled("TODO: log mask should not use regexp as the pattern is flawed")
public void testMaskLongJson() throws Exception {
DefaultMaskingFormatter formatter = new DefaultMaskingFormatter();
diff --git a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java
index 1d91f78..d05ad7e 100644
--- a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java
+++ b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java
@@ -33,9 +33,9 @@ public class DefaultMaskingFormatterTest {
assertEquals("<xmlPassword>\n xxxxx \n</xmlPassword>\n<user password=\"xxxxx\"/>", answer);
answer = formatter.format(
- "{\"key\" : \"value\", \"My Password\":\"foo\", \"My SecretPassphrase\" : \"foo bar\", \"My SecretKey2\" : \"!@#$%^&*() -+[]{};:'\"}");
+ "{\"key\" : \"value\", \"Password\":\"foo\", \"Passphrase\" : \"foo bar\", \"SecretKey\" : \"!@#$%^&*() -+[]{};:'\"}");
assertEquals(
- "{\"key\" : \"value\", \"My Password\":\"xxxxx\", \"My SecretPassphrase\" : \"xxxxx\", \"My SecretKey2\" : \"xxxxx\"}",
+ "{\"key\" : \"value\", \"Password\":\"xxxxx\", \"Passphrase\" : \"xxxxx\", \"SecretKey\" : \"xxxxx\"}",
answer);
}
@@ -50,9 +50,9 @@ public class DefaultMaskingFormatterTest {
assertEquals("<xmlPassword>\n xxxxx \n</xmlPassword>\n<user password=\"asdf qwert\"/>", answer);
answer = formatter.format(
- "{\"key\" : \"value\", \"My Password\":\"foo\", \"My SecretPassphrase\" : \"foo bar\", \"My SecretKey2\" : \"!@#$%^&*() -+[]{};:'\"}");
+ "{\"key\" : \"value\", \"Password\":\"foo\", \"Passphrase\" : \"foo bar\", \"SecretKey\" : \"!@#$%^&*() -+[]{};:'\"}");
assertEquals(
- "{\"key\" : \"value\", \"My Password\":\"xxxxx\", \"My SecretPassphrase\" : \"xxxxx\", \"My SecretKey2\" : \"xxxxx\"}",
+ "{\"key\" : \"value\", \"Password\":\"xxxxx\", \"Passphrase\" : \"xxxxx\", \"SecretKey\" : \"xxxxx\"}",
answer);
}
@@ -67,9 +67,9 @@ public class DefaultMaskingFormatterTest {
assertEquals("<xmlPassword>\n foo bar \n</xmlPassword>\n<user password=\"xxxxx\"/>", answer);
answer = formatter.format(
- "{\"key\" : \"value\", \"My Password\":\"foo\", \"My SecretPassphrase\" : \"foo bar\", \"My SecretKey2\" : \"!@#$%^&*() -+[]{};:'\"}");
+ "{\"key\" : \"value\", \"Password\":\"foo\", \"Passphrase\" : \"foo bar\", \"SecretKey\" : \"!@#$%^&*() -+[]{};:'\"}");
assertEquals(
- "{\"key\" : \"value\", \"My Password\":\"xxxxx\", \"My SecretPassphrase\" : \"xxxxx\", \"My SecretKey2\" : \"xxxxx\"}",
+ "{\"key\" : \"value\", \"Password\":\"xxxxx\", \"Passphrase\" : \"xxxxx\", \"SecretKey\" : \"xxxxx\"}",
answer);
}
@@ -102,9 +102,9 @@ public class DefaultMaskingFormatterTest {
assertEquals("<xmlPassword>\n ********** \n</xmlPassword>\n<user password=\"**********\"/>", answer);
answer = formatter.format(
- "{\"key\" : \"value\", \"My Password\":\"foo\", \"My SecretPassphrase\" : \"foo bar\", \"My SecretKey2\" : \"!@#$%^&*() -+[]{};:'\"}");
+ "{\"key\" : \"value\", \"Password\":\"foo\", \"Passphrase\" : \"foo bar\", \"SecretKey\" : \"!@#$%^&*() -+[]{};:'\"}");
assertEquals(
- "{\"key\" : \"value\", \"My Password\":\"**********\", \"My SecretPassphrase\" : \"**********\", \"My SecretKey2\" : \"**********\"}",
+ "{\"key\" : \"value\", \"Password\":\"**********\", \"Passphrase\" : \"**********\", \"SecretKey\" : \"**********\"}",
answer);
}
diff --git a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
index 53f952f..008af2a 100644
--- a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
+++ b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
@@ -68,7 +68,7 @@
"sslCertificatePath": "",
"isTokenService": 0,
"wsazxcvgbgthyujAmount": 1070,
- "plsjdhfhPusherProcessorClass": "",
+ "plsjdhfhPusherProcessorClass": "abc",
"requestOriginIDs": [
"21"
],
diff --git a/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java b/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
index ab9f4ad..21b23ab 100644
--- a/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
+++ b/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
@@ -21,6 +21,8 @@ import java.util.regex.Pattern;
import org.apache.camel.spi.MaskingFormatter;
import org.apache.camel.util.SensitiveUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The {@link MaskingFormatter} that searches the specified keywords in the source and replace its value with mask
@@ -28,6 +30,8 @@ import org.apache.camel.util.SensitiveUtils;
*/
public class DefaultMaskingFormatter implements MaskingFormatter {
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultMaskingFormatter.class);
+
private Set<String> keywords;
private boolean maskKeyValue;
private boolean maskXmlElement;
@@ -70,7 +74,7 @@ public class DefaultMaskingFormatter implements MaskingFormatter {
answer = keyValueMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
}
} else if (json) {
- answer = jsonMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
+ answer = jsonMaskPattern.matcher(answer).replaceAll("\"$1\"$2:$3\"" + maskString + "\"");
} else if (maskKeyValue) {
// key=value paris
answer = keyValueMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
@@ -133,6 +137,10 @@ public class DefaultMaskingFormatter implements MaskingFormatter {
}
regex.insert(0, "([\\w]*(?:");
regex.append(")[\\w]*[\\s]*?=[\\s]*?)([\\S&&[^'\",\\}\\]\\)]]+[\\S&&[^,\\}\\]\\)>]]*?|\"[^\"]*?\"|'[^']*?')");
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("KeyValue Pattern: {}", regex);
+ }
return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE);
}
@@ -143,6 +151,10 @@ public class DefaultMaskingFormatter implements MaskingFormatter {
}
regex.insert(0, "(<([\\w]*(?:");
regex.append(")[\\w]*)(?:[\\s]+.+)*?>[\\s]*?)(?:[\\S&&[^<]]+(?:\\s+[\\S&&[^<]]+)*?)([\\s]*?</\\2>)");
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("XML Pattern: {}", regex);
+ }
return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE);
}
@@ -151,8 +163,12 @@ public class DefaultMaskingFormatter implements MaskingFormatter {
if (regex == null) {
return null;
}
- regex.insert(0, "(\"(?:[^\"]|(?:\\\"))*?(?:");
- regex.append(")(?:[^\"]|(?:\\\"))*?\"\\s*?\\:\\s*?)(?:\"(?:[^\"]|(?:\\\"))*?\")");
+ regex.insert(0, "\\\"(");
+ regex.append(")\\\"(\\s*?):(\\s*?)\\\"([^\"]*)\\\"");
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("JSon Pattern: {}", regex);
+ }
return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE);
}
[camel] 03/04: CAMEL-16908: log mask should use all known sensitive
keys instead of a few hardcoded values.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 350d10e60af9e805dc395e781d5fa0254369d161
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Thu Sep 2 08:47:11 2021 +0200
CAMEL-16908: log mask should use all known sensitive keys instead of a few hardcoded values.
---
.../processor/DefaultMaskingFormatterLongJsonTest.java | 2 ++
.../support/processor/DefaultMaskingFormatterTest.java | 17 +++++++++++++++++
.../camel/support/processor/long-json-masked.json | 4 ++--
.../org/apache/camel/support/processor/long-json.json | 2 +-
.../support/processor/DefaultMaskingFormatter.java | 9 +++------
.../main/java/org/apache/camel/util/SensitiveUtils.java | 2 ++
.../camel/maven/packaging/UpdateSensitizeHelper.java | 2 +-
7 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
index f537a58..2564a1e 100644
--- a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
+++ b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
@@ -18,11 +18,13 @@ package org.apache.camel.support.processor;
import org.apache.camel.util.IOHelper;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
public class DefaultMaskingFormatterLongJsonTest {
@Test
+ @Disabled("TODO: log mask should not use regexp as the pattern is flawed")
public void testMaskLongJson() throws Exception {
DefaultMaskingFormatter formatter = new DefaultMaskingFormatter();
diff --git a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java
index d8fa0a2..1d91f78 100644
--- a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java
+++ b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterTest.java
@@ -108,4 +108,21 @@ public class DefaultMaskingFormatterTest {
answer);
}
+ @Test
+ public void testDifferentSensitiveKeys() throws Exception {
+ DefaultMaskingFormatter formatter = new DefaultMaskingFormatter();
+ String answer
+ = formatter.format("key=value, myAccessKey=foo,\n authkey=\"foo bar\", refreshtoken='!@#$%^&*() -+[]{};:'");
+ assertEquals("key=value, myAccessKey=\"xxxxx\",\n authkey=\"xxxxx\", refreshtoken=\"xxxxx\"", answer);
+
+ answer = formatter.format("<subscribeKey>\n foo bar \n</subscribeKey>\n<user verificationCode=\"asdf qwert\"/>");
+ assertEquals("<subscribeKey>\n xxxxx \n</subscribeKey>\n<user verificationCode=\"xxxxx\"/>", answer);
+
+ answer = formatter.format(
+ "{\"key\" : \"value\", \"subscribeKey\":\"foo\", \"verificationCode\" : \"foo bar\", \"RefreshToken\" : \"!@#$%^&*() -+[]{};:'\"}");
+ assertEquals(
+ "{\"key\" : \"value\", \"subscribeKey\":\"xxxxx\", \"verificationCode\" : \"xxxxx\", \"RefreshToken\" : \"xxxxx\"}",
+ answer);
+ }
+
}
diff --git a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json
index 6f5798c..3645572 100644
--- a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json
+++ b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json
@@ -49,7 +49,7 @@
"invoiceNumber": "",
"erdfghytIsdgftgDate": "2021-08-25 22:45:20",
"lkjditgfJhsgruaCustomerName": "Customer",
- "apiUserName": "",
+ "apiUserName": "xxxxx",
"ojudhfgKaswsxsUUID": "1102967588",
"iuytredfHsdededAccountNumber": "4622785548",
"tyghfderrMode": [
@@ -68,7 +68,7 @@
"sslCertificatePath": "",
"isTokenService": 0,
"wsazxcvgbgthyujAmount": 1070,
- "plsjdhfhPusherProcessorClass": "",
+ "plsjdhfhPusherProcessorClass": "abc",
"requestOriginIDs": [
"21"
],
diff --git a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
index 4612724..53f952f 100644
--- a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
+++ b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
@@ -49,7 +49,7 @@
"invoiceNumber": "",
"erdfghytIsdgftgDate": "2021-08-25 22:45:20",
"lkjditgfJhsgruaCustomerName": "Customer",
- "apiUserName": "",
+ "apiUserName": "myuser",
"ojudhfgKaswsxsUUID": "1102967588",
"iuytredfHsdededAccountNumber": "4622785548",
"tyghfderrMode": [
diff --git a/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java b/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
index 3d84f70..ab9f4ad 100644
--- a/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
+++ b/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
@@ -16,12 +16,11 @@
*/
package org.apache.camel.support.processor;
-import java.util.Arrays;
-import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.camel.spi.MaskingFormatter;
+import org.apache.camel.util.SensitiveUtils;
/**
* The {@link MaskingFormatter} that searches the specified keywords in the source and replace its value with mask
@@ -29,8 +28,6 @@ import org.apache.camel.spi.MaskingFormatter;
*/
public class DefaultMaskingFormatter implements MaskingFormatter {
- private static final Set<String> DEFAULT_KEYWORDS
- = new HashSet<>(Arrays.asList("passphrase", "password", "secretKey"));
private Set<String> keywords;
private boolean maskKeyValue;
private boolean maskXmlElement;
@@ -41,11 +38,11 @@ public class DefaultMaskingFormatter implements MaskingFormatter {
private Pattern jsonMaskPattern;
public DefaultMaskingFormatter() {
- this(DEFAULT_KEYWORDS, true, true, true);
+ this(SensitiveUtils.getSensitiveKeys(), true, true, true);
}
public DefaultMaskingFormatter(boolean maskKeyValue, boolean maskXml, boolean maskJson) {
- this(DEFAULT_KEYWORDS, maskKeyValue, maskXml, maskJson);
+ this(SensitiveUtils.getSensitiveKeys(), maskKeyValue, maskXml, maskJson);
}
public DefaultMaskingFormatter(Set<String> keywords, boolean maskKeyValue, boolean maskXmlElement, boolean maskJson) {
diff --git a/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java b/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
index e5d4996..ef9a629 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
@@ -35,6 +35,8 @@ public final class SensitiveUtils {
"accountsid",
"acltoken",
"apipassword",
+ "apiuser",
+ "apiusername",
"authkey",
"authorizationtoken",
"blobaccesskey",
diff --git a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java
index 0b629e9..88020f4 100644
--- a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java
+++ b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java
@@ -53,7 +53,7 @@ public class UpdateSensitizeHelper extends AbstractGeneratorMojo {
// extra keys that are regarded as secret which may not yet been in any component
// they MUST be in lowercase and without a dash
- private static final String[] EXTRA_KEYS = new String[] { "apipassword" };
+ private static final String[] EXTRA_KEYS = new String[] { "apipassword", "apiuser", "apiusername" };
@Parameter(defaultValue = "${project.basedir}/src/generated/resources/org/apache/camel/catalog/")
protected File jsonDir;
[camel] 02/04: CAMEL-16908: Added apipassword as known secret key.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 5ee388fe44a90d09561cc083c117a2b3fcd07fb9
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Sep 1 09:36:44 2021 +0200
CAMEL-16908: Added apipassword as known secret key.
---
.../src/main/java/org/apache/camel/util/SensitiveUtils.java | 1 +
.../org/apache/camel/maven/packaging/UpdateSensitizeHelper.java | 9 ++++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java b/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
index 599359c..e5d4996 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
@@ -34,6 +34,7 @@ public final class SensitiveUtils {
"accountkey",
"accountsid",
"acltoken",
+ "apipassword",
"authkey",
"authorizationtoken",
"blobaccesskey",
diff --git a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java
index ab1b8ff..0b629e9 100644
--- a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java
+++ b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateSensitizeHelper.java
@@ -18,6 +18,7 @@ package org.apache.camel.maven.packaging;
import java.io.File;
import java.nio.file.Path;
+import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -50,6 +51,10 @@ public class UpdateSensitizeHelper extends AbstractGeneratorMojo {
private static final String START_TOKEN = "// SENSITIVE-KEYS: START";
private static final String END_TOKEN = "// SENSITIVE-KEYS: END";
+ // extra keys that are regarded as secret which may not yet been in any component
+ // they MUST be in lowercase and without a dash
+ private static final String[] EXTRA_KEYS = new String[] { "apipassword" };
+
@Parameter(defaultValue = "${project.basedir}/src/generated/resources/org/apache/camel/catalog/")
protected File jsonDir;
@@ -124,9 +129,11 @@ public class UpdateSensitizeHelper extends AbstractGeneratorMojo {
} catch (Exception e) {
throw new MojoExecutionException("Error loading json: " + name, e);
}
-
}
+ // add extra keys
+ secrets.addAll(Arrays.asList(EXTRA_KEYS));
+
getLog().info("There are " + secrets.size()
+ " distinct secret options across all the Camel components/dataformats/languages");
[camel] 01/04: CAMEL-16895: Log masking optimized to detect if
input is xml,
json or key value and mask accordingly. Added unit test for user reporting
issue.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit d577d5832d75337870a2bd1be450c89c81557c2c
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Sep 1 09:28:43 2021 +0200
CAMEL-16895: Log masking optimized to detect if input is xml,json or key value and mask accordingly. Added unit test for user reporting issue.
---
.../DefaultMaskingFormatterLongJsonTest.java | 37 ++++++++++
.../camel/support/processor/long-json-masked.json | 84 ++++++++++++++++++++++
.../apache/camel/support/processor/long-json.json | 84 ++++++++++++++++++++++
.../support/processor/DefaultMaskingFormatter.java | 23 ++++--
.../java/org/apache/camel/util/SensitiveUtils.java | 15 +++-
5 files changed, 233 insertions(+), 10 deletions(-)
diff --git a/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
new file mode 100644
index 0000000..f537a58
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/support/processor/DefaultMaskingFormatterLongJsonTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.support.processor;
+
+import org.apache.camel.util.IOHelper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class DefaultMaskingFormatterLongJsonTest {
+
+ @Test
+ public void testMaskLongJson() throws Exception {
+ DefaultMaskingFormatter formatter = new DefaultMaskingFormatter();
+
+ String in = IOHelper.loadText(DefaultMaskingFormatterLongJsonTest.class.getResourceAsStream("long-json.json"));
+ String out = formatter.format(in);
+
+ String expected
+ = IOHelper.loadText(DefaultMaskingFormatterLongJsonTest.class.getResourceAsStream("long-json-masked.json"));
+ Assertions.assertEquals(expected, out);
+ }
+
+}
diff --git a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json
new file mode 100644
index 0000000..6f5798c
--- /dev/null
+++ b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json-masked.json
@@ -0,0 +1,84 @@
+{
+ "qwjabrlanfssaPaymentID": 93974430,
+ "apiPortNumber": 0,
+ "pollerFrequency": 0,
+ "serviceCode": "QWERETEMKLJLANDD",
+ "extraData": {
+ "97280991": {
+ "pushToOriginator": true,
+ "uidID": 41189716,
+ "metaData": [
+ {
+ "Value": 1070,
+ "Name": "Amount"
+ },
+ {
+ "Value": "PHP82TK6MY",
+ "Name": "MpesaQwasdrtjknsNumber"
+ },
+ {
+ "Name": "Balance"
+ },
+ {
+ "Value": 20210825224512,
+ "Name": "TransactionDate"
+ },
+ {
+ "Value": 599726145860,
+ "Name": "PhoneNumber"
+ }
+ ],
+ "pokjytsAmountCharged": 1070,
+ "hardwareRequestId": null,
+ "ProductCode": "UJHANSNLWII",
+ "isTillNumber": true,
+ "uniqueID": "83859103",
+ "vouchercode": " ",
+ "paybillNumber": "431553",
+ "password": "xxxxx"
+ }
+ },
+ "oiusdftrPaymentStatus": 139,
+ "serviceUrl": "",
+ "lkjmnbsdcvfdertCurrencyCode": "YUM",
+ "wrapperUrl": "",
+ "wsdlFile": "",
+ "protocol": "default",
+ "sslEnabled": 0,
+ "numberOfSends": 0,
+ "invoiceNumber": "",
+ "erdfghytIsdgftgDate": "2021-08-25 22:45:20",
+ "lkjditgfJhsgruaCustomerName": "Customer",
+ "apiUserName": "",
+ "ojudhfgKaswsxsUUID": "1102967588",
+ "iuytredfHsdededAccountNumber": "4622785548",
+ "tyghfderrMode": [
+ "TUND_PUDF"
+ ],
+ "zxcvfgtrNsdfdfdMSISDN": "254726173860",
+ "apiFunctionName": "",
+ "kjhxcTransactionID": [
+ "PHP82TK6MY"
+ ],
+ "apiPassword": "xxxxx",
+ "kjhcvbnUUID": "1102967588",
+ "dfcxvbnClientCode": [
+ "SAFKE"
+ ],
+ "sslCertificatePath": "",
+ "isTokenService": 0,
+ "wsazxcvgbgthyujAmount": 1070,
+ "plsjdhfhPusherProcessorClass": "",
+ "requestOriginIDs": [
+ "21"
+ ],
+ "firstSend": "",
+ "maxNumberOfSends": 0,
+ "narration": "The service request is processed successfully.",
+ "hubCreationDate": "2021-08-25 22:45:20",
+ "allowsPolling": 1,
+ "serviceID": 3004,
+ "lastSend": "2021-08-25 19:45:20",
+ "nextSend": "2021-08-25 22:45:20",
+ "autoAcknowledgejfhtysidj": 0
+}
\ No newline at end of file
diff --git a/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
new file mode 100644
index 0000000..4612724
--- /dev/null
+++ b/core/camel-core/src/test/resources/org/apache/camel/support/processor/long-json.json
@@ -0,0 +1,84 @@
+{
+ "qwjabrlanfssaPaymentID": 93974430,
+ "apiPortNumber": 0,
+ "pollerFrequency": 0,
+ "serviceCode": "QWERETEMKLJLANDD",
+ "extraData": {
+ "97280991": {
+ "pushToOriginator": true,
+ "uidID": 41189716,
+ "metaData": [
+ {
+ "Value": 1070,
+ "Name": "Amount"
+ },
+ {
+ "Value": "PHP82TK6MY",
+ "Name": "MpesaQwasdrtjknsNumber"
+ },
+ {
+ "Name": "Balance"
+ },
+ {
+ "Value": 20210825224512,
+ "Name": "TransactionDate"
+ },
+ {
+ "Value": 599726145860,
+ "Name": "PhoneNumber"
+ }
+ ],
+ "pokjytsAmountCharged": 1070,
+ "hardwareRequestId": null,
+ "ProductCode": "UJHANSNLWII",
+ "isTillNumber": true,
+ "uniqueID": "83859103",
+ "vouchercode": " ",
+ "paybillNumber": "431553",
+ "password": "secret"
+ }
+ },
+ "oiusdftrPaymentStatus": 139,
+ "serviceUrl": "",
+ "lkjmnbsdcvfdertCurrencyCode": "YUM",
+ "wrapperUrl": "",
+ "wsdlFile": "",
+ "protocol": "default",
+ "sslEnabled": 0,
+ "numberOfSends": 0,
+ "invoiceNumber": "",
+ "erdfghytIsdgftgDate": "2021-08-25 22:45:20",
+ "lkjditgfJhsgruaCustomerName": "Customer",
+ "apiUserName": "",
+ "ojudhfgKaswsxsUUID": "1102967588",
+ "iuytredfHsdededAccountNumber": "4622785548",
+ "tyghfderrMode": [
+ "TUND_PUDF"
+ ],
+ "zxcvfgtrNsdfdfdMSISDN": "254726173860",
+ "apiFunctionName": "",
+ "kjhxcTransactionID": [
+ "PHP82TK6MY"
+ ],
+ "apiPassword": "mysecret",
+ "kjhcvbnUUID": "1102967588",
+ "dfcxvbnClientCode": [
+ "SAFKE"
+ ],
+ "sslCertificatePath": "",
+ "isTokenService": 0,
+ "wsazxcvgbgthyujAmount": 1070,
+ "plsjdhfhPusherProcessorClass": "",
+ "requestOriginIDs": [
+ "21"
+ ],
+ "firstSend": "",
+ "maxNumberOfSends": 0,
+ "narration": "The service request is processed successfully.",
+ "hubCreationDate": "2021-08-25 22:45:20",
+ "allowsPolling": 1,
+ "serviceID": 3004,
+ "lastSend": "2021-08-25 19:45:20",
+ "nextSend": "2021-08-25 22:45:20",
+ "autoAcknowledgejfhtysidj": 0
+}
\ No newline at end of file
diff --git a/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java b/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
index 516a7f6..3d84f70 100644
--- a/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
+++ b/core/camel-support/src/main/java/org/apache/camel/support/processor/DefaultMaskingFormatter.java
@@ -29,7 +29,8 @@ import org.apache.camel.spi.MaskingFormatter;
*/
public class DefaultMaskingFormatter implements MaskingFormatter {
- private static final Set<String> DEFAULT_KEYWORDS = new HashSet<>(Arrays.asList("passphrase", "password", "secretKey"));
+ private static final Set<String> DEFAULT_KEYWORDS
+ = new HashSet<>(Arrays.asList("passphrase", "password", "secretKey"));
private Set<String> keywords;
private boolean maskKeyValue;
private boolean maskXmlElement;
@@ -60,16 +61,24 @@ public class DefaultMaskingFormatter implements MaskingFormatter {
return source;
}
+ // xml,json or key=value pairs is the formats supported
+ boolean xml = maskXmlElement && source.startsWith("<");
+ boolean json = maskJson && !xml && (source.startsWith("{") || source.startsWith("["));
+
String answer = source;
- if (maskKeyValue) {
- answer = keyValueMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
- }
- if (maskXmlElement) {
+ if (xml) {
answer = xmlElementMaskPattern.matcher(answer).replaceAll("$1" + maskString + "$3");
- }
- if (maskJson) {
+ if (maskKeyValue) {
+ // used for the attributes in the XML tags
+ answer = keyValueMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
+ }
+ } else if (json) {
answer = jsonMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
+ } else if (maskKeyValue) {
+ // key=value paris
+ answer = keyValueMaskPattern.matcher(answer).replaceAll("$1\"" + maskString + "\"");
}
+
return answer;
}
diff --git a/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java b/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
index f4aebd5..599359c 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/SensitiveUtils.java
@@ -14,16 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.camel.util;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
public final class SensitiveUtils {
- private static final Set<String> SENSITIVE_KEYS = new HashSet<>(
+
+ private static final Set<String> SENSITIVE_KEYS = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList(
// Generated by camel build tools - do NOT edit this list!
// SENSITIVE-KEYS: START
@@ -92,12 +93,19 @@ public final class SensitiveUtils {
"verificationcode",
"zookeeperpassword"
// SENSITIVE-KEYS: END
- ));
+ )));
private SensitiveUtils() {
}
/**
+ * All the sensitive keys (unmodifiable) in lower-case
+ */
+ public static Set<String> getSensitiveKeys() {
+ return SENSITIVE_KEYS;
+ }
+
+ /**
* Whether the given configuration property contains a sensitive a sensitive key (such as password, accesstoken,
* etc.)
*
@@ -113,4 +121,5 @@ public final class SensitiveUtils {
text = StringHelper.replaceAll(text, "-", "");
return SENSITIVE_KEYS.contains(text);
}
+
}