You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by rh...@apache.org on 2019/08/13 15:11:13 UTC
[kafka] branch 2.0 updated: KAFKA-8774: Regex can be found anywhere
in config value (#7197)
This is an automated email from the ASF dual-hosted git repository.
rhauch pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/2.0 by this push:
new 921937b KAFKA-8774: Regex can be found anywhere in config value (#7197)
921937b is described below
commit 921937ba3b897b9a16b806bab2111362064ff83c
Author: Arjun Satish <wi...@users.noreply.github.com>
AuthorDate: Tue Aug 13 07:40:12 2019 -0700
KAFKA-8774: Regex can be found anywhere in config value (#7197)
Corrected the AbstractHerder to correctly identify task configs that contain variables for externalized secrets. The original method incorrectly used `matcher.matches()` instead of `matcher.find()`. The former method expects the entire string to match the regex, whereas the second one can find a pattern anywhere within the input string (which fits this use case more correctly).
Added unit tests to cover various cases of a config with externalized secrets, and updated system tests to cover case where config value contains additional characters besides secret that requires regex pattern to be found anywhere in the string (as opposed to complete match).
Author: Arjun Satish <ar...@confluent.io>
Reviewer: Randall Hauch <rh...@gmail.com>
---
.../kafka/connect/runtime/AbstractHerder.java | 5 ++--
.../kafka/connect/runtime/AbstractHerderTest.java | 28 ++++++++++++++++++++++
tests/kafkatest/tests/connect/connect_rest_test.py | 2 +-
3 files changed, 32 insertions(+), 3 deletions(-)
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index 82fdecc..106b788 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -461,12 +461,13 @@ public abstract class AbstractHerder implements Herder, TaskStatus.Listener, Con
return result;
}
- private static Set<String> keysWithVariableValues(Map<String, String> rawConfig, Pattern pattern) {
+ // Visible for testing
+ static Set<String> keysWithVariableValues(Map<String, String> rawConfig, Pattern pattern) {
Set<String> keys = new HashSet<>();
for (Map.Entry<String, String> config : rawConfig.entrySet()) {
if (config.getValue() != null) {
Matcher matcher = pattern.matcher(config.getValue());
- if (matcher.matches()) {
+ if (matcher.find()) {
keys.add(config.getKey());
}
}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
index 8dbda18..c85764e 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
@@ -18,6 +18,7 @@ package org.apache.kafka.connect.runtime;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.config.ConfigTransformer;
import org.apache.kafka.connect.connector.ConnectRecord;
import org.apache.kafka.connect.connector.Connector;
import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
@@ -52,6 +53,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.apache.kafka.connect.runtime.AbstractHerder.keysWithVariableValues;
import static org.powermock.api.easymock.PowerMock.verifyAll;
import static org.powermock.api.easymock.PowerMock.replayAll;
import static org.easymock.EasyMock.strictMock;
@@ -322,6 +324,32 @@ public class AbstractHerderTest {
assertFalse(reverseTransformed.get(0).containsKey(TEST_KEY3));
}
+ @Test
+ public void testConfigProviderRegex() {
+ testConfigProviderRegex("\"${::}\"");
+ testConfigProviderRegex("${::}");
+ testConfigProviderRegex("\"${:/a:somevar}\"");
+ testConfigProviderRegex("\"${file::somevar}\"");
+ testConfigProviderRegex("${file:/a/b/c:}");
+ testConfigProviderRegex("${file:/tmp/somefile.txt:somevar}");
+ testConfigProviderRegex("\"${file:/tmp/somefile.txt:somevar}\"");
+ testConfigProviderRegex("plain.PlainLoginModule required username=\"${file:/tmp/somefile.txt:somevar}\"");
+ testConfigProviderRegex("plain.PlainLoginModule required username=${file:/tmp/somefile.txt:somevar}");
+ testConfigProviderRegex("plain.PlainLoginModule required username=${file:/tmp/somefile.txt:somevar} not null");
+ testConfigProviderRegex("plain.PlainLoginModule required username=${file:/tmp/somefile.txt:somevar} password=${file:/tmp/somefile.txt:othervar}");
+ testConfigProviderRegex("plain.PlainLoginModule required username", false);
+ }
+
+ private void testConfigProviderRegex(String rawConnConfig) {
+ testConfigProviderRegex(rawConnConfig, true);
+ }
+
+ private void testConfigProviderRegex(String rawConnConfig, boolean expected) {
+ Set<String> keys = keysWithVariableValues(Collections.singletonMap("key", rawConnConfig), ConfigTransformer.DEFAULT_PATTERN);
+ boolean actual = keys != null && !keys.isEmpty() && keys.contains("key");
+ assertEquals(String.format("%s should have matched regex", rawConnConfig), expected, actual);
+ }
+
private AbstractHerder createConfigValidationHerder(Class<? extends Connector> connectorClass) {
diff --git a/tests/kafkatest/tests/connect/connect_rest_test.py b/tests/kafkatest/tests/connect/connect_rest_test.py
index c13515b..ce37418 100644
--- a/tests/kafkatest/tests/connect/connect_rest_test.py
+++ b/tests/kafkatest/tests/connect/connect_rest_test.py
@@ -43,7 +43,7 @@ class ConnectRestApiTest(KafkaTest):
INPUT_FILE2 = "/mnt/connect.input2"
OUTPUT_FILE = "/mnt/connect.output"
- TOPIC = "${file:%s:topic.external}" % ConnectServiceBase.EXTERNAL_CONFIGS_FILE
+ TOPIC = "topic-${file:%s:topic.external}" % ConnectServiceBase.EXTERNAL_CONFIGS_FILE
TOPIC_TEST = "test"
DEFAULT_BATCH_SIZE = "2000"