[GitHub] [kafka] mimaison commented on a diff in pull request #12992: KAFKA-14376-KIP887: Add ConfigProvider to make use of environment variables

["mimaison (via GitHub)" <gi...@apache.org>]

mimaison commented on code in PR #12992:
URL: https://github.com/apache/kafka/pull/12992#discussion_r1158216658


##########
clients/src/main/java/org/apache/kafka/common/config/provider/EnvVarConfigProvider.java:
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.config.provider;
+
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.config.ConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class EnvVarConfigProvider implements ConfigProvider {
+
+    public static final String ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG = "allowlist.pattern";
+    public static final String ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG_DOC = "A pattern / regular expression that needs to match for environment variables" +
+            " to be used by this config provider.";
+    private final Map<String, String> envVarMap;
+    private Pattern envVarPattern;
+
+    public EnvVarConfigProvider() {
+        envVarMap = getEnvVars();
+    }
+
+    public EnvVarConfigProvider(Map<String, String> envVarsAsArgument) {
+        envVarMap = envVarsAsArgument;
+    }
+
+    private static final Logger log = LoggerFactory.getLogger(EnvVarConfigProvider.class);
+
+    @Override
+    public void configure(Map<String, ?> configs) {
+        if (configs.containsKey(ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG)) {
+            envVarPattern = Pattern.compile(
+                    String.valueOf(configs.get(ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG))
+            );
+        } else {
+            envVarPattern = Pattern.compile(".*");
+            log.info("No pattern for environment variables provided. Using default pattern '(.*)'.");
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    /**
+     * @param s unused
+     * @return returns environment variables as configuration
+     */
+    @Override
+    public ConfigData get(String s) {
+        return get(s, null);
+    }
+
+    /**
+     * @param path    path, not used for environment variables
+     * @param keys the keys whose values will be retrieved.
+     * @return the configuration data.
+     */
+    @Override
+    public ConfigData get(String path, Set<String> keys) {
+
+        if (path != null && !path.isEmpty()) {
+            log.error("Path is not supported for EnvVarConfigProvider, invalid value '{}'", path);
+            throw new ConfigException("Path is not supported for EnvVarConfigProvider, invalid value '" + path + "'");
+        }
+
+        if (envVarMap == null) {
+            return new ConfigData(new HashMap<>());
+        }
+
+        Map<String, String> filteredEnvVarMap = envVarMap.entrySet().stream()

Review Comment:
   Can we compute this once in `configure()` instead of recomputing it each time keys are retrieved?



##########
clients/src/main/java/org/apache/kafka/common/config/provider/EnvVarConfigProvider.java:
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.config.provider;
+
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.config.ConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class EnvVarConfigProvider implements ConfigProvider {
+
+    public static final String ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG = "allowlist.pattern";
+    public static final String ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG_DOC = "A pattern / regular expression that needs to match for environment variables" +
+            " to be used by this config provider.";
+    private final Map<String, String> envVarMap;
+    private Pattern envVarPattern;
+
+    public EnvVarConfigProvider() {
+        envVarMap = getEnvVars();
+    }
+
+    public EnvVarConfigProvider(Map<String, String> envVarsAsArgument) {
+        envVarMap = envVarsAsArgument;
+    }
+
+    private static final Logger log = LoggerFactory.getLogger(EnvVarConfigProvider.class);
+
+    @Override
+    public void configure(Map<String, ?> configs) {
+        if (configs.containsKey(ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG)) {
+            envVarPattern = Pattern.compile(
+                    String.valueOf(configs.get(ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG))
+            );
+        } else {
+            envVarPattern = Pattern.compile(".*");
+            log.info("No pattern for environment variables provided. Using default pattern '(.*)'.");
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    /**
+     * @param s unused
+     * @return returns environment variables as configuration
+     */
+    @Override
+    public ConfigData get(String s) {
+        return get(s, null);
+    }
+
+    /**
+     * @param path    path, not used for environment variables
+     * @param keys the keys whose values will be retrieved.
+     * @return the configuration data.
+     */
+    @Override
+    public ConfigData get(String path, Set<String> keys) {
+
+        if (path != null && !path.isEmpty()) {
+            log.error("Path is not supported for EnvVarConfigProvider, invalid value '{}'", path);
+            throw new ConfigException("Path is not supported for EnvVarConfigProvider, invalid value '" + path + "'");
+        }
+
+        if (envVarMap == null) {
+            return new ConfigData(new HashMap<>());
+        }
+
+        Map<String, String> filteredEnvVarMap = envVarMap.entrySet().stream()
+                .filter(envVar -> envVarPattern.matcher(envVar.getKey()).matches())
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)
+                );
+
+        if (keys == null) {
+            return new ConfigData(filteredEnvVarMap);
+        }
+
+        HashMap<String, String> filteredData = new HashMap<>(filteredEnvVarMap);

Review Comment:
   We can use `Map` on the left side



##########
clients/src/test/java/org/apache/kafka/common/config/provider/EnvVarConfigProviderTest.java:
##########
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.config.provider;
+
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.config.ConfigException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashSet;
+import java.util.HashMap;
+
+import static org.apache.kafka.common.config.provider.EnvVarConfigProvider.ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class EnvVarConfigProviderTest {
+
+    private EnvVarConfigProvider envVarConfigProvider = null;
+    @BeforeEach
+    public void setup() {
+        Map<String, String> testEnvVars = new HashMap<String, String>() {
+            {
+                put("test_var1", "value1");
+                put("secret_var2", "value2");
+                put("new_var3", "value3");
+                put("not_so_secret_var4", "value4");
+            }
+        };
+        envVarConfigProvider = new EnvVarConfigProvider(testEnvVars);
+        envVarConfigProvider.configure(Collections.singletonMap("", ""));
+    }
+
+    @Test
+    void testGetAllEnvVarsNotEmpty() {
+        ConfigData properties = envVarConfigProvider.get("");
+        assertNotEquals(0, properties.data().size());
+    }
+
+    @Test
+    void testGetMultipleKeysAndCompare() {
+        ConfigData properties = envVarConfigProvider.get("");
+        assertNotEquals(0, properties.data().size());
+        assertEquals("value1", properties.data().get("test_var1"));
+        assertEquals("value2", properties.data().get("secret_var2"));
+        assertEquals("value3", properties.data().get("new_var3"));
+        assertEquals("value4", properties.data().get("not_so_secret_var4"));
+    }
+
+    @Test
+    public void testGetOneKeyWithNullPath() {
+        ConfigData config = envVarConfigProvider.get(null, Collections.singleton("secret_var2"));
+        Map<String, String> data = config.data();
+
+        assertEquals(1, data.size());
+        assertEquals("value2", data.get("secret_var2"));
+    }
+
+    @Test
+    public void testGetOneKeyWithEmptyPath() {
+        ConfigData config = envVarConfigProvider.get("", Collections.singleton("test_var1"));
+        Map<String, String> data = config.data();
+
+        assertEquals(1, data.size());
+        assertEquals("value1", data.get("test_var1"));
+    }
+
+    @Test
+    void testGetWhitelistedEnvVars() {

Review Comment:
   Can we replace `whitelist` with `keys` here?
   Same below, let's avoid using `whitelist`



##########
clients/src/main/java/org/apache/kafka/common/config/provider/EnvVarConfigProvider.java:
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.config.provider;
+
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.config.ConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class EnvVarConfigProvider implements ConfigProvider {
+
+    public static final String ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG = "allowlist.pattern";

Review Comment:
   What about simply using `ALLOWLIST_PATTERN_CONFIG`?
   Same below



##########
clients/src/test/java/org/apache/kafka/common/config/provider/EnvVarConfigProviderTest.java:
##########
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.config.provider;
+
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.config.ConfigException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashSet;
+import java.util.HashMap;
+
+import static org.apache.kafka.common.config.provider.EnvVarConfigProvider.ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class EnvVarConfigProviderTest {
+
+    private EnvVarConfigProvider envVarConfigProvider = null;
+    @BeforeEach
+    public void setup() {
+        Map<String, String> testEnvVars = new HashMap<String, String>() {
+            {
+                put("test_var1", "value1");
+                put("secret_var2", "value2");
+                put("new_var3", "value3");
+                put("not_so_secret_var4", "value4");
+            }
+        };
+        envVarConfigProvider = new EnvVarConfigProvider(testEnvVars);
+        envVarConfigProvider.configure(Collections.singletonMap("", ""));
+    }
+
+    @Test
+    void testGetAllEnvVarsNotEmpty() {
+        ConfigData properties = envVarConfigProvider.get("");
+        assertNotEquals(0, properties.data().size());
+    }
+
+    @Test
+    void testGetMultipleKeysAndCompare() {
+        ConfigData properties = envVarConfigProvider.get("");
+        assertNotEquals(0, properties.data().size());
+        assertEquals("value1", properties.data().get("test_var1"));
+        assertEquals("value2", properties.data().get("secret_var2"));
+        assertEquals("value3", properties.data().get("new_var3"));
+        assertEquals("value4", properties.data().get("not_so_secret_var4"));
+    }
+
+    @Test
+    public void testGetOneKeyWithNullPath() {
+        ConfigData config = envVarConfigProvider.get(null, Collections.singleton("secret_var2"));
+        Map<String, String> data = config.data();
+
+        assertEquals(1, data.size());
+        assertEquals("value2", data.get("secret_var2"));
+    }
+
+    @Test
+    public void testGetOneKeyWithEmptyPath() {
+        ConfigData config = envVarConfigProvider.get("", Collections.singleton("test_var1"));
+        Map<String, String> data = config.data();
+
+        assertEquals(1, data.size());
+        assertEquals("value1", data.get("test_var1"));
+    }
+
+    @Test
+    void testGetWhitelistedEnvVars() {
+        Set<String> whiteList = new HashSet<>(Arrays.asList("test_var1", "secret_var2"));
+        Set<String> keys = envVarConfigProvider.get(null, whiteList).data().keySet();
+        assertEquals(whiteList, keys);
+    }
+    @Test
+    void testNotNullPathNonEmptyThrowsException() {
+        assertThrows(ConfigException.class, () -> envVarConfigProvider.get("test-path", Collections.singleton("test_var1")));
+    }
+
+    @Test void testRegExpEnvVarsSingleEntryWhitelist() {
+        Map<String, String> testConfigMap = Collections.singletonMap(ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG, "secret_.*");
+        envVarConfigProvider.configure(testConfigMap);
+        Set<String> whiteList = Collections.singleton("secret_var2");
+        Set<String> keys = envVarConfigProvider.get(null, Collections.singleton("secret_var2")).data().keySet();
+
+        assertEquals(whiteList, keys);
+    }
+
+    @Test void testRegExpEnvVarsNoWhitelist() {
+        Map<String, String> testConfigMap = Collections.singletonMap(ENV_VAR_CONFIG_PROVIDER_PATTERN_CONFIG, "secret_.*");
+        envVarConfigProvider.configure(testConfigMap);
+        Set<String> keys = envVarConfigProvider.get("").data().keySet();
+
+        assertEquals(Collections.singleton("secret_var2"), keys);
+    }
+
+}

Review Comment:
   Can we add a newline?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org