You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2022/03/29 17:36:37 UTC

[GitHub] [kafka] mumrah commented on a change in pull request #11941: KAFKA-13749: CreateTopics in KRaft must return configs

mumrah commented on a change in pull request #11941:
URL: https://github.com/apache/kafka/pull/11941#discussion_r837560543



##########
File path: metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java
##########
@@ -530,21 +543,38 @@ private ApiError createTopic(CreatableTopic topic,
                         " time(s): " + e.getMessage());
             }
             ApiError error = maybeCheckCreateTopicPolicy(() -> {
-                Map<String, String> configs = new HashMap<>();
-                topic.configs().forEach(config -> configs.put(config.name(), config.value()));
                 return new CreateTopicPolicy.RequestMetadata(
-                    topic.name(), numPartitions, replicationFactor, null, configs);
+                    topic.name(), numPartitions, replicationFactor, null, creationConfigs);
             });
             if (error.isFailure()) return error;
         }
         Uuid topicId = Uuid.randomUuid();
-        successes.put(topic.name(), new CreatableTopicResult().
+        CreatableTopicResult result = new CreatableTopicResult().
             setName(topic.name()).
             setTopicId(topicId).
-            setErrorCode((short) 0).
-            setErrorMessage(null).
-            setNumPartitions(newParts.size()).
-            setReplicationFactor((short) newParts.get(0).replicas.length));
+            setErrorCode(NONE.code()).
+            setErrorMessage(null);
+        if (includeConfigs) {
+            Map<String, ConfigEntry> effectiveConfig = configurationControl.
+                computeEffectiveTopicConfigs(creationConfigs);
+            List<String> configNames = new ArrayList<>(effectiveConfig.keySet());
+            configNames.sort(String::compareTo);
+            for (String configName : configNames) {
+                ConfigEntry entry = effectiveConfig.get(configName);
+                result.configs().add(new CreateTopicsResponseData.CreatableTopicConfigs().
+                    setName(entry.name()).
+                    setValue(entry.isSensitive() ? null : entry.value()).
+                    setReadOnly(entry.isReadOnly()).
+                    setConfigSource(KafkaConfigSchema.translateConfigSource(entry.source()).id()).
+                    setIsSensitive(entry.isSensitive()));
+            }
+            result.setNumPartitions(newParts.size());
+            result.setReplicationFactor((short) newParts.get(0).replicas.length);

Review comment:
       Why don't we include the partitions and replication factor in the response if the topic authz fails?

##########
File path: metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java
##########
@@ -462,7 +475,9 @@ public void replay(RemoveTopicRecord record) {
 
     private ApiError createTopic(CreatableTopic topic,
                                  List<ApiMessageAndVersion> records,
-                                 Map<String, CreatableTopicResult> successes) {
+                                 Map<String, CreatableTopicResult> successes,
+                                 boolean includeConfigs) {

Review comment:
       I wonder if "configsAuthorized" or similar would be better here. "includeConfigs" sounds like request option rather than authz enforcement

##########
File path: metadata/src/main/java/org/apache/kafka/metadata/KafkaConfigSchema.java
##########
@@ -84,4 +157,91 @@ public String getDefault(ConfigResource.Type type, String key) {
         }
         return ConfigDef.convertToString(configKey.defaultValue, configKey.type);
     }
+
+    public Map<String, ConfigEntry> resolveEffectiveTopicConfigs(
+            Map<String, ? extends Object> staticNodeConfig,
+            Map<String, ? extends Object> dynamicClusterConfigs,
+            Map<String, ? extends Object> dynamicNodeConfigs,
+            Map<String, ? extends Object> dynamicTopicConfigs) {
+        ConfigDef configDef = configDefs.getOrDefault(ConfigResource.Type.TOPIC, EMPTY_CONFIG_DEF);
+        HashMap<String, ConfigEntry> effectiveConfigs = new HashMap<>();
+        for (ConfigDef.ConfigKey configKey : configDef.configKeys().values()) {
+            ConfigEntry entry = resolveEffectiveTopicConfig(configKey, staticNodeConfig,
+                dynamicClusterConfigs, dynamicNodeConfigs, dynamicTopicConfigs);
+            effectiveConfigs.put(entry.name(), entry);
+        }
+        return effectiveConfigs;
+    }
+
+    private ConfigEntry resolveEffectiveTopicConfig(ConfigDef.ConfigKey configKey,
+            Map<String, ? extends Object> staticNodeConfig,
+            Map<String, ? extends Object> dynamicClusterConfigs,
+            Map<String, ? extends Object> dynamicNodeConfigs,
+            Map<String, ? extends Object> dynamicTopicConfigs) {
+        if (dynamicTopicConfigs.containsKey(configKey.name)) {
+            return toConfigEntry(configKey,
+                dynamicTopicConfigs.get(configKey.name),
+                ConfigSource.DYNAMIC_TOPIC_CONFIG, IDENTITY);
+        }
+        List<ConfigSynonym> synonyms = logConfigSynonyms.getOrDefault(configKey.name, emptyList());
+        for (ConfigSynonym synonym : synonyms) {
+            if (dynamicNodeConfigs.containsKey(synonym.name())) {
+                return toConfigEntry(configKey, dynamicNodeConfigs.get(synonym.name()),
+                    ConfigSource.DYNAMIC_BROKER_CONFIG, synonym.converter());
+            }
+        }
+        for (ConfigSynonym synonym : synonyms) {
+            if (dynamicClusterConfigs.containsKey(synonym.name())) {
+                return toConfigEntry(configKey, dynamicClusterConfigs.get(synonym.name()),
+                    ConfigSource.DYNAMIC_DEFAULT_BROKER_CONFIG, synonym.converter());
+            }
+        }
+        for (ConfigSynonym synonym : synonyms) {
+            if (staticNodeConfig.containsKey(synonym.name())) {
+                return toConfigEntry(configKey, staticNodeConfig.get(synonym.name()),
+                    ConfigSource.STATIC_BROKER_CONFIG, synonym.converter());
+            }
+        }
+        return toConfigEntry(configKey, configKey.hasDefault() ? configKey.defaultValue : null,
+            ConfigSource.DEFAULT_CONFIG, IDENTITY);
+    }
+
+    private ConfigEntry toConfigEntry(ConfigDef.ConfigKey configKey,
+                                  Object value,

Review comment:
       nit: whitespace is off a bit

##########
File path: metadata/src/main/java/org/apache/kafka/metadata/ConfigSynonym.java
##########
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.metadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+
+public class ConfigSynonym {
+    private static final Logger log = LoggerFactory.getLogger(ConfigSynonym.class);
+
+    public static final Function<String, String> IDENTITY = a -> a;
+
+    public static final Function<String, String> HOURS_TO_MILLISECONDS = input -> {
+        int hours = valueToInt(input, 0, "hoursToMilliseconds");
+        return String.valueOf(TimeUnit.MILLISECONDS.convert(hours, TimeUnit.HOURS));
+    };
+
+    public static final Function<String, String> MINUTES_TO_MILLISECONDS = input -> {
+        int hours = valueToInt(input, 0, "minutesToMilliseconds");
+        return String.valueOf(TimeUnit.MILLISECONDS.convert(hours, TimeUnit.MINUTES));
+    };
+
+    private static int valueToInt(String input, int defaultValue, String what) {
+        String trimmedInput = input.trim();

Review comment:
       Can `input` ever be null here?

##########
File path: metadata/src/main/java/org/apache/kafka/metadata/ConfigSynonym.java
##########
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.metadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+
+public class ConfigSynonym {

Review comment:
       nit: java doc for this class?

##########
File path: core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala
##########
@@ -189,6 +192,15 @@ abstract class BaseAdminIntegrationTest extends IntegrationTestHarness with Logg
 
   override def modifyConfigs(configs: Seq[Properties]): Unit = {
     super.modifyConfigs(configs)
+    // For testCreateTopicsReturnsConfigs, set some static broker configurations so that we can
+    // verify that they show up in the "configs" output of CreateTopics.

Review comment:
       Could we make a separate test class for `testCreateTopicsReturnsConfigs` so we can override the node's properties with a method? It will be very hard for someone to debug problems with this test if they don't know we've defined this special bootstrapping here.

##########
File path: metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java
##########
@@ -51,27 +52,100 @@
 
 
 public class ConfigurationControlManager {
+    public static final ConfigResource DEFAULT_NODE = new ConfigResource(Type.BROKER, "");
+
     private final Logger log;
     private final SnapshotRegistry snapshotRegistry;
     private final KafkaConfigSchema configSchema;
     private final Consumer<ConfigResource> existenceChecker;
     private final Optional<AlterConfigPolicy> alterConfigPolicy;
     private final ConfigurationValidator validator;
     private final TimelineHashMap<ConfigResource, TimelineHashMap<String, String>> configData;
+    private final Map<String, Object> staticConfig;
+    private final ConfigResource currentNode;

Review comment:
       nit: "node" is kind of generic/overloaded -- maybe currentConfigResource or thisConfigResource?

##########
File path: metadata/src/main/java/org/apache/kafka/metadata/ConfigSynonym.java
##########
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.metadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+
+public class ConfigSynonym {
+    private static final Logger log = LoggerFactory.getLogger(ConfigSynonym.class);
+
+    public static final Function<String, String> IDENTITY = a -> a;
+
+    public static final Function<String, String> HOURS_TO_MILLISECONDS = input -> {
+        int hours = valueToInt(input, 0, "hoursToMilliseconds");
+        return String.valueOf(TimeUnit.MILLISECONDS.convert(hours, TimeUnit.HOURS));
+    };
+
+    public static final Function<String, String> MINUTES_TO_MILLISECONDS = input -> {
+        int hours = valueToInt(input, 0, "minutesToMilliseconds");
+        return String.valueOf(TimeUnit.MILLISECONDS.convert(hours, TimeUnit.MINUTES));
+    };
+
+    private static int valueToInt(String input, int defaultValue, String what) {
+        String trimmedInput = input.trim();
+        if (trimmedInput.isEmpty()) {
+            return defaultValue;
+        }
+        try {
+            return Integer.parseInt(trimmedInput);
+        } catch (Exception e) {
+            log.error("{} failed: unable to parse '{}' as an integer.", what, trimmedInput, e);
+            return defaultValue;
+        }
+    }
+
+    private final String name;
+    private final Function<String, String> converter;
+
+    public ConfigSynonym(String name, Function<String, String> converter) {
+        this.name = name;
+        this.converter = converter;
+    }
+
+    public ConfigSynonym(String name) {
+        this(name, IDENTITY);
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public Function<String, String> converter() {
+        return converter;
+    }
+
+    public String convert(String input) {

Review comment:
       Is this used anywhere?

##########
File path: metadata/src/main/java/org/apache/kafka/metadata/ConfigSynonym.java
##########
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.metadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+
+public class ConfigSynonym {
+    private static final Logger log = LoggerFactory.getLogger(ConfigSynonym.class);
+
+    public static final Function<String, String> IDENTITY = a -> a;

Review comment:
       There is a built-in for this `Function.identity()`




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org