You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2024/11/07 21:48:04 UTC
(accumulo) branch main updated (7da7ac89cc -> c19fe486a6)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 7da7ac89cc Fixes discrepancy between `admin fate delete` and `UserFateStore.delete` (#5016)
add ed4161ea21 Restrict group names in cluster.yaml to valid bash var name characters (#5038)
add 51633db8df Merge branch '2.1' into 3.1
new c19fe486a6 Merge branch '3.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../core/conf/cluster/ClusterConfigParser.java | 34 +++++++++++++++++-----
.../core/conf/cluster/ClusterConfigParserTest.java | 17 +++++++++++
2 files changed, 44 insertions(+), 7 deletions(-)
(accumulo) 01/01: Merge branch '3.1'
Posted by dl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c19fe486a6b90ef067b2bfb6358325623aa203ed
Merge: 7da7ac89cc 51633db8df
Author: Dave Marion <dl...@apache.org>
AuthorDate: Thu Nov 7 21:47:49 2024 +0000
Merge branch '3.1'
.../core/conf/cluster/ClusterConfigParser.java | 34 +++++++++++++++++-----
.../core/conf/cluster/ClusterConfigParserTest.java | 17 +++++++++++
2 files changed, 44 insertions(+), 7 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
index 967d58c146,ff71848d2f..016d5daa80
--- a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
@@@ -38,25 -39,25 +39,36 @@@ import edu.umd.cs.findbugs.annotations.
public class ClusterConfigParser {
+ private static final Pattern GROUP_NAME_PATTERN =
+ Pattern.compile("^[a-zA-Z_]{1,}[a-zA-Z0-9_]{0,}$");
+
- public static void validateGroupNames(Set<String> names) {
++ public static void validateGroupNames(List<String> names) {
+ for (String name : names) {
+ if (!GROUP_NAME_PATTERN.matcher(name).matches()) {
+ throw new RuntimeException("Group name: " + name + " contains invalid characters");
+ }
+ }
+ }
+
private static final String PROPERTY_FORMAT = "%s=\"%s\"%n";
- private static final String[] SECTIONS = new String[] {"manager", "monitor", "gc", "tserver"};
+ private static final String COMPACTOR_PREFIX = "compactor.";
+ private static final String COMPACTORS_PER_HOST_PREFIX = "compactors_per_host.";
+ private static final String GC_KEY = "gc";
+ private static final String MANAGER_KEY = "manager";
+ private static final String MONITOR_KEY = "monitor";
+ private static final String SSERVER_PREFIX = "sserver.";
+ private static final String SSERVERS_PER_HOST_PREFIX = "sservers_per_host.";
+ private static final String TSERVER_PREFIX = "tserver.";
+ private static final String TSERVERS_PER_HOST_KEY = "tservers_per_host";
+
+ private static final String[] UNGROUPED_SECTIONS =
+ new String[] {MANAGER_KEY, MONITOR_KEY, GC_KEY};
private static final Set<String> VALID_CONFIG_KEYS =
- Set.of("manager", "monitor", "gc", "tserver", "tservers_per_host", "compaction.coordinator");
+ Set.of(MANAGER_KEY, MONITOR_KEY, GC_KEY, TSERVERS_PER_HOST_KEY);
- private static final Set<String> VALID_CONFIG_PREFIXES =
- Set.of("compaction.compactor.", "sserver.", "compactors_per_host.", "sservers_per_host.");
+ private static final Set<String> VALID_CONFIG_PREFIXES = Set.of(COMPACTOR_PREFIX, SSERVER_PREFIX,
+ TSERVER_PREFIX, SSERVERS_PER_HOST_PREFIX, COMPACTORS_PER_HOST_PREFIX);
private static final Predicate<String> VALID_CONFIG_SECTIONS =
section -> VALID_CONFIG_KEYS.contains(section)
@@@ -124,42 -124,39 +136,45 @@@
}
}
- if (config.containsKey("compaction.coordinator")) {
- out.printf(PROPERTY_FORMAT, "COORDINATOR_HOSTS", config.get("compaction.coordinator"));
- }
-
- String compactorPrefix = "compaction.compactor.";
- Set<String> compactorQueues =
- config.keySet().stream().filter(k -> k.startsWith(compactorPrefix))
- .map(k -> k.substring(compactorPrefix.length())).collect(Collectors.toSet());
- validateGroupNames(compactorQueues);
-
- if (!compactorQueues.isEmpty()) {
- out.printf(PROPERTY_FORMAT, "COMPACTION_QUEUES",
- compactorQueues.stream().collect(Collectors.joining(" ")));
- for (String queue : compactorQueues) {
- out.printf(PROPERTY_FORMAT, "COMPACTOR_HOSTS_" + queue,
- config.get("compaction.compactor." + queue));
- String numCompactors = config.getOrDefault("compactors_per_host." + queue, "1");
- out.printf(PROPERTY_FORMAT, "NUM_COMPACTORS_" + queue, numCompactors);
+ List<String> compactorGroups =
+ config.keySet().stream().filter(k -> k.startsWith(COMPACTOR_PREFIX))
+ .map(k -> k.substring(COMPACTOR_PREFIX.length())).sorted().collect(Collectors.toList());
++ validateGroupNames(compactorGroups);
+
+ if (!compactorGroups.isEmpty()) {
+ out.printf(PROPERTY_FORMAT, "COMPACTOR_GROUPS",
+ compactorGroups.stream().collect(Collectors.joining(" ")));
+ for (String group : compactorGroups) {
+ out.printf(PROPERTY_FORMAT, "COMPACTOR_HOSTS_" + group,
+ config.get(COMPACTOR_PREFIX + group));
+ String numCompactors = config.getOrDefault("compactors_per_host." + group, "1");
+ out.printf(PROPERTY_FORMAT, "NUM_COMPACTORS_" + group, numCompactors);
}
}
- String sserverPrefix = "sserver.";
- Set<String> sserverGroups = config.keySet().stream().filter(k -> k.startsWith(sserverPrefix))
- .map(k -> k.substring(sserverPrefix.length())).collect(Collectors.toSet());
+ List<String> sserverGroups = config.keySet().stream().filter(k -> k.startsWith(SSERVER_PREFIX))
+ .map(k -> k.substring(SSERVER_PREFIX.length())).sorted().collect(Collectors.toList());
+ validateGroupNames(sserverGroups);
if (!sserverGroups.isEmpty()) {
out.printf(PROPERTY_FORMAT, "SSERVER_GROUPS",
sserverGroups.stream().collect(Collectors.joining(" ")));
sserverGroups.forEach(ssg -> out.printf(PROPERTY_FORMAT, "SSERVER_HOSTS_" + ssg,
- config.get(sserverPrefix + ssg)));
+ config.get(SSERVER_PREFIX + ssg)));
sserverGroups.forEach(ssg -> out.printf(PROPERTY_FORMAT, "NUM_SSERVERS_" + ssg,
config.getOrDefault("sservers_per_host." + ssg, "1")));
+
+ }
+
+ List<String> tserverGroups = config.keySet().stream().filter(k -> k.startsWith(TSERVER_PREFIX))
+ .map(k -> k.substring(TSERVER_PREFIX.length())).sorted().collect(Collectors.toList());
++ validateGroupNames(tserverGroups);
+
+ if (!tserverGroups.isEmpty()) {
+ out.printf(PROPERTY_FORMAT, "TSERVER_GROUPS",
+ tserverGroups.stream().collect(Collectors.joining(" ")));
+ tserverGroups.forEach(tsg -> out.printf(PROPERTY_FORMAT, "TSERVER_HOSTS_" + tsg,
+ config.get(TSERVER_PREFIX + tsg)));
}
String numTservers = config.getOrDefault("tservers_per_host", "1");
diff --cc core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
index a50578914c,9777abc9b9..4793201355
--- a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
@@@ -34,7 -34,8 +34,8 @@@ import java.net.URL
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
++import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.function.Function;
import org.junit.jupiter.api.Test;
@@@ -276,4 -272,20 +277,20 @@@ public class ClusterConfigParserTest
assertTrue(exception.getMessage().contains("vserver"));
}
}
+
+ @Test
+ public void testGroupNamePattern() {
- ClusterConfigParser.validateGroupNames(Set.of("a"));
- ClusterConfigParser.validateGroupNames(Set.of("a", "b"));
- ClusterConfigParser.validateGroupNames(Set.of("default", "reg_ular"));
- ClusterConfigParser.validateGroupNames(Set.of("a1b2c3d4__"));
++ ClusterConfigParser.validateGroupNames(List.of("a"));
++ ClusterConfigParser.validateGroupNames(List.of("a", "b"));
++ ClusterConfigParser.validateGroupNames(List.of("default", "reg_ular"));
++ ClusterConfigParser.validateGroupNames(List.of("a1b2c3d4__"));
+ assertThrows(RuntimeException.class,
- () -> ClusterConfigParser.validateGroupNames(Set.of("0abcde")));
++ () -> ClusterConfigParser.validateGroupNames(List.of("0abcde")));
+ assertThrows(RuntimeException.class,
- () -> ClusterConfigParser.validateGroupNames(Set.of("a-b")));
++ () -> ClusterConfigParser.validateGroupNames(List.of("a-b")));
+ assertThrows(RuntimeException.class,
- () -> ClusterConfigParser.validateGroupNames(Set.of("a*b")));
++ () -> ClusterConfigParser.validateGroupNames(List.of("a*b")));
+ assertThrows(RuntimeException.class,
- () -> ClusterConfigParser.validateGroupNames(Set.of("a?b")));
++ () -> ClusterConfigParser.validateGroupNames(List.of("a?b")));
+ }
}