You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dd...@apache.org on 2023/11/27 22:22:47 UTC
(accumulo) branch main updated: Remove deprecated prefix references (#3989)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 764f55e0c8 Remove deprecated prefix references (#3989)
764f55e0c8 is described below
commit 764f55e0c8e5e3588ff0464561f808b34bf13495
Author: Daniel Roberts <dd...@gmail.com>
AuthorDate: Mon Nov 27 17:22:40 2023 -0500
Remove deprecated prefix references (#3989)
Switch the remaining hardcoded references from
`tserver.compaction.major.service` to `compaction.service`.
---
.../core/client/admin/InstanceOperations.java | 12 +++----
.../core/spi/compaction/CompactionPlanner.java | 12 +++----
.../spi/compaction/DefaultCompactionPlanner.java | 28 +++++++--------
.../server/conf/CheckCompactionConfigTest.java | 42 +++++++++++-----------
.../test/compaction/CompactionRateLimitingIT.java | 7 ++--
.../compaction/ExternalCompactionTestUtils.java | 32 ++++++++---------
6 files changed, 66 insertions(+), 67 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index 85008d4ad6..9cc3aa3658 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@ -87,22 +87,22 @@ public interface InstanceOperations {
* {@code
* AccumuloClient client = getClient();
* Map<String,String> acceptedProps = client.instanceOperations().modifyProperties(currProps -> {
- * var planner = currProps.get("tserver.compaction.major.service.default.planner");
+ * var planner = currProps.get("compaction.service.default.planner");
* //This code will only change the compaction planner if its currently set to default settings.
* //The endsWith() function was used to make the example short, would be better to use equals().
* if(planner != null && planner.endsWith("DefaultCompactionPlanner") {
* // tservers will eventually see these compaction planner changes and when they do they will see all of the changes at once
* currProps.keySet().removeIf(
- * prop -> prop.startsWith("tserver.compaction.major.service.default.planner.opts."));
- * currProps.put("tserver.compaction.major.service.default.planner","MyPlannerClassName");
- * currProps.put("tserver.compaction.major.service.default.planner.opts.myOpt1","val1");
- * currProps.put("tserver.compaction.major.service.default.planner.opts.myOpt2","val2");
+ * prop -> prop.startsWith("compaction.service.default.planner.opts."));
+ * currProps.put("compaction.service.default.planner","MyPlannerClassName");
+ * currProps.put("compaction.service.default.planner.opts.myOpt1","val1");
+ * currProps.put("compaction.service.default.planner.opts.myOpt2","val2");
* }
* });
*
* // Since three properties were set may want to check for the values of all
* // three, just checking one in this example to keep it short.
- * if("MyPlannerClassName".equals(acceptedProps.get("tserver.compaction.major.service.default.planner"))){
+ * if("MyPlannerClassName".equals(acceptedProps.get("compaction.service.default.planner"))){
* // the compaction planner change was accepted or already existed, so take action for that outcome
* } else {
* // the compaction planner change was not done, so take action for that outcome
diff --git a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
index f96ed6f022..222ee651ef 100644
--- a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
@@ -45,11 +45,10 @@ public interface CompactionPlanner {
/**
* @return The configured options. For example if the system properties
- * {@code tserver.compaction.major.service.s1.planner.opts.p1=abc} and
- * {@code tserver.compaction.major.service.s1.planner.opts.p9=123} were set, then this
- * map would contain {@code p1=abc} and {@code p9=123}. In this example {@code s1} is
- * the identifier for the compaction service. Each compaction service has a single
- * planner.
+ * {@code compaction.service.s1.planner.opts.p1=abc} and
+ * {@code compaction.service.s1.planner.opts.p9=123} were set, then this map would
+ * contain {@code p1=abc} and {@code p9=123}. In this example {@code s1} is the
+ * identifier for the compaction service. Each compaction service has a single planner.
*/
Map<String,String> getOptions();
@@ -57,8 +56,7 @@ public interface CompactionPlanner {
* @return For a given key from the map returned by {@link #getOptions()} determines the fully
* qualified tablet property for that key. For example if a planner was being
* initialized for compaction service {@code CS9} and this method were passed
- * {@code prop1} then it would return
- * {@code tserver.compaction.major.service.CS9.planner.opts.prop1}.
+ * {@code prop1} then it would return {@code compaction.service.CS9.planner.opts.prop1}.
*/
String getFullyQualifiedOption(String key);
diff --git a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index cf6c099bf3..61ba4b39cf 100644
--- a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -57,8 +57,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* compaction service you are configuring.
*
* <ul>
- * <li>{@code tserver.compaction.major.service.<service>.opts.executors} This is a json array of
- * objects where each object has the fields:
+ * <li>{@code compaction.service.<service>.opts.executors} This is a json array of objects where
+ * each object has the fields:
* <table>
* <caption>Default Compaction Planner Executor options</caption>
* <tr>
@@ -90,18 +90,18 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* </table>
* <br>
* Note: The "executors" option has been deprecated in 3.1 and will be removed in a future release.
- * The property prefix "tserver.compaction.major.service" has also been deprecated in 3.1 and will
- * be removed in a future release. The maxSize field determines the maximum size of compaction that
- * will run on an executor. The maxSize field can have a suffix of K,M,G for kilobytes, megabytes,
- * or gigabytes and represents the sum of the input files for a given compaction. One executor can
- * have no max size and it will run everything that is too large for the other executors. If all
- * executors have a max size, then system compactions will only run for compactions smaller than the
- * largest max size. User, chop, and selector compactions will always run, even if there is no
- * executor for their size. These compactions will run on the executor with the largest max size.
- * The following example value for this property will create 3 threads to run compactions of files
- * whose file size sum is less than 100M, 3 threads to run compactions of files whose file size sum
- * is less than 500M, and run all other compactions on Compactors configured to run compactions for
- * Queue1:
+ * This example uses the new `compaction.service` prefix. The property prefix
+ * "tserver.compaction.major.service" has also been deprecated in 3.1 and will be removed in a
+ * future release. The maxSize field determines the maximum size of compaction that will run on an
+ * executor. The maxSize field can have a suffix of K,M,G for kilobytes, megabytes, or gigabytes and
+ * represents the sum of the input files for a given compaction. One executor can have no max size
+ * and it will run everything that is too large for the other executors. If all executors have a max
+ * size, then system compactions will only run for compactions smaller than the largest max size.
+ * User, chop, and selector compactions will always run, even if there is no executor for their
+ * size. These compactions will run on the executor with the largest max size. The following example
+ * value for this property will create 3 threads to run compactions of files whose file size sum is
+ * less than 100M, 3 threads to run compactions of files whose file size sum is less than 500M, and
+ * run all other compactions on Compactors configured to run compactions for Queue1:
*
* <pre>
* {@code
diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
index 4aa38abf90..ec903814df 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
@@ -47,9 +47,9 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testValidInput1() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\"");
@@ -61,15 +61,15 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testValidInput2() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}] \n"
- + "tserver.compaction.major.service.cs2.planner="
+ + "compaction.service.cs2.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs2.planner.opts.executors=\\\n"
+ + "compaction.service.cs2.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':7},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':5},\\\n"
+ "{'name':'large','type':'external','queue':'DCQ1'}]").replaceAll("'", "\"");
@@ -81,15 +81,15 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testValidInput3() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}] \n"
- + "tserver.compaction.major.service.cs2.planner="
+ + "compaction.service.cs2.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs2.planner.opts.executors=\\\n"
+ + "compaction.service.cs2.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':7},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':5},\\\n"
+ "{'name':'large','type':'external','queue':'DCQ1'}] \n"
@@ -104,9 +104,9 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testThrowsExternalNumThreadsError() throws IOException {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'external','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\"");
@@ -121,9 +121,9 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testNegativeThreadCount() throws IOException {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':-4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\"");
@@ -138,7 +138,7 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testNoPlanner() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ String inputString = ("compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\"");
@@ -153,9 +153,9 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testRepeatedCompactionExecutorID() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'small','type':'internal','numThreads':2}]").replaceAll("'", "\"");
@@ -188,9 +188,9 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testInvalidTypeValue() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internl','numThreads':2}]").replaceAll("'", "\"");
@@ -205,9 +205,9 @@ public class CheckCompactionConfigTest extends WithTestNames {
@Test
public void testInvalidMaxSize() throws Exception {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+ "[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+ "{'name':'medium','type':'internal','maxSize':'0M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\"");
diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
index 46e1060e24..a7e01466b4 100644
--- a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.test.compaction;
+import static org.apache.accumulo.core.conf.Property.COMPACTION_SERVICE_PREFIX;
import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -50,10 +51,10 @@ public class CompactionRateLimitingIT extends ConfigurableMacBase {
cfg.setProperty(Property.TABLE_MAJC_RATIO, "20");
cfg.setProperty(Property.TABLE_FILE_COMPRESSION_TYPE, "none");
- cfg.setProperty("tserver.compaction.major.service.test.rate.limit", RATE + "B");
- cfg.setProperty("tserver.compaction.major.service.test.planner",
+ cfg.setProperty(COMPACTION_SERVICE_PREFIX.getKey() + "test.rate.limit", RATE + "B");
+ cfg.setProperty(COMPACTION_SERVICE_PREFIX.getKey() + "test.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.test.planner.opts.executors",
+ cfg.setProperty(COMPACTION_SERVICE_PREFIX.getKey() + "test.planner.opts.executors",
"[{'name':'all','numThreads':2}]".replaceAll("'", "\""));
}
diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
index 76c3f5c54c..de242583fc 100644
--- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
+++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
@@ -197,37 +197,37 @@ public class ExternalCompactionTestUtils {
clProps.put(ClientProperty.BATCH_WRITER_LATENCY_MAX.getKey(), "2s");
cfg.setClientProps(clProps);
- cfg.setProperty("tserver.compaction.major.service.cs1.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs1.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner.opts.executors",
"[{'name':'all', 'type': 'external', 'queue': '" + QUEUE1 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs2.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs2.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs2.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs2.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE2 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs3.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs3.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs3.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs3.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE3 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs4.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs4.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs4.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs4.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE4 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs5.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs5.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs5.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs5.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE5 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs6.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs6.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs6.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs6.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE6 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs7.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs7.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs7.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs7.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE7 + "'}]");
- cfg.setProperty("tserver.compaction.major.service.cs8.planner",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs8.planner",
DefaultCompactionPlanner.class.getName());
- cfg.setProperty("tserver.compaction.major.service.cs8.planner.opts.executors",
+ cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs8.planner.opts.executors",
"[{'name':'all', 'type': 'external','queue': '" + QUEUE8 + "'}]");
cfg.setProperty(Property.COMPACTION_COORDINATOR_FINALIZER_COMPLETION_CHECK_INTERVAL, "5s");
cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL, "5s");