You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by do...@apache.org on 2023/02/07 18:32:57 UTC
[accumulo-proxy] branch main updated: Fix CompactionSelecor test (#72)
This is an automated email from the ASF dual-hosted git repository.
domgarguilo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-proxy.git
The following commit(s) were added to refs/heads/main by this push:
new 9520eec Fix CompactionSelecor test (#72)
9520eec is described below
commit 9520eec3a56b10f3969511dc050ac80c62a5aa20
Author: Dom G <do...@apache.org>
AuthorDate: Tue Feb 7 13:32:51 2023 -0500
Fix CompactionSelecor test (#72)
* ix CompactionSelecor test
* Use TooManyDeletesSelector
* Add comment describing the test
---
.../accumulo/proxy/its/SelectHalfSelector.java | 45 -----------
.../apache/accumulo/proxy/its/SimpleProxyBase.java | 87 +++++++++++++++++++---
2 files changed, 75 insertions(+), 57 deletions(-)
diff --git a/src/test/java/org/apache/accumulo/proxy/its/SelectHalfSelector.java b/src/test/java/org/apache/accumulo/proxy/its/SelectHalfSelector.java
deleted file mode 100644
index 758c618..0000000
--- a/src/test/java/org/apache/accumulo/proxy/its/SelectHalfSelector.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.proxy.its;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
-import org.apache.accumulo.core.client.admin.compaction.CompactionSelector;
-
-/**
- * Select half of the files to compact
- */
-public class SelectHalfSelector implements CompactionSelector {
-
- @Override
- public void init(InitParameters iparams) {}
-
- @Override
- public Selection select(SelectionParameters sparams) {
- final var totalFiles = sparams.getAvailableFiles();
-
- final int halfOfFileCount = totalFiles.size() / 2;
-
- final List<CompactableFile> toCompact =
- totalFiles.stream().limit(halfOfFileCount).collect(Collectors.toList());
-
- return new Selection(toCompact);
- }
-
-}
diff --git a/src/test/java/org/apache/accumulo/proxy/its/SimpleProxyBase.java b/src/test/java/org/apache/accumulo/proxy/its/SimpleProxyBase.java
index 7af98c0..a42ef8e 100644
--- a/src/test/java/org/apache/accumulo/proxy/its/SimpleProxyBase.java
+++ b/src/test/java/org/apache/accumulo/proxy/its/SimpleProxyBase.java
@@ -52,8 +52,10 @@ import java.util.stream.Stream;
import org.apache.accumulo.cluster.ClusterUser;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.admin.compaction.TooManyDeletesSelector;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.client.summary.summarizers.DeletesSummarizer;
import org.apache.accumulo.core.clientImpl.ClientInfo;
import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.conf.DefaultConfiguration;
@@ -2216,6 +2218,7 @@ public abstract class SimpleProxyBase extends SharedMiniClusterBase {
break;
}
}
+ client.closeScanner(scanner);
return result;
}
@@ -2257,27 +2260,87 @@ public abstract class SimpleProxyBase extends SharedMiniClusterBase {
assertEquals(range.stop.timestamp, range.stop.timestamp);
}
+ private void addFile(String tableName, int startRow, int endRow, boolean delete)
+ throws TException {
+ Map<ByteBuffer,List<ColumnUpdate>> mutation = new HashMap<>();
+
+ for (int i = startRow; i < endRow; i++) {
+ String row = String.format("%09d", i);
+ ColumnUpdate columnUpdate = newColUpdate("cf", "cq", "v" + i);
+ columnUpdate.setDeleteCell(delete);
+ mutation.put(s2bb(row), List.of(columnUpdate));
+ }
+
+ client.updateAndFlush(sharedSecret, tableName, mutation);
+ client.flushTable(sharedSecret, tableName, null, null, true);
+ }
+
+ /**
+ * Test to make sure we can specify a Selector from the proxy client, and it will take effect when
+ * compactions occur
+ */
@Test
public void testCompactionSelector() throws Exception {
- String[] data = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split(" ");
- final int expectedFileCount = data.length;
+ // delete table so new tables won't have the same name
+ client.deleteTable(sharedSecret, tableName);
+
+ final String[] tableNames = getUniqueNameArray(3);
- for (String datum : data) {
- client.addSplits(sharedSecret, tableName, Set.of(s2bb(datum)));
- client.updateAndFlush(sharedSecret, tableName, mutation(datum, "cf", "cq", datum));
- client.flushTable(sharedSecret, tableName, null, null, true);
+ // for each table, set the summarizer property needed for TooManyDeletesSelector
+ final String summarizerKey = Property.TABLE_SUMMARIZER_PREFIX + "s1";
+ final String summarizerClassName = DeletesSummarizer.class.getName();
+
+ for (String tableName : tableNames) {
+ client.createTable(sharedSecret, tableName, true, TimeType.MILLIS);
+ client.setTableProperty(sharedSecret, tableName, summarizerKey, summarizerClassName);
}
- assertEquals(expectedFileCount, countFiles(tableName), "Unexpected file count");
+ // add files to each table
+
+ addFile(tableNames[0], 1, 1000, false);
+ addFile(tableNames[0], 1, 1000, true);
+
+ addFile(tableNames[1], 1, 1000, false);
+ addFile(tableNames[1], 1000, 2000, false);
+
+ addFile(tableNames[2], 1, 2000, false);
+ addFile(tableNames[2], 1, 1000, true);
+
+ final String messagePrefix = "Unexpected file count on table ";
+
+ for (String tableName : tableNames) {
+ assertEquals(2, countFiles(tableName), messagePrefix + tableName);
+ }
+
+ // compact the tables and check for expected file counts
+
+ final String selectorClassname = TooManyDeletesSelector.class.getName();
+ PluginConfig selector = new PluginConfig(selectorClassname, Map.of("threshold", ".99"));
+
+ for (String tableName : tableNames) {
+ client.compactTable(sharedSecret, tableName, null, null, null, true, true, selector, null);
+ }
+
+ assertEquals(0, countFiles(tableNames[0]), messagePrefix + tableNames[0]);
+ assertEquals(2, countFiles(tableNames[1]), messagePrefix + tableNames[1]);
+ assertEquals(2, countFiles(tableNames[2]), messagePrefix + tableNames[2]);
+
+ // create a selector with different properties then compact and check file counts
+
+ selector = new PluginConfig(selectorClassname, Map.of("threshold", ".40"));
+
+ for (String tableName : tableNames) {
+ client.compactTable(sharedSecret, tableName, null, null, null, true, true, selector, null);
+ }
- String selectorClassname = SelectHalfSelector.class.getName();
- PluginConfig selector = new PluginConfig(selectorClassname, Map.of());
+ assertEquals(0, countFiles(tableNames[0]), messagePrefix + tableNames[0]);
+ assertEquals(2, countFiles(tableNames[1]), messagePrefix + tableNames[1]);
+ assertEquals(1, countFiles(tableNames[2]), messagePrefix + tableNames[2]);
- client.compactTable(sharedSecret, tableName, null, null, null, true, true, selector, null);
+ client.compactTable(sharedSecret, tableNames[1], null, null, null, true, true, null, null);
- // SelectHalfSelector should lead to half the files being compacted
- Wait.waitFor(() -> countFiles(tableName) == (expectedFileCount / 2));
+ assertEquals(1, countFiles(tableNames[1]), messagePrefix + tableNames[2]);
}
@Test