You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2021/09/20 20:43:14 UTC
[accumulo] branch main updated: Added additional tests in
CompactionIT (#2275)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 279ee5f Added additional tests in CompactionIT (#2275)
279ee5f is described below
commit 279ee5f634253581462ad14602b35e07e8570930
Author: Dave Marion <dl...@apache.org>
AuthorDate: Mon Sep 20 16:43:08 2021 -0400
Added additional tests in CompactionIT (#2275)
Added tests in CompactionIT for:
compaction cancelled on table during user compaction
table delete during user compaction
compaction of subset of files
setting compression compaction iterator
Closes #2114
Co-authored-by: Jeffrey Manno <je...@gmail.com>
---
.../accumulo/test/functional/CompactionIT.java | 276 ++++++++++++++++++++-
1 file changed, 275 insertions(+), 1 deletion(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index e93c6d1..a489f89 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@ -20,35 +20,53 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.PluginConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.client.admin.compaction.CompactionSelector;
+import org.apache.accumulo.core.client.admin.compaction.CompressionConfigurer;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.DevNull;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.CompactionExecutorIT;
+import org.apache.accumulo.test.ExternalCompactionIT.FSelector;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.accumulo.test.VerifyIngest.VerifyParams;
import org.apache.hadoop.conf.Configuration;
@@ -56,16 +74,48 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
+import org.bouncycastle.util.Arrays;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@SuppressWarnings("removal")
public class CompactionIT extends AccumuloClusterHarness {
+ public static class TestFilter extends Filter {
+
+ int modulus = 1;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options,
+ IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+
+ // if the init function is never called at all, then not setting the modulus option should
+ // cause the test to fail
+ if (options.containsKey("modulus")) {
+ Preconditions.checkArgument(!options.containsKey("pmodulus"));
+ modulus = Integer.parseInt(options.get("modulus"));
+ }
+
+ // use when partial compaction is expected
+ if (options.containsKey("pmodulus")) {
+ Preconditions.checkArgument(!options.containsKey("modulus"));
+ modulus = Integer.parseInt(options.get("pmodulus"));
+ }
+ }
+
+ @Override
+ public boolean accept(Key k, Value v) {
+ return Integer.parseInt(v.toString()) % modulus == 0;
+ }
+
+ }
+
public static class RandomErrorThrowingSelector implements CompactionSelector {
public static final String FILE_LIST_PARAM = "filesToCompact";
@@ -100,6 +150,8 @@ public class CompactionIT extends AccumuloClusterHarness {
private static final Logger log = LoggerFactory.getLogger(CompactionIT.class);
+ private static final int MAX_DATA = 1000;
+
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
@@ -160,7 +212,229 @@ public class CompactionIT extends AccumuloClusterHarness {
}
@Test
- public void test() throws Exception {
+ public void testCompactionWithTableIterator() throws Exception {
+ String table1 = this.getUniqueNames(1)[0];
+ try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
+ client.tableOperations().create(table1);
+ try (BatchWriter bw = client.createBatchWriter(table1)) {
+ for (int i = 1; i <= 4; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put("cf", "cq", new Value());
+ bw.addMutation(m);
+ bw.flush();
+ client.tableOperations().flush(table1, null, null, true);
+ }
+ }
+
+ IteratorSetting setting = new IteratorSetting(50, "delete", DevNull.class);
+ client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc));
+ client.tableOperations().compact(table1, new CompactionConfig().setWait(true));
+
+ try (Scanner s = client.createScanner(table1)) {
+ assertFalse(s.iterator().hasNext());
+ }
+ }
+ }
+
+ @Test
+ public void testUserCompactionCancellation() throws Exception {
+ final String table1 = this.getUniqueNames(1)[0];
+ try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
+ client.tableOperations().create(table1);
+ try (BatchWriter bw = client.createBatchWriter(table1)) {
+ for (int i = 1; i <= MAX_DATA; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put("cf", "cq", new Value());
+ bw.addMutation(m);
+ bw.flush();
+ client.tableOperations().flush(table1, null, null, true);
+ }
+ }
+
+ final AtomicReference<Exception> error = new AtomicReference<>();
+ final AtomicBoolean started = new AtomicBoolean(false);
+ Thread t = new Thread(() -> {
+ try {
+ started.set(true);
+ IteratorSetting setting = new IteratorSetting(50, "sleepy", SlowIterator.class);
+ setting.addOption("sleepTime", "3000");
+ setting.addOption("seekSleepTime", "3000");
+ client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc));
+ client.tableOperations().compact(table1, new CompactionConfig().setWait(true));
+ } catch (AccumuloSecurityException | TableNotFoundException | AccumuloException e) {
+ error.set(e);
+ }
+ });
+ t.start();
+ while (!started.get()) {
+ Thread.sleep(1000);
+ }
+ client.tableOperations().cancelCompaction(table1);
+ t.join();
+ Exception e = error.get();
+ assertNotNull(e);
+ assertEquals("Compaction canceled", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTableDeletedDuringUserCompaction() throws Exception {
+ final String table1 = this.getUniqueNames(1)[0];
+ try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
+ client.tableOperations().create(table1);
+ try (BatchWriter bw = client.createBatchWriter(table1)) {
+ for (int i = 1; i <= MAX_DATA; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put("cf", "cq", new Value());
+ bw.addMutation(m);
+ bw.flush();
+ client.tableOperations().flush(table1, null, null, true);
+ }
+ }
+
+ final AtomicReference<Exception> error = new AtomicReference<>();
+ final AtomicBoolean started = new AtomicBoolean(false);
+ Thread t = new Thread(() -> {
+ try {
+ started.set(true);
+ IteratorSetting setting = new IteratorSetting(50, "sleepy", SlowIterator.class);
+ setting.addOption("sleepTime", "3000");
+ setting.addOption("seekSleepTime", "3000");
+ client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc));
+ client.tableOperations().compact(table1, new CompactionConfig().setWait(true));
+ } catch (AccumuloSecurityException | TableNotFoundException | AccumuloException e) {
+ error.set(e);
+ }
+ });
+ t.start();
+ while (!started.get()) {
+ Thread.sleep(1000);
+ }
+ client.tableOperations().delete(table1);
+ t.join();
+ Exception e = error.get();
+ assertNotNull(e);
+ assertEquals("Compaction canceled", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartialCompaction() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ try (final AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
+
+ client.tableOperations().create(tableName);
+
+ // Insert MAX_DATA rows
+ try (BatchWriter bw = client.createBatchWriter(tableName)) {
+ for (int i = 0; i < MAX_DATA; i++) {
+ Mutation m = new Mutation(String.format("r:%04d", i));
+ m.put("", "", "" + i);
+ bw.addMutation(m);
+ }
+ }
+ client.tableOperations().flush(tableName);
+ IteratorSetting iterSetting = new IteratorSetting(100, TestFilter.class);
+ // make sure iterator options make it to compactor process
+ iterSetting.addOption("modulus", 17 + "");
+ CompactionConfig config =
+ new CompactionConfig().setIterators(List.of(iterSetting)).setWait(true);
+ client.tableOperations().compact(tableName, config);
+
+ // Insert 2 * MAX_DATA rows
+ try (BatchWriter bw = client.createBatchWriter(tableName)) {
+ for (int i = MAX_DATA; i < MAX_DATA * 2; i++) {
+ Mutation m = new Mutation(String.format("r:%04d", i));
+ m.put("", "", "" + i);
+ bw.addMutation(m);
+ }
+ }
+ // this should create an F file
+ client.tableOperations().flush(tableName);
+
+ // run a compaction that only compacts F files
+ iterSetting = new IteratorSetting(100, TestFilter.class);
+ // compact F file w/ different modulus and user pmodulus option for partial compaction
+ iterSetting.addOption("pmodulus", 19 + "");
+ config = new CompactionConfig().setIterators(List.of(iterSetting)).setWait(true)
+ .setSelector(new PluginConfig(FSelector.class.getName()));
+ client.tableOperations().compact(tableName, config);
+
+ try (Scanner scanner = client.createScanner(tableName)) {
+ int count = 0;
+ for (Entry<Key,Value> entry : scanner) {
+
+ int v = Integer.parseInt(entry.getValue().toString());
+ int modulus = v < MAX_DATA ? 17 : 19;
+
+ assertTrue(String.format("%s %s %d != 0", entry.getValue(), "%", modulus),
+ Integer.parseInt(entry.getValue().toString()) % modulus == 0);
+ count++;
+ }
+
+ // Verify
+ int expectedCount = 0;
+ for (int i = 0; i < MAX_DATA * 2; i++) {
+ int modulus = i < MAX_DATA ? 17 : 19;
+ if (i % modulus == 0) {
+ expectedCount++;
+ }
+ }
+ assertEquals(expectedCount, count);
+ }
+
+ }
+ }
+
+ @Test
+ public void testConfigurer() throws Exception {
+ String tableName = this.getUniqueNames(1)[0];
+
+ try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
+
+ Map<String,String> props = Map.of(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
+ NewTableConfiguration ntc = new NewTableConfiguration().setProperties(props);
+ client.tableOperations().create(tableName, ntc);
+
+ byte[] data = new byte[100000];
+ Arrays.fill(data, (byte) 65);
+ try (var writer = client.createBatchWriter(tableName)) {
+ for (int row = 0; row < 10; row++) {
+ Mutation m = new Mutation(row + "");
+ m.at().family("big").qualifier("stuff").put(data);
+ writer.addMutation(m);
+ }
+ }
+ client.tableOperations().flush(tableName, null, null, true);
+
+ // without compression, expect file to be large
+ long sizes = CompactionExecutorIT.getFileSizes(client, tableName);
+ assertTrue("Unexpected files sizes : " + sizes,
+ sizes > data.length * 10 && sizes < data.length * 11);
+
+ client.tableOperations().compact(tableName,
+ new CompactionConfig().setWait(true)
+ .setConfigurer(new PluginConfig(CompressionConfigurer.class.getName(),
+ Map.of(CompressionConfigurer.LARGE_FILE_COMPRESSION_TYPE, "gz",
+ CompressionConfigurer.LARGE_FILE_COMPRESSION_THRESHOLD, data.length + ""))));
+
+ // after compacting with compression, expect small file
+ sizes = CompactionExecutorIT.getFileSizes(client, tableName);
+ assertTrue("Unexpected files sizes: data: " + data.length + ", file:" + sizes,
+ sizes < data.length);
+
+ client.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
+
+ // after compacting without compression, expect big files again
+ sizes = CompactionExecutorIT.getFileSizes(client, tableName);
+ assertTrue("Unexpected files sizes : " + sizes,
+ sizes > data.length * 10 && sizes < data.length * 11);
+
+ }
+ }
+
+ @Test
+ public void testSuccessfulCompaction() throws Exception {
try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
final String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);