You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2022/10/24 02:11:19 UTC
[accumulo] branch main updated: speed up test using shared cluster instance (#3036)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 35c9ce5b39 speed up test using shared cluster instance (#3036)
35c9ce5b39 is described below
commit 35c9ce5b392f90a2de0d7709fad8b58471fa821d
Author: EdColeman <de...@etcoleman.com>
AuthorDate: Sun Oct 23 22:11:12 2022 -0400
speed up test using shared cluster instance (#3036)
---
.../accumulo/test/conf/PropStoreConfigIT.java | 141 ++++++++++++++-------
1 file changed, 92 insertions(+), 49 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/conf/PropStoreConfigIT.java b/test/src/main/java/org/apache/accumulo/test/conf/PropStoreConfigIT.java
index 36de706ff5..69e657a3be 100644
--- a/test/src/main/java/org/apache/accumulo/test/conf/PropStoreConfigIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/conf/PropStoreConfigIT.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.test.conf;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getMemoryAsBytes;
import static org.apache.accumulo.harness.AccumuloITBase.MINI_CLUSTER_ONLY;
import static org.apache.accumulo.harness.AccumuloITBase.SUNNY_DAY;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -27,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -38,7 +40,6 @@ import java.util.function.Consumer;
import java.util.function.Supplier;
import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
@@ -46,12 +47,15 @@ import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.NamespacePropKey;
import org.apache.accumulo.server.conf.store.SystemPropKey;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.test.util.Wait;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
@@ -59,10 +63,33 @@ import org.slf4j.LoggerFactory;
@Tag(MINI_CLUSTER_ONLY)
@Tag(SUNNY_DAY)
-public class PropStoreConfigIT extends AccumuloClusterHarness {
+public class PropStoreConfigIT extends SharedMiniClusterBase {
private static final Logger log = LoggerFactory.getLogger(PropStoreConfigIT.class);
+ @Override
+ protected Duration defaultTimeout() {
+ return Duration.ofMinutes(1);
+ }
+
+ @BeforeAll
+ public static void setup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ }
+
+ @AfterAll
+ public static void teardown() {
+ SharedMiniClusterBase.stopMiniCluster();
+ }
+
+ @BeforeEach
+ public void clear() throws Exception {
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ client.instanceOperations().modifyProperties(Map::clear);
+ assertTrue(Wait.waitFor(() -> getStoredConfiguration().size() == 0, 5000, 500));
+ }
+ }
+
@Test
public void setTablePropTest() throws Exception {
String table = getUniqueNames(1)[0];
@@ -71,21 +98,26 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
client.tableOperations().create(table);
- log.info("Tables: {}", client.tableOperations().list());
+ log.debug("Tables: {}", client.tableOperations().list());
+ // override default in sys, and then over-ride that for table prop
client.instanceOperations().setProperty(Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+ client.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "false");
+
+ assertTrue(Wait.waitFor(() -> client.instanceOperations().getSystemConfiguration()
+ .get(Property.TABLE_BLOOM_ENABLED.getKey()).equals("true"), 5000, 500));
+ assertTrue(Wait.waitFor(() -> client.tableOperations().getConfiguration(table)
+ .get(Property.TABLE_BLOOM_ENABLED.getKey()).equals("false"), 5000, 500));
+
+ // revert sys, and then over-ride to true with table prop
+ client.instanceOperations().removeProperty(Property.TABLE_BLOOM_ENABLED.getKey());
client.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
- Thread.sleep(SECONDS.toMillis(3L));
+ assertTrue(Wait.waitFor(() -> client.instanceOperations().getSystemConfiguration()
+ .get(Property.TABLE_BLOOM_ENABLED.getKey()).equals("false"), 5000, 500));
+ assertTrue(Wait.waitFor(() -> client.tableOperations().getConfiguration(table)
+ .get(Property.TABLE_BLOOM_ENABLED.getKey()).equals("true"), 5000, 500));
- var props = client.tableOperations().getProperties(table);
- log.info("Props: {}", props);
- for (Map.Entry<String,String> e : props) {
- if (e.getKey().contains("table.bloom.enabled")) {
- log.info("after bloom property: {}={}", e.getKey(), e.getValue());
- assertEquals("true", e.getValue());
- }
- }
}
}
@@ -102,24 +134,31 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
log.info("Tables: {}", client.tableOperations().list());
- client.instanceOperations().setProperty(Property.TABLE_BLOOM_ENABLED.getKey(), "true");
- client.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
-
- Thread.sleep(SECONDS.toMillis(1L));
-
- var props = client.tableOperations().getProperties(table);
- log.info("Props: {}", props);
- for (Map.Entry<String,String> e : props) {
- if (e.getKey().contains("table.bloom.enabled")) {
- log.info("after bloom property: {}={}", e.getKey(), e.getValue());
- assertEquals("true", e.getValue());
- }
- }
+ client.instanceOperations().setProperty(Property.TABLE_BLOOM_SIZE.getKey(), "12345");
+ assertTrue(Wait.waitFor(() -> client.instanceOperations().getSystemConfiguration()
+ .get(Property.TABLE_BLOOM_SIZE.getKey()).equals("12345"), 5000, 500));
+ assertEquals("12345",
+ client.tableOperations().getConfiguration(table).get(Property.TABLE_BLOOM_SIZE.getKey()));
+
+ client.namespaceOperations().setProperty(namespace, Property.TABLE_BLOOM_SIZE.getKey(),
+ "23456");
+ assertTrue(Wait.waitFor(() -> client.namespaceOperations().getConfiguration(namespace)
+ .get(Property.TABLE_BLOOM_SIZE.getKey()).equals("23456"), 5000, 500));
+ assertEquals("23456",
+ client.tableOperations().getConfiguration(table).get(Property.TABLE_BLOOM_SIZE.getKey()));
+
+ client.tableOperations().setProperty(table, Property.TABLE_BLOOM_SIZE.getKey(), "34567");
+ assertTrue(Wait.waitFor(() -> client.tableOperations().getConfiguration(table)
+ .get(Property.TABLE_BLOOM_SIZE.getKey()).equals("34567"), 5000, 500));
+ assertEquals("12345", client.instanceOperations().getSystemConfiguration()
+ .get(Property.TABLE_BLOOM_SIZE.getKey()));
+ assertEquals("23456", client.namespaceOperations().getConfiguration(namespace)
+ .get(Property.TABLE_BLOOM_SIZE.getKey()));
var tableIdMap = client.tableOperations().tableIdMap();
var nsIdMap = client.namespaceOperations().namespaceIdMap();
- ServerContext context = getServerContext();
+ ServerContext context = getCluster().getServerContext();
NamespaceId nid = NamespaceId.of(nsIdMap.get(namespace));
TableId tid = TableId.of(tableIdMap.get(table));
@@ -159,12 +198,9 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
client.tableOperations().create(table1);
client.tableOperations().create(table2);
- client.instanceOperations().setProperty(Property.TABLE_BLOOM_ENABLED.getKey(), "true");
- client.tableOperations().setProperty(table1, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
-
Thread.sleep(SECONDS.toMillis(3L));
- ServerContext serverContext = cluster.getServerContext();
+ ServerContext serverContext = getCluster().getServerContext();
ZooReaderWriter zrw = serverContext.getZooReaderWriter();
// validate that a world-readable node has expected perms to validate test method
@@ -206,30 +242,38 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
Map<String,String> config = client.instanceOperations().getSystemConfiguration();
Map<String,String> properties = getStoredConfiguration();
+ final String origMaxOpenFiles = config.get(Property.TSERV_SCAN_MAX_OPENFILES.getKey());
+ final String origMaxMem = config.get(Property.TSERV_MAXMEM.getKey());
+ // final long origMaxMem = getMemoryAsBytes(config.get(Property.TSERV_MAXMEM.getKey()));
+
+ client.instanceOperations().modifyProperties(Map::clear);
+ assertTrue(Wait.waitFor(() -> getStoredConfiguration().size() == 0, 5000, 500));
+
// should be empty to start
- assertEquals(0, properties.size());
+ final int numProps = properties.size();
- final String originalClientPort = config.get(Property.TSERV_CLIENTPORT.getKey());
- final String originalMaxMem = config.get(Property.TSERV_MAXMEM.getKey());
+ final String expectedMaxOpenFiles = "" + (Integer.parseInt(origMaxOpenFiles) + 1);
+ final String expectMaxMem = (getMemoryAsBytes(origMaxMem) + 1024) + "M";
// Set properties in ZK
client.instanceOperations().modifyProperties(original -> {
- original.put(Property.TSERV_CLIENTPORT.getKey(), "9998");
- original.put(Property.TSERV_MAXMEM.getKey(), "35%");
+ original.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), expectedMaxOpenFiles);
+ original.put(Property.TSERV_MAXMEM.getKey(), expectMaxMem);
});
// Verify system properties added
- assertTrue(Wait.waitFor(() -> getStoredConfiguration().size() > 0, 5000, 500));
+ assertTrue(Wait.waitFor(() -> getStoredConfiguration().size() > numProps, 5000, 500));
// verify properties updated
properties = getStoredConfiguration();
- assertEquals("9998", properties.get(Property.TSERV_CLIENTPORT.getKey()));
- assertEquals("35%", properties.get(Property.TSERV_MAXMEM.getKey()));
+ assertEquals(expectedMaxOpenFiles,
+ properties.get(Property.TSERV_SCAN_MAX_OPENFILES.getKey()));
+ assertEquals(expectMaxMem, properties.get(Property.TSERV_MAXMEM.getKey()));
// verify properties updated in config as well
config = client.instanceOperations().getSystemConfiguration();
- assertEquals("9998", config.get(Property.TSERV_CLIENTPORT.getKey()));
- assertEquals("35%", config.get(Property.TSERV_MAXMEM.getKey()));
+ assertEquals(expectedMaxOpenFiles, config.get(Property.TSERV_SCAN_MAX_OPENFILES.getKey()));
+ assertEquals(expectMaxMem, config.get(Property.TSERV_MAXMEM.getKey()));
// Verify removal by sending empty map - only removes props that were set in previous values
// should be restored
@@ -239,8 +283,8 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
// verify default system config restored
config = client.instanceOperations().getSystemConfiguration();
- assertEquals(originalClientPort, config.get(Property.TSERV_CLIENTPORT.getKey()));
- assertEquals(originalMaxMem, config.get(Property.TSERV_MAXMEM.getKey()));
+ assertEquals(origMaxOpenFiles, config.get(Property.TSERV_SCAN_MAX_OPENFILES.getKey()));
+ assertEquals(origMaxMem, config.get(Property.TSERV_MAXMEM.getKey()));
}
}
@@ -390,7 +434,7 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
}
};
- runConcurrentPropsModificationTest(propShim, client);
+ runConcurrentPropsModificationTest(propShim);
}
}
@@ -414,7 +458,7 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
}
};
- runConcurrentPropsModificationTest(propShim, client);
+ runConcurrentPropsModificationTest(propShim);
}
}
@@ -435,7 +479,7 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
}
};
- runConcurrentPropsModificationTest(propShim, client);
+ runConcurrentPropsModificationTest(propShim);
}
}
@@ -444,8 +488,7 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
* modifications. The modifications build on each other and the test is written in such a way that
* if any single modification is lost it can be detected.
*/
- private static void runConcurrentPropsModificationTest(PropertyShim propShim,
- AccumuloClient client) throws Exception {
+ private static void runConcurrentPropsModificationTest(PropertyShim propShim) throws Exception {
ExecutorService executor = Executors.newFixedThreadPool(4);
final int iterations = 151;
@@ -579,7 +622,7 @@ public class PropStoreConfigIT extends AccumuloClusterHarness {
assertEquals(iterations * (3 + 11 + 17) + 3, afterB);
assertEquals(iterations * (5 + 13) + 5, afterC);
assertEquals(iterations * 7 + 7, afterD);
- assertEquals(iterations * 19 + 0, afterE);
+ assertEquals(iterations * 19, afterE);
executor.shutdown();
}