You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2020/12/10 21:27:09 UTC
[accumulo] branch main updated: Stabilize some build elements
(#1835)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 12395ee Stabilize some build elements (#1835)
12395ee is described below
commit 12395eeec06bc88d9a530ccc9927d924eb8108e8
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Thu Dec 10 16:26:57 2020 -0500
Stabilize some build elements (#1835)
* Update to exec-maven-plugin 3.0.0
* Make main classes public
* Remove ERROR about native library loading when an optional setting is
not configured
* Remove ERROR from metrics that occurs before compaction manager is
initialized
* Scale back the resources required to run WaitForBalanceIT and
SimpleBalancerFairnessIT, to improve their stability
* Fix compaction threads dying in some tests because of mixconfigured
compaction checking delay time (must be greater than 1ms; some tests
were setting it to 0, which was a non-sensical setting)
---
.../apache/accumulo/core/conf/ClientConfigGenerate.java | 2 +-
.../apache/accumulo/core/conf/ConfigurationDocGen.java | 2 +-
pom.xml | 3 +--
.../java/org/apache/accumulo/tserver/NativeMap.java | 5 ++++-
.../tserver/metrics/TabletServerMetricsUtil.java | 8 +++++---
.../java/org/apache/accumulo/test/WaitForBalanceIT.java | 4 +++-
.../functional/BalanceInPresenceOfOfflineTableIT.java | 2 +-
.../accumulo/test/functional/BatchScanSplitIT.java | 2 +-
.../apache/accumulo/test/functional/BinaryStressIT.java | 4 ++--
.../accumulo/test/functional/ChaoticBalancerIT.java | 2 +-
.../test/functional/SimpleBalancerFairnessIT.java | 17 +++++++++--------
11 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java b/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
index 8a1f74f..0ad0a8a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
@@ -32,7 +32,7 @@ import com.google.common.collect.Sets;
* Generates client-properties.md for documentation on Accumulo website and
* accumulo-client.properties for Accumulo distribution tarball
*/
-class ClientConfigGenerate {
+public class ClientConfigGenerate {
private abstract class Format {
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index b4df567..5fd3ec0 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@ -28,7 +28,7 @@ import java.util.TreeMap;
* This class generates documentation to inform users of the available configuration properties in a
* presentable form.
*/
-class ConfigurationDocGen {
+public class ConfigurationDocGen {
private PrintStream doc;
private final TreeMap<String,Property> sortedProps = new TreeMap<>();
diff --git a/pom.xml b/pom.xml
index 185bc0a..1fc6e14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -988,10 +988,9 @@
<version>3.0.0</version>
</plugin>
<plugin>
- <!-- version 1.6.0 is broken; see https://github.com/mojohaus/exec-maven-plugin/issues/75 -->
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
+ <version>3.0.0</version>
</plugin>
<plugin>
<groupId>net.revelc.code</groupId>
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java
index ea29495..d3d67c7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java
@@ -87,7 +87,10 @@ public class NativeMap implements Iterable<Map.Entry<Key,Value>> {
// Check LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on Mac)
if (!isLoaded()) {
- log.error("Tried and failed to load Accumulo native library from {}", accumuloNativeLibDirs);
+ if (accumuloNativeLibDirs != null) {
+ log.error("Tried and failed to load Accumulo native library from {}",
+ accumuloNativeLibDirs);
+ }
String ldLibraryPath = System.getProperty("java.library.path");
try {
System.loadLibrary("accumulo");
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
index b3ac078..022285c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
@@ -91,11 +91,13 @@ public class TabletServerMetricsUtil {
}
public int getMajorCompactions() {
- return tserver.getCompactionManager().getCompactionsRunning();
+ var mgr = tserver.getCompactionManager();
+ return mgr == null ? 0 : mgr.getCompactionsRunning();
}
public int getMajorCompactionsQueued() {
- return tserver.getCompactionManager().getCompactionsQueued();
+ var mgr = tserver.getCompactionManager();
+ return mgr == null ? 0 : mgr.getCompactionsQueued();
}
public int getMinorCompactions() {
@@ -117,7 +119,7 @@ public class TabletServerMetricsUtil {
}
public int getOnlineCount() {
- return tserver.getOnlineTablets().values().size();
+ return tserver.getOnlineTablets().size();
}
public int getOpeningCount() {
diff --git a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
index 9ed2d97..2355b6b 100644
--- a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
@@ -46,6 +46,8 @@ import com.google.common.collect.Iterators;
public class WaitForBalanceIT extends ConfigurableMacBase {
+ private static final int NUM_SPLITS = 50;
+
@Override
public int defaultTimeoutSeconds() {
return 120;
@@ -62,7 +64,7 @@ public class WaitForBalanceIT extends ConfigurableMacBase {
c.tableOperations().create(tableName);
c.instanceOperations().waitForBalance();
final SortedSet<Text> partitionKeys = new TreeSet<>();
- for (int i = 0; i < 1000; i++) {
+ for (int i = 0; i < NUM_SPLITS; i++) {
partitionKeys.add(new Text("" + i));
}
c.tableOperations().addSplits(tableName, partitionKeys);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java
index 6a5e868..c19924b 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java
@@ -71,7 +71,7 @@ public class BalanceInPresenceOfOfflineTableIT extends AccumuloClusterHarness {
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> siteConfig = cfg.getSiteConfig();
siteConfig.put(Property.TSERV_MAXMEM.getKey(), "10K");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
cfg.setSiteConfig(siteConfig);
// ensure we have two tservers
if (cfg.getNumTservers() < 2) {
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index 7bef19c..e8773d9 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@ -50,7 +50,7 @@ public class BatchScanSplitIT extends AccumuloClusterHarness {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "50ms");
}
@Override
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
index 915c75b..3e55044 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@ -55,7 +55,7 @@ public class BinaryStressIT extends AccumuloClusterHarness {
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setProperty(Property.TSERV_MAXMEM, "50K");
- cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "50ms");
}
private String majcDelay, maxMem;
@@ -71,7 +71,7 @@ public class BinaryStressIT extends AccumuloClusterHarness {
majcDelay = conf.get(Property.TSERV_MAJC_DELAY.getKey());
maxMem = conf.get(Property.TSERV_MAXMEM.getKey());
- iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "0");
+ iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
iops.setProperty(Property.TSERV_MAXMEM.getKey(), "50K");
getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java
index ba362c0..204eedb 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java
@@ -42,7 +42,7 @@ public class ChaoticBalancerIT extends AccumuloClusterHarness {
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> siteConfig = cfg.getSiteConfig();
siteConfig.put(Property.TSERV_MAXMEM.getKey(), "10K");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
// ChaoticLoadBalancer balances across all tables
siteConfig.put(Property.TABLE_LOAD_BALANCER.getKey(), ChaoticLoadBalancer.class.getName());
cfg.setSiteConfig(siteConfig);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java
index ac0ceb3..becbe9b 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java
@@ -50,10 +50,12 @@ import org.junit.Test;
public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
+ private static final int NUM_SPLITS = 50;
+
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_MAXMEM, "10K");
- cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+ cfg.setProperty(Property.TSERV_MAXMEM, "1K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "50ms");
cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 3,
MemoryUnit.BYTE);
}
@@ -67,15 +69,14 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
public void simpleBalancerFairness() throws Exception {
try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
c.tableOperations().create("test_ingest");
- c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
- "10K");
+ c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "1K");
c.tableOperations().create("unused");
- TreeSet<Text> splits = TestIngest.getSplitPoints(0, 10000000, 500);
+ TreeSet<Text> splits = TestIngest.getSplitPoints(0, 10000000, NUM_SPLITS);
log.info("Creating {} splits", splits.size());
c.tableOperations().addSplits("unused", splits);
List<String> tservers = c.instanceOperations().getTabletServers();
TestIngest.IngestParams params = new TestIngest.IngestParams(getClientProperties());
- params.rows = 50000;
+ params.rows = 5000;
TestIngest.ingest(c, params);
c.tableOperations().flush("test_ingest", null, null, false);
sleepUninterruptibly(45, TimeUnit.SECONDS);
@@ -83,7 +84,7 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
MasterMonitorInfo stats = null;
int unassignedTablets = 1;
- for (int i = 0; unassignedTablets > 0 && i < 10; i++) {
+ for (int i = 0; unassignedTablets > 0 && i < 20; i++) {
MasterClientService.Iface client = null;
while (true) {
try {
@@ -107,7 +108,7 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
}
}
- assertEquals("Unassigned tablets were not assigned within 30 seconds", 0, unassignedTablets);
+ assertEquals("Unassigned tablets were not assigned within 60 seconds", 0, unassignedTablets);
// Compute online tablets per tserver
List<Integer> counts = new ArrayList<>();