You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2020/12/10 21:27:09 UTC

[accumulo] branch main updated: Stabilize some build elements (#1835)

This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 12395ee  Stabilize some build elements (#1835)
12395ee is described below

commit 12395eeec06bc88d9a530ccc9927d924eb8108e8
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Thu Dec 10 16:26:57 2020 -0500

    Stabilize some build elements (#1835)
    
    * Update to exec-maven-plugin 3.0.0
    * Make main classes public
    * Remove ERROR about native library loading when an optional setting is
      not configured
    * Remove ERROR from metrics that occurs before compaction manager is
      initialized
    * Scale back the resources required to run WaitForBalanceIT and
      SimpleBalancerFairnessIT, to improve their stability
    * Fix compaction threads dying in some tests because of mixconfigured
      compaction checking delay time (must be greater than 1ms; some tests
      were setting it to 0, which was a non-sensical setting)
---
 .../apache/accumulo/core/conf/ClientConfigGenerate.java |  2 +-
 .../apache/accumulo/core/conf/ConfigurationDocGen.java  |  2 +-
 pom.xml                                                 |  3 +--
 .../java/org/apache/accumulo/tserver/NativeMap.java     |  5 ++++-
 .../tserver/metrics/TabletServerMetricsUtil.java        |  8 +++++---
 .../java/org/apache/accumulo/test/WaitForBalanceIT.java |  4 +++-
 .../functional/BalanceInPresenceOfOfflineTableIT.java   |  2 +-
 .../accumulo/test/functional/BatchScanSplitIT.java      |  2 +-
 .../apache/accumulo/test/functional/BinaryStressIT.java |  4 ++--
 .../accumulo/test/functional/ChaoticBalancerIT.java     |  2 +-
 .../test/functional/SimpleBalancerFairnessIT.java       | 17 +++++++++--------
 11 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java b/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
index 8a1f74f..0ad0a8a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
@@ -32,7 +32,7 @@ import com.google.common.collect.Sets;
  * Generates client-properties.md for documentation on Accumulo website and
  * accumulo-client.properties for Accumulo distribution tarball
  */
-class ClientConfigGenerate {
+public class ClientConfigGenerate {
 
   private abstract class Format {
 
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index b4df567..5fd3ec0 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@ -28,7 +28,7 @@ import java.util.TreeMap;
  * This class generates documentation to inform users of the available configuration properties in a
  * presentable form.
  */
-class ConfigurationDocGen {
+public class ConfigurationDocGen {
   private PrintStream doc;
   private final TreeMap<String,Property> sortedProps = new TreeMap<>();
 
diff --git a/pom.xml b/pom.xml
index 185bc0a..1fc6e14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -988,10 +988,9 @@
           <version>3.0.0</version>
         </plugin>
         <plugin>
-          <!-- version 1.6.0 is broken; see https://github.com/mojohaus/exec-maven-plugin/issues/75 -->
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.5.0</version>
+          <version>3.0.0</version>
         </plugin>
         <plugin>
           <groupId>net.revelc.code</groupId>
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java
index ea29495..d3d67c7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/NativeMap.java
@@ -87,7 +87,10 @@ public class NativeMap implements Iterable<Map.Entry<Key,Value>> {
 
     // Check LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on Mac)
     if (!isLoaded()) {
-      log.error("Tried and failed to load Accumulo native library from {}", accumuloNativeLibDirs);
+      if (accumuloNativeLibDirs != null) {
+        log.error("Tried and failed to load Accumulo native library from {}",
+            accumuloNativeLibDirs);
+      }
       String ldLibraryPath = System.getProperty("java.library.path");
       try {
         System.loadLibrary("accumulo");
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
index b3ac078..022285c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
@@ -91,11 +91,13 @@ public class TabletServerMetricsUtil {
   }
 
   public int getMajorCompactions() {
-    return tserver.getCompactionManager().getCompactionsRunning();
+    var mgr = tserver.getCompactionManager();
+    return mgr == null ? 0 : mgr.getCompactionsRunning();
   }
 
   public int getMajorCompactionsQueued() {
-    return tserver.getCompactionManager().getCompactionsQueued();
+    var mgr = tserver.getCompactionManager();
+    return mgr == null ? 0 : mgr.getCompactionsQueued();
   }
 
   public int getMinorCompactions() {
@@ -117,7 +119,7 @@ public class TabletServerMetricsUtil {
   }
 
   public int getOnlineCount() {
-    return tserver.getOnlineTablets().values().size();
+    return tserver.getOnlineTablets().size();
   }
 
   public int getOpeningCount() {
diff --git a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
index 9ed2d97..2355b6b 100644
--- a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
@@ -46,6 +46,8 @@ import com.google.common.collect.Iterators;
 
 public class WaitForBalanceIT extends ConfigurableMacBase {
 
+  private static final int NUM_SPLITS = 50;
+
   @Override
   public int defaultTimeoutSeconds() {
     return 120;
@@ -62,7 +64,7 @@ public class WaitForBalanceIT extends ConfigurableMacBase {
       c.tableOperations().create(tableName);
       c.instanceOperations().waitForBalance();
       final SortedSet<Text> partitionKeys = new TreeSet<>();
-      for (int i = 0; i < 1000; i++) {
+      for (int i = 0; i < NUM_SPLITS; i++) {
         partitionKeys.add(new Text("" + i));
       }
       c.tableOperations().addSplits(tableName, partitionKeys);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java
index 6a5e868..c19924b 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java
@@ -71,7 +71,7 @@ public class BalanceInPresenceOfOfflineTableIT extends AccumuloClusterHarness {
   public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
     Map<String,String> siteConfig = cfg.getSiteConfig();
     siteConfig.put(Property.TSERV_MAXMEM.getKey(), "10K");
-    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
     cfg.setSiteConfig(siteConfig);
     // ensure we have two tservers
     if (cfg.getNumTservers() < 2) {
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index 7bef19c..e8773d9 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@ -50,7 +50,7 @@ public class BatchScanSplitIT extends AccumuloClusterHarness {
 
   @Override
   public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-    cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+    cfg.setProperty(Property.TSERV_MAJC_DELAY, "50ms");
   }
 
   @Override
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
index 915c75b..3e55044 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@ -55,7 +55,7 @@ public class BinaryStressIT extends AccumuloClusterHarness {
   public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
     cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
     cfg.setProperty(Property.TSERV_MAXMEM, "50K");
-    cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+    cfg.setProperty(Property.TSERV_MAJC_DELAY, "50ms");
   }
 
   private String majcDelay, maxMem;
@@ -71,7 +71,7 @@ public class BinaryStressIT extends AccumuloClusterHarness {
       majcDelay = conf.get(Property.TSERV_MAJC_DELAY.getKey());
       maxMem = conf.get(Property.TSERV_MAXMEM.getKey());
 
-      iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "0");
+      iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
       iops.setProperty(Property.TSERV_MAXMEM.getKey(), "50K");
 
       getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java
index ba362c0..204eedb 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java
@@ -42,7 +42,7 @@ public class ChaoticBalancerIT extends AccumuloClusterHarness {
   public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
     Map<String,String> siteConfig = cfg.getSiteConfig();
     siteConfig.put(Property.TSERV_MAXMEM.getKey(), "10K");
-    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
     // ChaoticLoadBalancer balances across all tables
     siteConfig.put(Property.TABLE_LOAD_BALANCER.getKey(), ChaoticLoadBalancer.class.getName());
     cfg.setSiteConfig(siteConfig);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java
index ac0ceb3..becbe9b 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java
@@ -50,10 +50,12 @@ import org.junit.Test;
 
 public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
 
+  private static final int NUM_SPLITS = 50;
+
   @Override
   public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-    cfg.setProperty(Property.TSERV_MAXMEM, "10K");
-    cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+    cfg.setProperty(Property.TSERV_MAXMEM, "1K");
+    cfg.setProperty(Property.TSERV_MAJC_DELAY, "50ms");
     cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 3,
         MemoryUnit.BYTE);
   }
@@ -67,15 +69,14 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
   public void simpleBalancerFairness() throws Exception {
     try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
       c.tableOperations().create("test_ingest");
-      c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
-          "10K");
+      c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "1K");
       c.tableOperations().create("unused");
-      TreeSet<Text> splits = TestIngest.getSplitPoints(0, 10000000, 500);
+      TreeSet<Text> splits = TestIngest.getSplitPoints(0, 10000000, NUM_SPLITS);
       log.info("Creating {} splits", splits.size());
       c.tableOperations().addSplits("unused", splits);
       List<String> tservers = c.instanceOperations().getTabletServers();
       TestIngest.IngestParams params = new TestIngest.IngestParams(getClientProperties());
-      params.rows = 50000;
+      params.rows = 5000;
       TestIngest.ingest(c, params);
       c.tableOperations().flush("test_ingest", null, null, false);
       sleepUninterruptibly(45, TimeUnit.SECONDS);
@@ -83,7 +84,7 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
 
       MasterMonitorInfo stats = null;
       int unassignedTablets = 1;
-      for (int i = 0; unassignedTablets > 0 && i < 10; i++) {
+      for (int i = 0; unassignedTablets > 0 && i < 20; i++) {
         MasterClientService.Iface client = null;
         while (true) {
           try {
@@ -107,7 +108,7 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase {
         }
       }
 
-      assertEquals("Unassigned tablets were not assigned within 30 seconds", 0, unassignedTablets);
+      assertEquals("Unassigned tablets were not assigned within 60 seconds", 0, unassignedTablets);
 
       // Compute online tablets per tserver
       List<Integer> counts = new ArrayList<>();