You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2020/01/31 13:48:20 UTC
[accumulo] 02/02: Fix #1482 Increase timeouts on some slow ITs
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 2c1c72f5cf4375b8afdd3662783760d8ab62e5bf
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Fri Jan 31 07:47:26 2020 -0500
Fix #1482 Increase timeouts on some slow ITs
Stabilize some ITs by increasing timeouts to values which cause them to
pass more frequently on modest developer hardware and adding a timeout
to one that was missing it.
Also backport the removal of some flakey performance-related tests,
which were already removed in newer branches.
---
.../org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../test/RecoveryCompactionsAreFlushesIT.java | 2 +-
.../org/apache/accumulo/test/WaitForBalanceIT.java | 2 +-
.../accumulo/test/functional/ConstraintIT.java | 2 +-
.../test/functional/MetadataMaxFilesIT.java | 2 +-
.../accumulo/test/master/SuspendedTabletsIT.java | 9 +-
.../test/performance/RollWALPerformanceIT.java | 135 ---------------------
.../performance/metadata/FastBulkImportIT.java | 113 -----------------
.../accumulo/test/proxy/ProxyDurabilityIT.java | 2 +-
.../replication/MultiInstanceReplicationIT.java | 2 +-
10 files changed, 14 insertions(+), 257 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
index da22fa8..83815ec 100644
--- a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
@@ -59,7 +59,7 @@ import org.junit.Test;
public class ExistingMacIT extends ConfigurableMacBase {
@Override
public int defaultTimeoutSeconds() {
- return 2 * 60;
+ return 4 * 60;
}
@Override
diff --git a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
index 4e11274..255b228 100644
--- a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
@@ -46,7 +46,7 @@ public class RecoveryCompactionsAreFlushesIT extends AccumuloClusterHarness {
@Override
public int defaultTimeoutSeconds() {
- return 60;
+ return 180;
}
@Override
diff --git a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
index 45e307d..a1625bc 100644
--- a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java
@@ -43,7 +43,7 @@ public class WaitForBalanceIT extends ConfigurableMacBase {
@Override
public int defaultTimeoutSeconds() {
- return 60;
+ return 120;
}
@Test
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java
index d0b2540..5d1b511 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@ -51,7 +51,7 @@ public class ConstraintIT extends AccumuloClusterHarness {
@Override
protected int defaultTimeoutSeconds() {
- return 30;
+ return 60;
}
@Test
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
index b4b04d8..1cc9aaf 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
@@ -56,7 +56,7 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
@Override
protected int defaultTimeoutSeconds() {
- return 4 * 60;
+ return 5 * 60;
}
@Test
diff --git a/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java b/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java
index c86ecf0..b1009df 100644
--- a/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java
@@ -84,6 +84,11 @@ public class SuspendedTabletsIT extends ConfigurableMacBase {
public static final int TABLETS = 100;
@Override
+ protected int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration fsConf) {
cfg.setProperty(Property.TABLE_SUSPEND_DURATION, SUSPEND_DURATION + "ms");
cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
@@ -273,8 +278,8 @@ public class SuspendedTabletsIT extends ConfigurableMacBase {
assertTrue(recoverTime - killTime >= NANOSECONDS.convert(SUSPEND_DURATION, MILLISECONDS));
}
- private static interface TServerKiller {
- public void eliminateTabletServers(ClientContext ctx, TabletLocations locs, int count)
+ private interface TServerKiller {
+ void eliminateTabletServers(ClientContext ctx, TabletLocations locs, int count)
throws Exception;
}
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
deleted file mode 100644
index 8c5d1ab..0000000
--- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
-
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
-import org.apache.accumulo.test.categories.PerformanceTests;
-import org.apache.accumulo.test.continuous.ContinuousIngest;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({MiniClusterOnlyTests.class, PerformanceTests.class})
-public class RollWALPerformanceIT extends ConfigurableMacBase {
-
- @BeforeClass
- static public void checkMR() {
- assumeFalse(IntegrationTestMapReduce.isMapReduce());
- }
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1");
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
- cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100");
- cfg.setProperty(Property.GC_FILE_ARCHIVE, "false");
- cfg.setProperty(Property.GC_CYCLE_START, "1s");
- cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
- cfg.useMiniDFS(true);
- }
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 20 * 60;
- }
-
- private long ingest() throws Exception {
- final Connector c = getConnector();
- final String tableName = getUniqueNames(1)[0];
-
- log.info("Creating the table");
- c.tableOperations().create(tableName);
-
- log.info("Splitting the table");
- final long SPLIT_COUNT = 100;
- final long distance = Long.MAX_VALUE / SPLIT_COUNT;
- final SortedSet<Text> splits = new TreeSet<>();
- for (int i = 1; i < SPLIT_COUNT; i++) {
- splits.add(new Text(String.format("%016x", i * distance)));
- }
- c.tableOperations().addSplits(tableName, splits);
-
- log.info("Waiting for balance");
- c.instanceOperations().waitForBalance();
-
- final Instance inst = c.getInstance();
-
- log.info("Starting ingest");
- final long start = System.currentTimeMillis();
- final String args[] = {"-i", inst.getInstanceName(), "-z", inst.getZooKeepers(), "-u", "root",
- "-p", ROOT_PASSWORD, "--batchThreads", "2", "--table", tableName, "--num",
- Long.toString(50 * 1000), // 50K 100 byte entries
- };
-
- ContinuousIngest.main(args);
- final long result = System.currentTimeMillis() - start;
- log.debug(String.format("Finished in %,d ms", result));
- log.debug("Dropping table");
- c.tableOperations().delete(tableName);
- return result;
- }
-
- private long getAverage() throws Exception {
- final int REPEAT = 3;
- long totalTime = 0;
- for (int i = 0; i < REPEAT; i++) {
- totalTime += ingest();
- }
- return totalTime / REPEAT;
- }
-
- private void testWalPerformanceOnce() throws Exception {
- // get time with a small WAL, which will cause many WAL roll-overs
- long avg1 = getAverage();
- // use a bigger WAL max size to eliminate WAL roll-overs
- Connector c = getConnector();
- c.instanceOperations().setProperty(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1G");
- c.tableOperations().flush(MetadataTable.NAME, null, null, true);
- c.tableOperations().flush(RootTable.NAME, null, null, true);
- getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
- getCluster().start();
- long avg2 = getAverage();
- log.info(String.format("Average run time with small WAL %,d with large WAL %,d", avg1, avg2));
- assertTrue(avg1 > avg2);
- double percent = (100. * avg1) / avg2;
- log.info(String.format("Percent of large log: %.2f%%", percent));
- assertTrue(percent < 125.);
- }
-
- @Test
- public void testWalPerformance() throws Exception {
- testWalPerformanceOnce();
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java b/test/src/main/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
deleted file mode 100644
index 4bd301c..0000000
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance.metadata;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
-
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.FileOperations;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.RFile;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-// ACCUMULO-3327
-public class FastBulkImportIT extends ConfigurableMacBase {
-
- @BeforeClass
- static public void checkMR() {
- assumeFalse(IntegrationTestMapReduce.isMapReduce());
- }
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 120;
- }
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setNumTservers(3);
- cfg.setProperty(Property.TSERV_BULK_ASSIGNMENT_THREADS, "5");
- cfg.setProperty(Property.TSERV_BULK_PROCESS_THREADS, "5");
- cfg.setProperty(Property.TABLE_MAJC_RATIO, "9999");
- cfg.setProperty(Property.TABLE_FILE_MAX, "9999");
- cfg.setProperty(Property.TABLE_DURABILITY, "none");
- }
-
- @Test
- public void test() throws Exception {
- log.info("Creating table");
- final String tableName = getUniqueNames(1)[0];
- final Connector c = getConnector();
- c.tableOperations().create(tableName);
- log.info("Adding splits");
- SortedSet<Text> splits = new TreeSet<>();
- for (int i = 1; i < 0xfff; i += 7) {
- splits.add(new Text(Integer.toHexString(i)));
- }
- c.tableOperations().addSplits(tableName, splits);
-
- log.info("Creating lots of bulk import files");
- FileSystem fs = getCluster().getFileSystem();
- Path basePath = getCluster().getTemporaryPath();
- CachedConfiguration.setInstance(fs.getConf());
-
- Path base = new Path(basePath, "testBulkFail_" + tableName);
- fs.delete(base, true);
- fs.mkdirs(base);
- Path bulkFailures = new Path(base, "failures");
- Path files = new Path(base, "files");
- fs.mkdirs(bulkFailures);
- fs.mkdirs(files);
- for (int i = 0; i < 100; i++) {
- FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder()
- .forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf())
- .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build();
- writer.startDefaultLocalityGroup();
- for (int j = 0x100; j < 0xfff; j += 3) {
- writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
- }
- writer.close();
- }
- log.info("Waiting for balance");
- c.instanceOperations().waitForBalance();
-
- log.info("Bulk importing files");
- long now = System.currentTimeMillis();
- c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), true);
- double diffSeconds = (System.currentTimeMillis() - now) / 1000.;
- log.info(String.format("Import took %.2f seconds", diffSeconds));
- assertTrue(diffSeconds < 30);
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java b/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
index e1d5291..11e940b 100644
--- a/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
@@ -63,7 +63,7 @@ public class ProxyDurabilityIT extends ConfigurableMacBase {
@Override
protected int defaultTimeoutSeconds() {
- return 120;
+ return 240;
}
@Override
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java b/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
index ddf115a..aea9dd1 100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
@@ -84,7 +84,7 @@ public class MultiInstanceReplicationIT extends ConfigurableMacBase {
@Override
public int defaultTimeoutSeconds() {
- return 10 * 60;
+ return 15 * 60;
}
@Before