You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2019/09/17 15:49:07 UTC
[accumulo] branch master updated: Move performance tests to
accumulo-testing #1200 (#1264)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 9d78365 Move performance tests to accumulo-testing #1200 (#1264)
9d78365 is described below
commit 9d78365d8f4d7a97a7ca4f104fa107e1f900e5f9
Author: Laura Schanno <lb...@gmail.com>
AuthorDate: Tue Sep 17 11:49:01 2019 -0400
Move performance tests to accumulo-testing #1200 (#1264)
* Move the following performance tests to accumulo-testing:
* ManySplitsIT, BalanceFasterIT, DeleteTableDuringSplitIT, DurabilityIT, RollWALPerformanceIT
* Remove the PerformanceTest annotation from DurabilityIT since the
performance components were moved to DurabilityWriteSpeedPT.
* Remove PerformanceTests.java since it is no longer being used by any
classes.
* Delete ContinousIngest.java and ContinousOpts.java. These classes are no longer in use.
---
.../accumulo/test/categories/PerformanceTests.java | 27 --
.../org/apache/accumulo/test/BalanceFasterIT.java | 113 ---------
.../java/org/apache/accumulo/test/ManySplitIT.java | 113 ---------
.../test/functional/DeleteTableDuringSplitIT.java | 119 ---------
.../accumulo/test/functional/DurabilityIT.java | 20 +-
.../test/performance/ContinuousIngest.java | 272 ---------------------
.../accumulo/test/performance/ContinuousOpts.java | 57 -----
.../test/performance/RollWALPerformanceIT.java | 123 ----------
8 files changed, 9 insertions(+), 835 deletions(-)
diff --git a/start/src/main/java/org/apache/accumulo/test/categories/PerformanceTests.java b/start/src/main/java/org/apache/accumulo/test/categories/PerformanceTests.java
deleted file mode 100644
index 5a0bd82..0000000
--- a/start/src/main/java/org/apache/accumulo/test/categories/PerformanceTests.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.categories;
-
-/**
- * Annotate integration tests which test performance-related aspects of Accumulo or are sensitive to
- * timings and hardware capabilities.
- * <p>
- * Intended to be used with the JUnit Category annotation on integration test classes. The Category
- * annotation should be placed at the class-level. Test class names should still be suffixed with
- * 'IT' as the rest of the integration tests.
- */
-public interface PerformanceTests {}
diff --git a/test/src/main/java/org/apache/accumulo/test/BalanceFasterIT.java b/test/src/main/java/org/apache/accumulo/test/BalanceFasterIT.java
deleted file mode 100644
index 339b7d2..0000000
--- a/test/src/main/java/org/apache/accumulo/test/BalanceFasterIT.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
-import org.apache.accumulo.test.categories.PerformanceTests;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-// ACCUMULO-2952
-@Category({MiniClusterOnlyTests.class, PerformanceTests.class})
-public class BalanceFasterIT extends ConfigurableMacBase {
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setNumTservers(3);
- }
-
- @BeforeClass
- public static void checkMR() {
- assumeFalse(IntegrationTestMapReduce.isMapReduce());
- }
-
- @Test(timeout = 90 * 1000)
- public void test() throws Exception {
- // create a table, add a bunch of splits
- String tableName = getUniqueNames(1)[0];
- try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
- client.tableOperations().create(tableName);
- SortedSet<Text> splits = new TreeSet<>();
- for (int i = 0; i < 1000; i++) {
- splits.add(new Text("" + i));
- }
- client.tableOperations().addSplits(tableName, splits);
- // give a short wait for balancing
- sleepUninterruptibly(10, TimeUnit.SECONDS);
- // find out where the tablets are
- Iterator<Integer> i;
- try (Scanner s = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
- s.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
- s.setRange(MetadataSchema.TabletsSection.getRange());
- Map<String,Integer> counts = new HashMap<>();
- while (true) {
- int total = 0;
- counts.clear();
- for (Entry<Key,Value> kv : s) {
- String host = kv.getValue().toString();
- if (!counts.containsKey(host))
- counts.put(host, 0);
- counts.put(host, counts.get(host) + 1);
- total++;
- }
- // are enough tablets online?
- if (total > 1000)
- break;
- }
-
- // should be on all three servers
- assertEquals(3, counts.size());
- // and distributed evenly
- i = counts.values().iterator();
- }
-
- int a = i.next();
- int b = i.next();
- int c = i.next();
- assertTrue(Math.abs(a - b) < 3);
- assertTrue(Math.abs(a - c) < 3);
- assertTrue(a > 330);
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/ManySplitIT.java b/test/src/main/java/org/apache/accumulo/test/ManySplitIT.java
deleted file mode 100644
index 9b30c17..0000000
--- a/test/src/main/java/org/apache/accumulo/test/ManySplitIT.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
-
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.fate.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.MemoryUnit;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
-import org.apache.accumulo.test.categories.PerformanceTests;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({MiniClusterOnlyTests.class, PerformanceTests.class})
-public class ManySplitIT extends ConfigurableMacBase {
-
- final int SPLITS = 10_000;
-
- @BeforeClass
- public static void checkMR() {
- assumeFalse(IntegrationTestMapReduce.isMapReduce());
- }
-
- @Test(timeout = 4 * 60 * 1000)
- public void test() throws Exception {
- assumeFalse(IntegrationTestMapReduce.isMapReduce());
-
- final String tableName = getUniqueNames(1)[0];
-
- try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
-
- log.info("Creating table");
- log.info("splitting metadata table");
- client.tableOperations().create(tableName);
- SortedSet<Text> splits = new TreeSet<>();
- for (byte b : "123456789abcde".getBytes(UTF_8)) {
- splits.add(new Text(new byte[] {'1', ';', b}));
- }
- client.tableOperations().addSplits(MetadataTable.NAME, splits);
- splits.clear();
- for (int i = 0; i < SPLITS; i++) {
- splits.add(new Text(Integer.toHexString(i)));
- }
- log.info("Adding splits");
- // print out the number of splits so we have some idea of what's going on
- final AtomicBoolean stop = new AtomicBoolean(false);
- Thread t = new Thread() {
- @Override
- public void run() {
- while (!stop.get()) {
- UtilWaitThread.sleep(1000);
- try {
- log.info("splits: " + client.tableOperations().listSplits(tableName).size());
- } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
- };
- t.start();
- long now = System.currentTimeMillis();
- client.tableOperations().addSplits(tableName, splits);
- long diff = System.currentTimeMillis() - now;
- double splitsPerSec = SPLITS / (diff / 1000.);
- log.info("Done: {} splits per second", splitsPerSec);
- assertTrue("splits created too slowly", splitsPerSec > 100);
- stop.set(true);
- t.join();
- }
- }
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hdfs) {
- cfg.setNumTservers(1);
- cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 2,
- MemoryUnit.BYTE);
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
deleted file mode 100644
index 2cc58ee..0000000
--- a/test/src/main/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.Future;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.util.SimpleThreadPool;
-import org.apache.accumulo.fate.util.UtilWaitThread;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
-import org.apache.accumulo.test.categories.PerformanceTests;
-import org.apache.accumulo.test.categories.StandaloneCapableClusterTests;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-// ACCUMULO-2361
-@Category({StandaloneCapableClusterTests.class, PerformanceTests.class})
-public class DeleteTableDuringSplitIT extends AccumuloClusterHarness {
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 15 * 60;
- }
-
- @Test
- public void test() throws Exception {
-
- try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
-
- // 96 invocations, 8 at a time
- int batches = 12, batchSize = 8;
- String[] tableNames = getUniqueNames(batches * batchSize);
- // make a bunch of tables
- for (String tableName : tableNames) {
- client.tableOperations().create(tableName);
- }
- final SortedSet<Text> splits = new TreeSet<>();
- for (byte i = 0; i < 100; i++) {
- splits.add(new Text(new byte[] {0, 0, i}));
- }
-
- List<Future<?>> results = new ArrayList<>();
- List<Runnable> tasks = new ArrayList<>();
- SimpleThreadPool es = new SimpleThreadPool(batchSize * 2, "concurrent-api-requests");
- for (String tableName : tableNames) {
- final String finalName = tableName;
- tasks.add(new Runnable() {
- @Override
- public void run() {
- try {
- client.tableOperations().addSplits(finalName, splits);
- } catch (TableNotFoundException ex) {
- // expected, ignore
- } catch (Exception ex) {
- throw new RuntimeException(finalName, ex);
- }
- }
- });
- tasks.add(new Runnable() {
- @Override
- public void run() {
- try {
- UtilWaitThread.sleep(500);
- client.tableOperations().delete(finalName);
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
- });
- }
- Iterator<Runnable> itr = tasks.iterator();
- for (int batch = 0; batch < batches; batch++) {
- for (int i = 0; i < batchSize; i++) {
- Future<?> f = es.submit(itr.next());
- results.add(f);
- f = es.submit(itr.next());
- results.add(f);
- }
- for (Future<?> f : results) {
- f.get();
- }
- results.clear();
- }
- // Shut down the ES
- List<Runnable> queued = es.shutdownNow();
- assertTrue("Had more tasks to run", queued.isEmpty());
- assertFalse("Had more tasks that needed to be submitted", itr.hasNext());
- for (String tableName : tableNames) {
- assertFalse(client.tableOperations().exists(tableName));
- }
- }
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
index 496e2d1..69c37f5 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@ -16,14 +16,7 @@
*/
package org.apache.accumulo.test.functional;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
+import com.google.common.collect.Iterators;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
@@ -36,7 +29,6 @@ import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.miniclusterImpl.ProcessReference;
import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
-import org.apache.accumulo.test.categories.PerformanceTests;
import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.RawLocalFileSystem;
@@ -44,9 +36,15 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import com.google.common.collect.Iterators;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeFalse;
-@Category({MiniClusterOnlyTests.class, PerformanceTests.class})
+@Category({MiniClusterOnlyTests.class})
public class DurabilityIT extends ConfigurableMacBase {
@Override
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java b/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java
deleted file mode 100644
index 3e1d497..0000000
--- a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.zip.CRC32;
-import java.util.zip.Checksum;
-
-import org.apache.accumulo.core.cli.ClientOpts;
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.htrace.TraceScope;
-import org.apache.htrace.wrappers.TraceProxy;
-
-import com.beust.jcommander.Parameter;
-
-public class ContinuousIngest {
-
- private static final byte[] EMPTY_BYTES = new byte[0];
-
- private static List<ColumnVisibility> visibilities;
-
- private static void initVisibilities(ContinuousOpts opts) throws Exception {
- if (opts.visFile == null) {
- visibilities = Collections.singletonList(new ColumnVisibility());
- return;
- }
-
- visibilities = new ArrayList<>();
-
- FileSystem fs = FileSystem.get(new Configuration());
- BufferedReader in =
- new BufferedReader(new InputStreamReader(fs.open(new Path(opts.visFile)), UTF_8));
-
- String line;
-
- while ((line = in.readLine()) != null) {
- visibilities.add(new ColumnVisibility(line));
- }
-
- in.close();
- }
-
- private static ColumnVisibility getVisibility(Random rand) {
- return visibilities.get(rand.nextInt(visibilities.size()));
- }
-
- static class TestOpts extends ClientOpts {
- @Parameter(names = "--table", description = "table to use")
- String tableName = "ci";
- }
-
- public static void main(String[] args) throws Exception {
-
- ContinuousOpts opts = new ContinuousOpts();
- TestOpts clientOpts = new TestOpts();
- try (TraceScope clientSpan =
- clientOpts.parseArgsAndTrace(ContinuousIngest.class.getName(), args, opts)) {
-
- initVisibilities(opts);
-
- if (opts.min < 0 || opts.max < 0 || opts.max <= opts.min) {
- throw new IllegalArgumentException("bad min and max");
- }
- try (AccumuloClient client = Accumulo.newClient().from(clientOpts.getClientProps()).build()) {
-
- if (!client.tableOperations().exists(clientOpts.tableName)) {
- throw new TableNotFoundException(null, clientOpts.tableName,
- "Consult the README and create the table before starting ingest.");
- }
-
- BatchWriter bw = client.createBatchWriter(clientOpts.tableName);
- bw = TraceProxy.trace(bw, TraceUtil.countSampler(1024));
-
- Random r = new SecureRandom();
-
- byte[] ingestInstanceId = UUID.randomUUID().toString().getBytes(UTF_8);
-
- System.out.printf("UUID %d %s%n", System.currentTimeMillis(),
- new String(ingestInstanceId, UTF_8));
-
- long count = 0;
- final int flushInterval = 1000000;
- final int maxDepth = 25;
-
- // always want to point back to flushed data. This way the previous item should
- // always exist in accumulo when verifying data. To do this make insert N point
- // back to the row from insert (N - flushInterval). The array below is used to keep
- // track of this.
- long[] prevRows = new long[flushInterval];
- long[] firstRows = new long[flushInterval];
- int[] firstColFams = new int[flushInterval];
- int[] firstColQuals = new int[flushInterval];
-
- long lastFlushTime = System.currentTimeMillis();
-
- out: while (true) {
- // generate first set of nodes
- ColumnVisibility cv = getVisibility(r);
-
- for (int index = 0; index < flushInterval; index++) {
- long rowLong = genLong(opts.min, opts.max, r);
- prevRows[index] = rowLong;
- firstRows[index] = rowLong;
-
- int cf = r.nextInt(opts.maxColF);
- int cq = r.nextInt(opts.maxColQ);
-
- firstColFams[index] = cf;
- firstColQuals[index] = cq;
-
- Mutation m =
- genMutation(rowLong, cf, cq, cv, ingestInstanceId, count, null, opts.checksum);
- count++;
- bw.addMutation(m);
- }
-
- lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
- if (count >= opts.num)
- break out;
-
- // generate subsequent sets of nodes that link to previous set of nodes
- for (int depth = 1; depth < maxDepth; depth++) {
- for (int index = 0; index < flushInterval; index++) {
- long rowLong = genLong(opts.min, opts.max, r);
- byte[] prevRow = genRow(prevRows[index]);
- prevRows[index] = rowLong;
- Mutation m = genMutation(rowLong, r.nextInt(opts.maxColF), r.nextInt(opts.maxColQ),
- cv, ingestInstanceId, count, prevRow, opts.checksum);
- count++;
- bw.addMutation(m);
- }
-
- lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
- if (count >= opts.num)
- break out;
- }
-
- // create one big linked list, this makes all of the first inserts
- // point to something
- for (int index = 0; index < flushInterval - 1; index++) {
- Mutation m = genMutation(firstRows[index], firstColFams[index], firstColQuals[index],
- cv, ingestInstanceId, count, genRow(prevRows[index + 1]), opts.checksum);
- count++;
- bw.addMutation(m);
- }
- lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
- if (count >= opts.num)
- break out;
- }
-
- bw.close();
- }
- }
- }
-
- private static long flush(BatchWriter bw, long count, final int flushInterval, long lastFlushTime)
- throws MutationsRejectedException {
- long t1 = System.currentTimeMillis();
- bw.flush();
- long t2 = System.currentTimeMillis();
- System.out.printf("FLUSH %d %d %d %d %d%n", t2, (t2 - lastFlushTime), (t2 - t1), count,
- flushInterval);
- lastFlushTime = t2;
- return lastFlushTime;
- }
-
- public static Mutation genMutation(long rowLong, int cfInt, int cqInt, ColumnVisibility cv,
- byte[] ingestInstanceId, long count, byte[] prevRow, boolean checksum) {
- // Adler32 is supposed to be faster, but according to wikipedia is not good for small data....
- // so used CRC32 instead
- CRC32 cksum = null;
-
- byte[] rowString = genRow(rowLong);
-
- byte[] cfString = FastFormat.toZeroPaddedString(cfInt, 4, 16, EMPTY_BYTES);
- byte[] cqString = FastFormat.toZeroPaddedString(cqInt, 4, 16, EMPTY_BYTES);
-
- if (checksum) {
- cksum = new CRC32();
- cksum.update(rowString);
- cksum.update(cfString);
- cksum.update(cqString);
- cksum.update(cv.getExpression());
- }
-
- Mutation m = new Mutation(new Text(rowString));
-
- m.put(new Text(cfString), new Text(cqString), cv,
- createValue(ingestInstanceId, count, prevRow, cksum));
- return m;
- }
-
- public static final long genLong(long min, long max, Random r) {
- return ((r.nextLong() & 0x7fffffffffffffffL) % (max - min)) + min;
- }
-
- static final byte[] genRow(long min, long max, Random r) {
- return genRow(genLong(min, max, r));
- }
-
- static final byte[] genRow(long rowLong) {
- return FastFormat.toZeroPaddedString(rowLong, 16, 16, EMPTY_BYTES);
- }
-
- private static Value createValue(byte[] ingestInstanceId, long count, byte[] prevRow,
- Checksum cksum) {
- int dataLen = ingestInstanceId.length + 16 + (prevRow == null ? 0 : prevRow.length) + 3;
- if (cksum != null)
- dataLen += 8;
- byte[] val = new byte[dataLen];
- System.arraycopy(ingestInstanceId, 0, val, 0, ingestInstanceId.length);
- int index = ingestInstanceId.length;
- val[index++] = ':';
- int added = FastFormat.toZeroPaddedString(val, index, count, 16, 16, EMPTY_BYTES);
- if (added != 16)
- throw new RuntimeException(" " + added);
- index += 16;
- val[index++] = ':';
- if (prevRow != null) {
- System.arraycopy(prevRow, 0, val, index, prevRow.length);
- index += prevRow.length;
- }
-
- val[index++] = ':';
-
- if (cksum != null) {
- cksum.update(val, 0, index);
- cksum.getValue();
- FastFormat.toZeroPaddedString(val, index, cksum.getValue(), 8, 16, EMPTY_BYTES);
- }
-
- // System.out.println("val "+new String(val));
-
- return new Value(val);
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousOpts.java b/test/src/main/java/org/apache/accumulo/test/performance/ContinuousOpts.java
deleted file mode 100644
index dfbfafa..0000000
--- a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousOpts.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance;
-
-import com.beust.jcommander.IStringConverter;
-import com.beust.jcommander.Parameter;
-
-/**
- * Common CLI arguments for the Continuous Ingest suite.
- */
-public class ContinuousOpts {
-
- public static class ShortConverter implements IStringConverter<Short> {
- @Override
- public Short convert(String value) {
- return Short.valueOf(value);
- }
- }
-
- @Parameter(names = "--min", description = "lowest random row number to use")
- long min = 0;
-
- @Parameter(names = "--max", description = "maximum random row number to use")
- long max = Long.MAX_VALUE;
-
- @Parameter(names = "--num", description = "the number of entries to ingest")
- long num = Long.MAX_VALUE;
-
- @Parameter(names = "--maxColF", description = "maximum column family value to use",
- converter = ShortConverter.class)
- short maxColF = Short.MAX_VALUE;
-
- @Parameter(names = "--maxColQ", description = "maximum column qualifier value to use",
- converter = ShortConverter.class)
- short maxColQ = Short.MAX_VALUE;
-
- @Parameter(names = "--addCheckSum", description = "turn on checksums")
- boolean checksum = false;
-
- @Parameter(names = "--visibilities",
- description = "read the visibilities to ingest with from a file")
- String visFile = null;
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
deleted file mode 100644
index 9d6d41e..0000000
--- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
-
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
-import org.apache.accumulo.test.categories.PerformanceTests;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({MiniClusterOnlyTests.class, PerformanceTests.class})
-public class RollWALPerformanceIT extends ConfigurableMacBase {
-
- @BeforeClass
- public static void checkMR() {
- assumeFalse(IntegrationTestMapReduce.isMapReduce());
- }
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1");
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
- cfg.setProperty(Property.TSERV_WALOG_MAX_REFERENCED, "100");
- cfg.setProperty(Property.GC_CYCLE_START, "1s");
- cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
- cfg.useMiniDFS(true);
- }
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 5 * 60;
- }
-
- private long ingest(AccumuloClient c) throws Exception {
- final String tableName = getUniqueNames(1)[0];
-
- log.info("Creating the table");
- c.tableOperations().create(tableName);
-
- log.info("Splitting the table");
- final long SPLIT_COUNT = 100;
- final long distance = Long.MAX_VALUE / SPLIT_COUNT;
- final SortedSet<Text> splits = new TreeSet<>();
- for (int i = 1; i < SPLIT_COUNT; i++) {
- splits.add(new Text(String.format("%016x", i * distance)));
- }
- c.tableOperations().addSplits(tableName, splits);
-
- log.info("Waiting for balance");
- c.instanceOperations().waitForBalance();
-
- log.info("Starting ingest");
- final long start = System.nanoTime();
- // Load 50K 100 byte entries
- ContinuousIngest.main(new String[] {"-c", cluster.getClientPropsPath(), "--table", tableName,
- "--num", Long.toString(50 * 1000)});
- final long result = System.nanoTime() - start;
- log.debug(String.format("Finished in %,d ns", result));
- log.debug("Dropping table");
- c.tableOperations().delete(tableName);
- return result;
- }
-
- private long getAverage(AccumuloClient c) throws Exception {
- final int REPEAT = 3;
- long totalTime = 0;
- for (int i = 0; i < REPEAT; i++) {
- totalTime += ingest(c);
- }
- return totalTime / REPEAT;
- }
-
- @Test
- public void testWalPerformanceOnce() throws Exception {
- try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
- // get time with a small WAL, which will cause many WAL roll-overs
- long avg1 = getAverage(c);
- // use a bigger WAL max size to eliminate WAL roll-overs
- c.instanceOperations().setProperty(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1G");
- c.tableOperations().flush(MetadataTable.NAME, null, null, true);
- c.tableOperations().flush(RootTable.NAME, null, null, true);
- getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
- getCluster().start();
- long avg2 = getAverage(c);
- log.info(String.format("Average run time with small WAL %,d with large WAL %,d", avg1, avg2));
- assertTrue(avg1 > avg2);
- double percent = (100. * avg1) / avg2;
- log.info(String.format("Percent of large log: %.2f%%", percent));
- }
- }
-
-}