You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2018/12/12 22:41:29 UTC
[accumulo] branch master updated: Removed unused code (#826)
This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new ee785ec Removed unused code (#826)
ee785ec is described below
commit ee785ec3584bdf4cc36bddfce1d69dbe3a6eeaf3
Author: Mike Walch <mw...@apache.org>
AuthorDate: Wed Dec 12 17:41:25 2018 -0500
Removed unused code (#826)
* Code was not used by Accumulo, unit tests or ITs
* If this code is ever brought back to life, it should probably
live in accumulo-testing repo or a personal repo
---
.../org/apache/accumulo/test/CreateRFiles.java | 99 -------
.../apache/accumulo/test/CreateRandomRFile.java | 102 -------
.../accumulo/test/FaultyConditionalWriter.java | 85 ------
.../org/apache/accumulo/test/IMMLGBenchmark.java | 197 -------------
.../java/org/apache/accumulo/test/ListTables.java | 35 ---
.../org/apache/accumulo/test/NullBatchWriter.java | 56 ----
.../apache/accumulo/test/QueryMetadataTable.java | 156 ----------
.../ReplicationTablesPrinterThread.java | 48 ----
.../test/util/memory/EstimateInMemMapOverhead.java | 50 ----
.../util/memory/InMemoryMapMemoryUsageCheck.java | 110 --------
.../util/memory/IntObjectMemoryUsageCheck.java | 65 -----
.../test/util/memory/MemoryUsageCheck.java | 69 -----
.../test/util/memory/MutationMemoryUsageCheck.java | 98 -------
.../test/util/memory/TextMemoryUsageCheck.java | 84 ------
.../test/util/metadata/MetadataBatchScan.java | 249 ----------------
.../test/util/nativemap/NativeMapConcurrency.java | 194 -------------
.../test/util/nativemap/NativeMapPerformance.java | 205 --------------
.../test/util/nativemap/NativeMapStress.java | 314 ---------------------
.../accumulo/test/util/tablet/WrongTablet.java | 59 ----
19 files changed, 2275 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/CreateRFiles.java b/test/src/main/java/org/apache/accumulo/test/CreateRFiles.java
deleted file mode 100644
index 4cefefa..0000000
--- a/test/src/main/java/org/apache/accumulo/test/CreateRFiles.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.cli.Help;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-public class CreateRFiles {
-
- private static final Logger log = LoggerFactory.getLogger(CreateRFiles.class);
-
- static class Opts extends Help {
-
- @Parameter(names = "--output", description = "the destiation directory")
- String outputDirectory;
-
- @Parameter(names = "--numThreads",
- description = "number of threads to use when generating files")
- int numThreads = 4;
-
- @Parameter(names = "--start", description = "the start number for test data")
- long start = 0;
-
- @Parameter(names = "--end", description = "the maximum number for test data")
- long end = 10 * 1000 * 1000;
-
- @Parameter(names = "--splits", description = "the number of splits in the data")
- long numsplits = 4;
- }
-
- public static void main(String[] args) {
- Opts opts = new Opts();
- opts.parseArgs(CreateRFiles.class.getName(), args);
-
- long splitSize = Math.round((opts.end - opts.start) / (double) opts.numsplits);
-
- long currStart = opts.start;
- long currEnd = opts.start + splitSize;
-
- ExecutorService threadPool = Executors.newFixedThreadPool(opts.numThreads);
-
- int count = 0;
- while (currEnd <= opts.end && currStart < currEnd) {
-
- final String tia = String.format(
- "--rfile %s/mf%05d --timestamp 1 --size 50 --random 56 --rows %d --start %d --user root",
- opts.outputDirectory, count, currEnd - currStart, currStart);
-
- Runnable r = new Runnable() {
-
- @Override
- public void run() {
- try {
- TestIngest.main(tia.split(" "));
- } catch (Exception e) {
- log.error("Could not run " + TestIngest.class.getName() + ".main using the input '"
- + tia + "'", e);
- }
- }
-
- };
-
- threadPool.execute(r);
-
- count++;
- currStart = currEnd;
- currEnd = Math.min(opts.end, currStart + splitSize);
- }
-
- threadPool.shutdown();
- while (!threadPool.isTerminated())
- try {
- threadPool.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/CreateRandomRFile.java b/test/src/main/java/org/apache/accumulo/test/CreateRandomRFile.java
deleted file mode 100644
index 0f10bab..0000000
--- a/test/src/main/java/org/apache/accumulo/test/CreateRandomRFile.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.io.IOException;
-import java.security.SecureRandom;
-import java.util.Arrays;
-import java.util.Random;
-
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.crypto.CryptoServiceFactory;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.RFileOperations;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-
-public class CreateRandomRFile {
- private static int num;
- private static String file;
-
- public static byte[] createValue(long rowid, int dataSize) {
- Random r = new SecureRandom();
- r.setSeed(rowid);
- byte value[] = new byte[dataSize];
-
- r.nextBytes(value);
-
- // transform to printable chars
- for (int j = 0; j < value.length; j++) {
- value[j] = (byte) (((0xff & value[j]) % 92) + ' ');
- }
-
- return value;
- }
-
- public static void main(String[] args) {
- if (args.length != 2) {
- System.err.println("Usage CreateRandomRFile <filename> <size>");
- System.exit(-1);
- }
- file = args[0];
- num = Integer.parseInt(args[1]);
- long rands[] = new long[num];
-
- Random r = new SecureRandom();
-
- for (int i = 0; i < rands.length; i++) {
- rands[i] = (r.nextLong() & 0x7fffffffffffffffL) % 10000000000L;
- }
-
- Arrays.sort(rands);
-
- Configuration conf = CachedConfiguration.getInstance();
- FileSKVWriter mfw;
- try {
- FileSystem fs = FileSystem.get(conf);
- mfw = new RFileOperations().newWriterBuilder()
- .forFile(file, fs, conf, CryptoServiceFactory.newDefaultInstance())
- .withTableConfiguration(DefaultConfiguration.getInstance()).build();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- for (int i = 0; i < rands.length; i++) {
- Text row = new Text(String.format("row_%010d", rands[i]));
- Key key = new Key(row);
-
- Value dv = new Value(createValue(rands[i], 40));
-
- try {
- mfw.append(key, dv);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- try {
- mfw.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/FaultyConditionalWriter.java b/test/src/main/java/org/apache/accumulo/test/FaultyConditionalWriter.java
deleted file mode 100644
index 968ced4..0000000
--- a/test/src/main/java/org/apache/accumulo/test/FaultyConditionalWriter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.ConditionalWriter;
-import org.apache.accumulo.core.data.ConditionalMutation;
-
-/**
- * A writer that will sometimes return unknown. When it returns unknown the condition may or may not
- * have been written.
- */
-public class FaultyConditionalWriter implements ConditionalWriter {
-
- private ConditionalWriter cw;
- private double up;
- private Random rand;
- private double wp;
-
- public FaultyConditionalWriter(ConditionalWriter cw, double unknownProbability,
- double writeProbability) {
- this.cw = cw;
- this.up = unknownProbability;
- this.wp = writeProbability;
- this.rand = new SecureRandom();
-
- }
-
- @Override
- public Iterator<Result> write(Iterator<ConditionalMutation> mutations) {
- ArrayList<Result> resultList = new ArrayList<>();
- ArrayList<ConditionalMutation> writes = new ArrayList<>();
-
- while (mutations.hasNext()) {
- ConditionalMutation cm = mutations.next();
- if (rand.nextDouble() <= up && rand.nextDouble() > wp)
- resultList.add(new Result(Status.UNKNOWN, cm, null));
- else
- writes.add(cm);
- }
-
- if (writes.size() > 0) {
- Iterator<Result> results = cw.write(writes.iterator());
-
- while (results.hasNext()) {
- Result result = results.next();
-
- if (rand.nextDouble() <= up && rand.nextDouble() <= wp)
- result = new Result(Status.UNKNOWN, result.getMutation(), result.getTabletServer());
- resultList.add(result);
- }
- }
- return resultList.iterator();
- }
-
- @Override
- public Result write(ConditionalMutation mutation) {
- return write(Collections.singleton(mutation).iterator()).next();
- }
-
- @Override
- public void close() {
- cw.close();
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/IMMLGBenchmark.java b/test/src/main/java/org/apache/accumulo/test/IMMLGBenchmark.java
deleted file mode 100644
index 4384e21..0000000
--- a/test/src/main/java/org/apache/accumulo/test/IMMLGBenchmark.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.core.util.Stat;
-import org.apache.hadoop.io.Text;
-
-import com.google.common.collect.Iterators;
-
-public class IMMLGBenchmark {
- public static void main(String[] args) throws Exception {
-
- AccumuloClient client = Accumulo.newClient().to("test16", "localhost").as("root", "secret")
- .build();
-
- int numlg = Integer.parseInt(args[0]);
-
- ArrayList<byte[]> cfset = new ArrayList<>();
-
- for (int i = 0; i < 32; i++) {
- cfset.add(String.format("%04x", i).getBytes());
- }
-
- Map<String,Stat> stats = new TreeMap<>();
-
- for (int i = 0; i < 5; i++) {
- runTest(client, numlg, cfset, i > 1 ? stats : null);
- System.out.println();
- }
-
- for (Entry<String,Stat> entry : stats.entrySet()) {
- System.out.printf("%20s : %6.2f\n", entry.getKey(), entry.getValue().mean());
- }
-
- }
-
- private static void runTest(AccumuloClient client, int numlg, ArrayList<byte[]> cfset,
- Map<String,Stat> stats) throws Exception {
- String table = "immlgb";
-
- try {
- client.tableOperations().delete(table);
- } catch (TableNotFoundException tnfe) {}
-
- client.tableOperations().create(table);
- client.tableOperations().setProperty(table, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(),
- "snappy");
-
- setupLocalityGroups(client, numlg, cfset, table);
-
- addStat(stats, "write", write(client, cfset, table));
- addStat(stats, "scan cf", scan(client, cfset, table, false));
- addStat(stats, "scan cf:cq", scan(client, cfset, table, true));
- // TODO time reading all data
-
- long t1 = System.currentTimeMillis();
- client.tableOperations().flush(table, null, null, true);
- long t2 = System.currentTimeMillis();
-
- addStat(stats, "flush", t2 - t1);
- }
-
- private static void addStat(Map<String,Stat> stats, String s, long wt) {
- System.out.println(s + ":" + wt);
-
- if (stats == null)
- return;
-
- Stat stat = stats.get(s);
- if (stat == null) {
- stat = new Stat();
- stats.put(s, stat);
- }
- stat.addStat(wt);
- }
-
- private static long scan(AccumuloClient client, ArrayList<byte[]> cfset, String table, boolean cq)
- throws TableNotFoundException {
- try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
-
- if (!cq)
- scanner.fetchColumnFamily(new Text(cfset.get(15)));
- else
- scanner.fetchColumn(new Text(cfset.get(15)), new Text(cfset.get(15)));
-
- long t1 = System.currentTimeMillis();
-
- Iterators.size(scanner.iterator());
-
- long t2 = System.currentTimeMillis();
-
- return t2 - t1;
- }
- }
-
- private static long write(AccumuloClient client, ArrayList<byte[]> cfset, String table)
- throws TableNotFoundException, MutationsRejectedException {
- Random rand = new SecureRandom();
-
- byte val[] = new byte[50];
-
- BatchWriter bw = client.createBatchWriter(table, new BatchWriterConfig());
-
- long t1 = System.currentTimeMillis();
-
- for (int i = 0; i < 1 << 15; i++) {
- byte[] row = FastFormat.toZeroPaddedString(abs(rand.nextLong()), 16, 16, new byte[0]);
-
- Mutation m = new Mutation(row);
- for (byte[] cf : cfset) {
- byte[] cq = FastFormat.toZeroPaddedString(rand.nextInt(1 << 16), 4, 16, new byte[0]);
- rand.nextBytes(val);
- m.put(cf, cq, val);
- }
-
- bw.addMutation(m);
- }
-
- bw.close();
-
- long t2 = System.currentTimeMillis();
-
- return t2 - t1;
- }
-
- private static void setupLocalityGroups(AccumuloClient client, int numlg, ArrayList<byte[]> cfset,
- String table) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- if (numlg > 1) {
- int numCF = cfset.size() / numlg;
- int gNum = 0;
-
- Iterator<byte[]> cfiter = cfset.iterator();
- Map<String,Set<Text>> groups = new HashMap<>();
- while (cfiter.hasNext()) {
- HashSet<Text> groupCols = new HashSet<>();
- for (int i = 0; i < numCF && cfiter.hasNext(); i++) {
- groupCols.add(new Text(cfiter.next()));
- }
-
- groups.put("lg" + (gNum++), groupCols);
- }
-
- client.tableOperations().setLocalityGroups(table, groups);
- client.tableOperations().offline(table);
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- client.tableOperations().online(table);
- }
- }
-
- public static long abs(long l) {
- l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE...
- if (l < 0)
- return 0;
- return l;
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/ListTables.java b/test/src/main/java/org/apache/accumulo/test/ListTables.java
deleted file mode 100644
index e52abb7..0000000
--- a/test/src/main/java/org/apache/accumulo/test/ListTables.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.clientImpl.Table;
-import org.apache.accumulo.core.clientImpl.Tables;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-
-/**
- * This little program is used by the functional test to get a list of table ids.
- */
-public class ListTables {
- public static void main(String[] args) throws Exception {
- ServerUtilOpts opts = new ServerUtilOpts();
- opts.parseArgs(ListTables.class.getName(), args);
- for (Entry<String,Table.ID> table : Tables.getNameToIdMap(opts.getClientContext()).entrySet())
- System.out.println(table.getKey() + " => " + table.getValue());
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/NullBatchWriter.java b/test/src/main/java/org/apache/accumulo/test/NullBatchWriter.java
deleted file mode 100644
index 4158b2c..0000000
--- a/test/src/main/java/org/apache/accumulo/test/NullBatchWriter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.data.Mutation;
-
-public class NullBatchWriter implements BatchWriter {
-
- private int mutationsAdded;
- private long startTime;
-
- @Override
- public void addMutation(Mutation m) throws MutationsRejectedException {
- if (mutationsAdded == 0) {
- startTime = System.currentTimeMillis();
- }
- mutationsAdded++;
- m.numBytes();
- }
-
- @Override
- public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
- for (Mutation mutation : iterable) {
- addMutation(mutation);
- }
- }
-
- @Override
- public void close() throws MutationsRejectedException {
- flush();
- }
-
- @Override
- public void flush() throws MutationsRejectedException {
- System.out.printf("Mutation add rate : %,6.2f mutations/sec%n",
- mutationsAdded / ((System.currentTimeMillis() - startTime) / 1000.0));
- mutationsAdded = 0;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/QueryMetadataTable.java b/test/src/main/java/org/apache/accumulo/test/QueryMetadataTable.java
deleted file mode 100644
index 32c5ca2..0000000
--- a/test/src/main/java/org/apache/accumulo/test/QueryMetadataTable.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-public class QueryMetadataTable {
- private static final Logger log = LoggerFactory.getLogger(QueryMetadataTable.class);
-
- static String location;
-
- static class MDTQuery implements Runnable {
-
- private AccumuloClient client;
- private Text row;
-
- MDTQuery(AccumuloClient client, Text row) {
- this.client = client;
- this.row = row;
- }
-
- @Override
- public void run() {
- Scanner mdScanner = null;
- try {
- KeyExtent extent = new KeyExtent(row, (Text) null);
-
- mdScanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- Text row = extent.getMetadataEntry();
-
- mdScanner.setRange(new Range(row));
-
- for (Entry<Key,Value> entry : mdScanner) {
- if (!entry.getKey().getRow().equals(row))
- break;
- }
-
- } catch (TableNotFoundException e) {
- log.error("Table '" + MetadataTable.NAME + "' not found.", e);
- throw new RuntimeException(e);
- } finally {
- if (mdScanner != null) {
- mdScanner.close();
- }
- }
- }
- }
-
- static class Opts extends ServerUtilOpts {
- @Parameter(names = "--numQueries", description = "number of queries to run")
- int numQueries = 1;
- @Parameter(names = "--numThreads", description = "number of threads used to run the queries")
- int numThreads = 1;
- }
-
- public static void main(String[] args) throws TableNotFoundException {
- Opts opts = new Opts();
- ScannerOpts scanOpts = new ScannerOpts();
- opts.parseArgs(QueryMetadataTable.class.getName(), args, scanOpts);
-
- try (AccumuloClient accumuloClient = opts.createClient()) {
- HashSet<Text> rowSet = new HashSet<>();
-
- int count = 0;
-
- try (Scanner scanner = accumuloClient.createScanner(MetadataTable.NAME, opts.auths)) {
- scanner.setBatchSize(scanOpts.scanBatchSize);
- Text mdrow = new Text(TabletsSection.getRow(MetadataTable.ID, null));
-
- for (Entry<Key,Value> entry : scanner) {
- System.out.print(".");
- if (count % 72 == 0) {
- System.out.printf(" %,d%n", count);
- }
- if (entry.getKey().compareRow(mdrow) == 0 && entry.getKey().getColumnFamily()
- .compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
- System.out.println(entry.getKey() + " " + entry.getValue());
- location = entry.getValue().toString();
- }
-
- if (!entry.getKey().getRow().toString().startsWith(MetadataTable.ID.canonicalID()))
- rowSet.add(entry.getKey().getRow());
- count++;
- }
- }
-
- System.out.printf(" %,d%n", count);
-
- ArrayList<Text> rows = new ArrayList<>(rowSet);
-
- Random r = new SecureRandom();
-
- ExecutorService tp = Executors.newFixedThreadPool(opts.numThreads);
-
- long t1 = System.currentTimeMillis();
-
- for (int i = 0; i < opts.numQueries; i++) {
- int index = r.nextInt(rows.size());
- MDTQuery mdtq = new MDTQuery(accumuloClient, rows.get(index));
- tp.submit(mdtq);
- }
-
- tp.shutdown();
-
- try {
- tp.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- log.error("Failed while awaiting the ExcecutorService to terminate.", e);
- throw new RuntimeException(e);
- }
-
- long t2 = System.currentTimeMillis();
- double delta = (t2 - t1) / 1000.0;
- System.out.println("time : " + delta + " queries per sec : " + (opts.numQueries / delta));
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationTablesPrinterThread.java b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationTablesPrinterThread.java
deleted file mode 100644
index 59105c2..0000000
--- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationTablesPrinterThread.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.replication;
-
-import java.io.PrintStream;
-
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.util.Daemon;
-import org.apache.accumulo.server.replication.PrintReplicationRecords;
-
-public class ReplicationTablesPrinterThread extends Daemon {
-
- private PrintStream out;
- private PrintReplicationRecords printer;
-
- public ReplicationTablesPrinterThread(AccumuloClient client, PrintStream out) {
- printer = new PrintReplicationRecords(client, out);
- this.out = out;
- }
-
- @Override
- public void run() {
- while (true) {
- printer.run();
- out.println();
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
- }
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java b/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java
deleted file mode 100644
index 779b170..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-public class EstimateInMemMapOverhead {
-
- private static void check(int numEntries, int keyLen, int colFamLen, int colQualLen,
- int colVisLen, int dataLen) {
- new IntObjectMemoryUsageCheck(numEntries).run();
- new InMemoryMapMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, colVisLen, dataLen)
- .run();
- new TextMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, dataLen).run();
- new MutationMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, dataLen).run();
- }
-
- public static void main(String[] args) {
- check(10000, 10, 4, 4, 4, 20);
- check(100000, 10, 4, 4, 4, 20);
- check(500000, 10, 4, 4, 4, 20);
- check(1000000, 10, 4, 4, 4, 20);
- check(2000000, 10, 4, 4, 4, 20);
-
- check(10000, 20, 5, 5, 5, 500);
- check(100000, 20, 5, 5, 5, 500);
- check(500000, 20, 5, 5, 5, 500);
- check(1000000, 20, 5, 5, 5, 500);
- check(2000000, 20, 5, 5, 5, 500);
-
- check(10000, 40, 10, 10, 10, 1000);
- check(100000, 40, 10, 10, 10, 1000);
- check(500000, 40, 10, 10, 10, 1000);
- check(1000000, 40, 10, 10, 10, 1000);
- check(2000000, 40, 10, 10, 10, 1000);
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
deleted file mode 100644
index f2c1060..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import java.util.Collections;
-
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
-import org.apache.accumulo.test.InMemoryMapIT;
-import org.apache.accumulo.tserver.InMemoryMap;
-import org.apache.hadoop.io.Text;
-
-class InMemoryMapMemoryUsageCheck extends MemoryUsageCheck {
-
- private int keyLen;
- private int colFamLen;
- private int colQualLen;
- private int colVisLen;
- private int dataLen;
-
- private InMemoryMap imm;
- private Text key;
- private Text colf;
- private Text colq;
- private ColumnVisibility colv;
- private int passes;
-
- InMemoryMapMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int colVisLen,
- int dataLen) {
- this.keyLen = keyLen;
- this.colFamLen = colFamLen;
- this.colQualLen = colQualLen;
- this.dataLen = dataLen;
- this.passes = passes;
- this.colVisLen = colVisLen;
-
- }
-
- @Override
- void init() {
- try {
- imm = new InMemoryMap(DefaultConfiguration.getInstance(), InMemoryMapIT.getServerContext());
- } catch (LocalityGroupConfigurationError e) {
- throw new RuntimeException(e);
- }
- key = new Text();
-
- colf = new Text(String.format("%0" + colFamLen + "d", 0));
- colq = new Text(String.format("%0" + colQualLen + "d", 0));
- colv = new ColumnVisibility(String.format("%0" + colVisLen + "d", 0));
- }
-
- @Override
- public void addEntry(int i) {
- key.set(String.format("%0" + keyLen + "d", i));
-
- Mutation m = new Mutation(key);
-
- byte data[] = new byte[dataLen];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) (j % 10 + 65);
- }
- Value idata = new Value(data);
-
- m.put(colf, colq, colv, idata);
-
- imm.mutate(Collections.singletonList(m));
-
- }
-
- @Override
- public int getEstimatedBytesPerEntry() {
- return keyLen + colFamLen + colQualLen + dataLen + 4 + colVisLen;
- }
-
- @Override
- public void clear() {
- imm = null;
- key = null;
- colf = null;
- colq = null;
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
- @Override
- String getName() {
- return "IMM " + keyLen + " " + colFamLen + " " + colQualLen + " " + dataLen;
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java
deleted file mode 100644
index 74016a2..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-class IntObjectMemoryUsageCheck extends MemoryUsageCheck {
-
- private int passes;
- private Object data[];
-
- static class SimpleObject {
- int d;
-
- SimpleObject(int d) {
- this.d = d;
- }
- }
-
- IntObjectMemoryUsageCheck(int numPasses) {
- this.passes = numPasses;
- }
-
- @Override
- void init() {
- data = new Object[passes];
- }
-
- @Override
- void addEntry(int i) {
- data[i] = new SimpleObject(i);
-
- }
-
- @Override
- void clear() {}
-
- @Override
- int getEstimatedBytesPerEntry() {
- return 4;
- }
-
- @Override
- String getName() {
- return "int obj";
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java
deleted file mode 100644
index 946d71f..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
-abstract class MemoryUsageCheck {
- abstract void addEntry(int i);
-
- abstract int getEstimatedBytesPerEntry();
-
- abstract void clear();
-
- abstract int getNumPasses();
-
- abstract String getName();
-
- abstract void init();
-
- @SuppressFBWarnings(value = "DM_GC", justification = "gc is okay for test")
- public void run() {
- System.gc();
- long usedMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- int count = 0;
- while (usedMem > 1024 * 1024 && count < 10) {
- System.gc();
- usedMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- count++;
- }
-
- init();
-
- for (int i = 0; i < getNumPasses(); i++) {
- addEntry(i);
- }
-
- System.gc();
-
- long memSize = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
- - usedMem;
-
- double actualBytesPerEntry = memSize / (double) getNumPasses();
- double expectedBytesPerEntry = getEstimatedBytesPerEntry();
- double diff = actualBytesPerEntry - expectedBytesPerEntry;
- double ratio = actualBytesPerEntry / expectedBytesPerEntry * 100;
-
- System.out.printf("%30s | %,10d | %6.2fGB | %6.2f | %6.2f | %6.2f | %6.2f%s%n", getName(),
- getNumPasses(), memSize / (1024 * 1024 * 1024.0), actualBytesPerEntry,
- expectedBytesPerEntry, diff, ratio, "%");
-
- clear();
-
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java
deleted file mode 100644
index f60d347..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-class MutationMemoryUsageCheck extends MemoryUsageCheck {
-
- private int keyLen;
- private int colFamLen;
- private int colQualLen;
- private int dataLen;
-
- private Mutation[] mutations;
- private Text key;
- private Text colf;
- private Text colq;
- private int passes;
-
- MutationMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) {
- this.keyLen = keyLen;
- this.colFamLen = colFamLen;
- this.colQualLen = colQualLen;
- this.dataLen = dataLen;
- this.passes = passes;
- mutations = new Mutation[passes];
-
- }
-
- @Override
- void init() {
- key = new Text();
-
- colf = new Text(String.format("%0" + colFamLen + "d", 0));
- colq = new Text(String.format("%0" + colQualLen + "d", 0));
-
- byte data[] = new byte[dataLen];
- for (int i = 0; i < data.length; i++) {
- data[i] = (byte) (i % 10 + 65);
- }
- }
-
- @Override
- public void addEntry(int i) {
- key.set(String.format("%0" + keyLen + "d", i));
-
- Mutation m = new Mutation(key);
-
- byte data[] = new byte[dataLen];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) (j % 10 + 65);
- }
- Value idata = new Value(data);
-
- m.put(colf, colq, idata);
-
- mutations[i] = m;
- }
-
- @Override
- public int getEstimatedBytesPerEntry() {
- return keyLen + colFamLen + colQualLen + dataLen;
- }
-
- @Override
- public void clear() {
- key = null;
- colf = null;
- colq = null;
- mutations = null;
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
- @Override
- String getName() {
- return "Mutation " + keyLen + " " + colFamLen + " " + colQualLen + " " + dataLen;
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java
deleted file mode 100644
index 72d8a33..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-class TextMemoryUsageCheck extends MemoryUsageCheck {
-
- private int keyLen;
- private int colFamLen;
- private int colQualLen;
- private int dataLen;
- private TreeMap<Text,Value> map;
- private int passes;
-
- TextMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) {
- this.keyLen = keyLen;
- this.colFamLen = colFamLen;
- this.colQualLen = colQualLen;
- this.dataLen = dataLen;
- this.passes = passes;
-
- }
-
- @Override
- void init() {
- map = new TreeMap<>();
- }
-
- @Override
- public void addEntry(int i) {
- Text key = new Text(
- String.format("%0" + keyLen + "d:%0" + colFamLen + "d:%0" + colQualLen + "d", i, 0, 0)
- .getBytes());
- //
- byte data[] = new byte[dataLen];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) (j % 10 + 65);
- }
- Value value = new Value(data);
-
- map.put(key, value);
-
- }
-
- @Override
- public void clear() {
- map.clear();
- map = null;
- }
-
- @Override
- public int getEstimatedBytesPerEntry() {
- return keyLen + colFamLen + colQualLen + dataLen;
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
- @Override
- String getName() {
- return "Text " + keyLen + " " + colFamLen + " " + colQualLen + " " + dataLen;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java b/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java
deleted file mode 100644
index d7fc1a9..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.metadata;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.TreeSet;
-import java.util.UUID;
-
-import org.apache.accumulo.core.cli.ClientOpts;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.clientImpl.Table;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.DataFileValue;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.core.util.Stat;
-import org.apache.accumulo.server.master.state.TServerInstance;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Iterators;
-
-/**
- * This little program can be used to write a lot of metadata entries and measure the performance of
- * varying numbers of threads doing metadata lookups using the batch scanner.
- */
-public class MetadataBatchScan {
-
- private static final Logger log = LoggerFactory.getLogger(MetadataBatchScan.class);
-
- public static void main(String[] args) throws Exception {
-
- ClientOpts opts = new ClientOpts();
- opts.parseArgs(MetadataBatchScan.class.getName(), args);
-
- TreeSet<Long> splits = new TreeSet<>();
- Random r = new SecureRandom();
-
- while (splits.size() < 99999) {
- splits.add((r.nextLong() & 0x7fffffffffffffffL) % 1000000000000L);
- }
-
- Table.ID tid = Table.ID.of("8");
- Text per = null;
-
- ArrayList<KeyExtent> extents = new ArrayList<>();
-
- for (Long split : splits) {
- Text er = new Text(String.format("%012d", split));
- KeyExtent ke = new KeyExtent(tid, er, per);
- per = er;
-
- extents.add(ke);
- }
-
- extents.add(new KeyExtent(tid, null, per));
-
- try (AccumuloClient accumuloClient = opts.createClient()) {
- if (args[0].equals("write")) {
-
- BatchWriter bw = accumuloClient.createBatchWriter(MetadataTable.NAME,
- new BatchWriterConfig());
-
- for (KeyExtent extent : extents) {
- Mutation mut = extent.getPrevRowUpdateMutation();
- new TServerInstance(HostAndPort.fromParts("192.168.1.100", 4567), "DEADBEEF")
- .putLocation(mut);
- bw.addMutation(mut);
- }
-
- bw.close();
- } else if (args[0].equals("writeFiles")) {
- BatchWriter bw = accumuloClient.createBatchWriter(MetadataTable.NAME,
- new BatchWriterConfig());
-
- for (KeyExtent extent : extents) {
-
- Mutation mut = new Mutation(extent.getMetadataEntry());
-
- String dir = "/t-" + UUID.randomUUID();
-
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut,
- new Value(dir.getBytes(UTF_8)));
-
- for (int i = 0; i < 5; i++) {
- mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"),
- new DataFileValue(10000, 1000000).encodeAsValue());
- }
-
- bw.addMutation(mut);
- }
-
- bw.close();
- } else if (args[0].equals("scan")) {
-
- int numThreads = Integer.parseInt(args[1]);
- final int numLoop = Integer.parseInt(args[2]);
- int numLookups = Integer.parseInt(args[3]);
-
- HashSet<Integer> indexes = new HashSet<>();
- while (indexes.size() < numLookups) {
- indexes.add(r.nextInt(extents.size()));
- }
-
- final List<Range> ranges = new ArrayList<>();
- for (Integer i : indexes) {
- ranges.add(extents.get(i).toMetadataRange());
- }
-
- Thread threads[] = new Thread[numThreads];
-
- for (int i = 0; i < threads.length; i++) {
- threads[i] = new Thread(() -> {
- try {
- System.out.println(runScanTest(accumuloClient, numLoop, ranges));
- } catch (Exception e) {
- log.error("Exception while running scan test.", e);
- }
- });
- }
-
- long t1 = System.currentTimeMillis();
-
- for (Thread thread : threads) {
- thread.start();
- }
-
- for (Thread thread : threads) {
- thread.join();
- }
-
- long t2 = System.currentTimeMillis();
-
- System.out.printf("tt : %6.2f%n", (t2 - t1) / 1000.0);
-
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- }
-
- private static ScanStats runScanTest(AccumuloClient accumuloClient, int numLoop,
- List<Range> ranges) throws Exception {
- ScanStats stats = new ScanStats();
-
- try (BatchScanner bs = accumuloClient.createBatchScanner(MetadataTable.NAME,
- Authorizations.EMPTY, 1)) {
- bs.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
- TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(bs);
-
- bs.setRanges(ranges);
-
- // System.out.println(ranges);
- for (int i = 0; i < numLoop; i++) {
- ScanStat ss = scan(bs, ranges, null);
- stats.merge(ss);
- }
- }
- return stats;
- }
-
- private static class ScanStat {
- long delta1;
- long delta2;
- int count1;
- int count2;
- }
-
- private static class ScanStats {
- Stat delta1 = new Stat();
- Stat delta2 = new Stat();
- Stat count1 = new Stat();
- Stat count2 = new Stat();
-
- void merge(ScanStat ss) {
- delta1.addStat(ss.delta1);
- delta2.addStat(ss.delta2);
- count1.addStat(ss.count1);
- count2.addStat(ss.count2);
- }
-
- @Override
- public String toString() {
- return "[" + delta1 + "] [" + delta2 + "]";
- }
- }
-
- private static ScanStat scan(BatchScanner bs, List<Range> ranges, Scanner scanner) {
-
- // System.out.println("ranges : "+ranges);
-
- ScanStat ss = new ScanStat();
-
- long t1 = System.currentTimeMillis();
- int count = Iterators.size(bs.iterator());
- bs.close();
- long t2 = System.currentTimeMillis();
-
- ss.delta1 = t2 - t1;
- ss.count1 = count;
-
- count = 0;
- t1 = System.currentTimeMillis();
- /*
- * for (Range range : ranges) { scanner.setRange(range); for (Entry<Key, Value> entry : scanner)
- * { count++; } }
- */
-
- t2 = System.currentTimeMillis();
-
- ss.delta2 = t2 - t1;
- ss.count2 = count;
-
- return ss;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java
deleted file mode 100644
index 5fe214e..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.nativemap;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.tserver.NativeMap;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-
-public class NativeMapConcurrency {
-
- private static final Logger log = LoggerFactory.getLogger(NativeMapConcurrency.class);
-
- private static final byte ROW_PREFIX[] = {'r'};
- private static final byte COL_PREFIX[] = {'c'};
-
- static Mutation nm(int r) {
- return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 6, 10, ROW_PREFIX)));
- }
-
- private static final Text ET = new Text();
-
- private static void pc(Mutation m, int c, Value v) {
- m.put(new Text(FastFormat.toZeroPaddedString(c, 3, 10, COL_PREFIX)), ET, v);
- }
-
- static NativeMap create(int numRows, int numCols) {
-
- NativeMap nm = new NativeMap();
-
- populate(0, numRows, numCols, nm);
-
- return nm;
-
- }
-
- private static void populate(int start, int numRows, int numCols, NativeMap nm) {
- long t1 = System.currentTimeMillis();
- int mc = 1;
- for (int i = 0; i < numRows; i++) {
- Mutation m = nm(i + start);
- for (int j = 0; j < numCols; j++) {
- Value val = new Value("test".getBytes(UTF_8));
- pc(m, j, val);
- }
- nm.mutate(m, mc++);
- }
- long t2 = System.currentTimeMillis();
-
- System.out.printf("inserted %,d in %,d %,d %,6.2f%n", (numRows * numCols), (t2 - t1), nm.size(),
- rate((numRows * numCols), (t2 - t1)));
- }
-
- private static double rate(int num, long ms) {
- return num / (ms / 1000.0);
- }
-
- static class Opts {
- @Parameter(names = "--rows", description = "rows", required = true)
- int rows = 0;
- @Parameter(names = "--cols", description = "cols")
- int cols = 1;
- @Parameter(names = "--threads", description = "threads")
- int threads = 1;
- @Parameter(names = "--writeThreads", description = "write threads")
- int writeThreads = 1;
- @Parameter(names = "-help", help = true)
- boolean help = false;
- }
-
- public static void main(String[] args) {
- Opts opts = new Opts();
- JCommander jc = new JCommander(opts);
- jc.setProgramName(NativeMapConcurrency.class.getName());
- jc.parse(args);
- if (opts.help) {
- jc.usage();
- return;
- }
- NativeMap nm = create(opts.rows, opts.cols);
- runTest(nm, opts.rows, opts.cols, opts.threads, opts.writeThreads);
- nm.delete();
- }
-
- static class ScanTask implements Runnable {
-
- private NativeMap nm;
-
- ScanTask(NativeMap nm) {
- this.nm = nm;
- }
-
- @Override
- public void run() {
-
- for (int i = 0; i < 10; i++) {
-
- Iterator<Entry<Key,Value>> iter = nm.iterator();
-
- long t1 = System.currentTimeMillis();
-
- int count = 0;
-
- while (iter.hasNext()) {
- count++;
- iter.next();
- }
-
- long t2 = System.currentTimeMillis();
-
- System.out.printf("%d %,d %,d %,d %,d %,6.2f%n", Thread.currentThread().getId(), (t2 - t1),
- t1, t2, count, rate(count, (t2 - t1)));
- }
- }
-
- }
-
- static class WriteTask implements Runnable {
-
- private int start;
- private int rows;
- private int cols;
- private NativeMap nm;
-
- WriteTask(int start, int rows, int cols, NativeMap nm) {
- this.start = start;
- this.rows = rows;
- this.cols = cols;
- this.nm = nm;
- }
-
- @Override
- public void run() {
- populate(start, rows, cols, nm);
- }
-
- }
-
- private static void runTest(NativeMap nm, int rows, int cols, int numReadThreads,
- int writeThreads) {
-
- Thread threads[] = new Thread[numReadThreads + writeThreads];
-
- for (int i = 0; i < numReadThreads; i++) {
- threads[i] = new Thread(new ScanTask(nm));
- }
-
- int start = 0;
- for (int i = numReadThreads; i < writeThreads + numReadThreads; i++) {
- threads[i] = new Thread(new WriteTask(start, rows, cols, nm));
- // start += rows;
- }
-
- for (Thread thread : threads) {
- thread.start();
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '" + thread.getName() + "'", e);
- }
- }
-
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java
deleted file mode 100644
index c9b05af..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.nativemap;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
-
-import java.security.SecureRandom;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.tserver.NativeMap;
-import org.apache.hadoop.io.Text;
-
-public class NativeMapPerformance {
-
- private static final byte ROW_PREFIX[] = {'r'};
- private static final byte COL_PREFIX[] = {'c'};
-
- static Key newKey(int r, int c) {
- return new Key(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)),
- new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)));
- }
-
- static Mutation newMutation(int r) {
- return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)));
- }
-
- static Text ET = new Text();
-
- private static void pc(Mutation m, int c, Value v) {
- m.put(new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)), ET, Long.MAX_VALUE, v);
- }
-
- static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
-
- SortedMap<Key,Value> tm = null;
- NativeMap nm = null;
-
- if (mapType.equals("SKIP_LIST"))
- tm = new ConcurrentSkipListMap<>();
- else if (mapType.equals("TREE_MAP"))
- tm = Collections.synchronizedSortedMap(new TreeMap<>());
- else if (mapType.equals("NATIVE_MAP"))
- nm = new NativeMap();
- else
- throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
-
- Random rand = new SecureRandom();
-
- // puts
- long tps = System.currentTimeMillis();
-
- if (nm != null) {
- for (int i = 0; i < numRows; i++) {
- int row = rand.nextInt(1000000000);
- Mutation m = newMutation(row);
- for (int j = 0; j < numCols; j++) {
- int col = rand.nextInt(1000000);
- Value val = new Value("test".getBytes(UTF_8));
- pc(m, col, val);
- }
- nm.mutate(m, i);
- }
- } else {
- for (int i = 0; i < numRows; i++) {
- int row = rand.nextInt(1000000000);
- for (int j = 0; j < numCols; j++) {
- int col = rand.nextInt(1000000);
- Key key = newKey(row, col);
- Value val = new Value("test".getBytes(UTF_8));
- tm.put(key, val);
- }
- }
- }
-
- long tpe = System.currentTimeMillis();
-
- // Iteration
- Iterator<Entry<Key,Value>> iter;
- if (nm != null) {
- iter = nm.iterator();
- } else {
- iter = tm.entrySet().iterator();
- }
-
- long tis = System.currentTimeMillis();
-
- while (iter.hasNext()) {
- iter.next();
- }
-
- long tie = System.currentTimeMillis();
-
- rand = new SecureRandom();
- int rowsToLookup[] = new int[numLookups];
- int colsToLookup[] = new int[numLookups];
- for (int i = 0; i < Math.min(numLookups, numRows); i++) {
- int row = rand.nextInt(1000000000);
- int col = -1;
- for (int j = 0; j < numCols; j++) {
- col = rand.nextInt(1000000);
- }
-
- rowsToLookup[i] = row;
- colsToLookup[i] = col;
- }
-
- // get
-
- long tgs = System.currentTimeMillis();
- if (nm != null) {
- for (int i = 0; i < numLookups; i++) {
- Key key = newKey(rowsToLookup[i], colsToLookup[i]);
- if (nm.get(key) == null) {
- throw new RuntimeException(
- "Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
- }
- }
- } else {
- for (int i = 0; i < numLookups; i++) {
- Key key = newKey(rowsToLookup[i], colsToLookup[i]);
- if (tm.get(key) == null) {
- throw new RuntimeException(
- "Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
- }
- }
- }
- long tge = System.currentTimeMillis();
-
- long memUsed = 0;
- if (nm != null) {
- memUsed = nm.getMemoryUsed();
- }
-
- int size = (nm == null ? tm.size() : nm.size());
-
- // delete
- long tds = System.currentTimeMillis();
-
- if (nm != null)
- nm.delete();
-
- long tde = System.currentTimeMillis();
-
- if (tm != null)
- tm.clear();
-
- System.gc();
- System.gc();
- System.gc();
- System.gc();
-
- sleepUninterruptibly(3, TimeUnit.SECONDS);
-
- System.out.printf(
- "mapType:%10s put rate:%,6.2f scan rate:%,6.2f get"
- + " rate:%,6.2f delete time : %6.2f mem : %,d%n",
- "" + mapType, (numRows * numCols) / ((tpe - tps) / 1000.0), (size) / ((tie - tis) / 1000.0),
- numLookups / ((tge - tgs) / 1000.0), (tde - tds) / 1000.0, memUsed);
-
- }
-
- public static void main(String[] args) {
-
- if (args.length != 3) {
- throw new IllegalArgumentException(
- "Usage : " + NativeMapPerformance.class.getName() + " <map type> <rows> <columns>");
- }
-
- String mapType = args[0];
- int rows = Integer.parseInt(args[1]);
- int cols = Integer.parseInt(args[2]);
-
- runPerformanceTest(rows, cols, 10000, mapType);
- runPerformanceTest(rows, cols, 10000, mapType);
- runPerformanceTest(rows, cols, 10000, mapType);
-
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java
deleted file mode 100644
index 15c1c5a..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.nativemap;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.OpTimer;
-import org.apache.accumulo.tserver.NativeMap;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class NativeMapStress {
-
- private static final Logger log = LoggerFactory.getLogger(NativeMapStress.class);
-
- public static void main(String[] args) {
- testLotsOfMapDeletes(true);
- testLotsOfMapDeletes(false);
- testLotsOfOverwrites();
- testLotsOfGetsAndScans();
- }
-
- private static void put(NativeMap nm, String row, String val, int mc) {
- Mutation m = new Mutation(new Text(row));
- m.put(new Text(), new Text(), Long.MAX_VALUE, new Value(val.getBytes(UTF_8)));
- nm.mutate(m, mc);
- }
-
- private static void testLotsOfGetsAndScans() {
-
- ArrayList<Thread> threads = new ArrayList<>();
-
- final int numThreads = 8;
- final int totalGets = 100000000;
- final int mapSizePerThread = (int) (4000000 / (double) numThreads);
- final int getsPerThread = (int) (totalGets / (double) numThreads);
-
- for (int tCount = 0; tCount < numThreads; tCount++) {
- Runnable r = new Runnable() {
- @Override
- public void run() {
- NativeMap nm = new NativeMap();
-
- Random r = new SecureRandom();
-
- OpTimer timer = null;
- AtomicLong nextOpid = new AtomicLong();
-
- if (log.isInfoEnabled()) {
- log.info("tid={} oid={} Creating map of size {}", Thread.currentThread().getId(),
- nextOpid.get(), mapSizePerThread);
- timer = new OpTimer().start();
- }
-
- for (int i = 0; i < mapSizePerThread; i++) {
- String row = String.format("r%08d", i);
- String val = row + "v";
- put(nm, row, val, i);
- }
-
- if (timer != null) {
-
- // stop and log created elapsed time
- timer.stop();
- log.info("tid={} oid={} Created map of size {} in {}", Thread.currentThread().getId(),
- nextOpid.getAndIncrement(), nm.size(),
- String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
-
- // start timer for gets
- log.info("tid={} oid={} Doing {} gets()", Thread.currentThread().getId(),
- nextOpid.get(), getsPerThread);
- timer.reset().start();
- }
-
- for (int i = 0; i < getsPerThread; i++) {
- String row = String.format("r%08d", r.nextInt(mapSizePerThread));
- String val = row + "v";
-
- Value value = nm.get(new Key(new Text(row)));
- if (value == null || !value.toString().equals(val)) {
- log.error("nm.get({}) failed", row);
- }
- }
-
- if (timer != null) {
-
- // stop and log created elapsed time
- timer.stop();
- log.info("tid={} oid={} Finished {} gets in {}", Thread.currentThread().getId(),
- nextOpid.getAndIncrement(), getsPerThread,
- String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
-
- // start timer for random iterations
- log.info("tid={} oid={} Doing {} random iterations", Thread.currentThread().getId(),
- nextOpid.get(), getsPerThread);
- timer.reset().start();
- }
-
- int scanned = 0;
-
- for (int i = 0; i < getsPerThread; i++) {
- int startRow = r.nextInt(mapSizePerThread);
- String row = String.format("r%08d", startRow);
-
- Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
-
- int count = 0;
-
- while (iter.hasNext() && count < 10) {
- String row2 = String.format("r%08d", startRow + count);
- String val2 = row2 + "v";
-
- Entry<Key,Value> entry = iter.next();
- if (!entry.getValue().toString().equals(val2)
- || !entry.getKey().equals(new Key(new Text(row2)))) {
- log.error("nm.iter({}) failed row = {} count = {} row2 = {} val2 = {}", row2, row,
- count, row, val2);
- }
-
- count++;
- }
-
- scanned += count;
- }
-
- if (timer != null) {
-
- // stop and log created elapsed time
- timer.stop();
- log.info("tid={} oid={} Finished {} random iterations (scanned = {}) in {}",
- Thread.currentThread().getId(), nextOpid.getAndIncrement(), getsPerThread, scanned,
- String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
-
- }
-
- nm.delete();
- }
- };
-
- Thread t = new Thread(r);
- t.start();
-
- threads.add(t);
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '{}'.", thread.getName(), e);
- throw new RuntimeException(e);
- }
- }
- }
-
- private static void testLotsOfMapDeletes(final boolean doRemoves) {
- final int numThreads = 8;
- final int rowRange = 10000;
- final int mapsPerThread = 50;
- final int totalInserts = 100000000;
- final int insertsPerMapPerThread = (int) (totalInserts / (double) numThreads / mapsPerThread);
-
- System.out.println("insertsPerMapPerThread " + insertsPerMapPerThread);
-
- ArrayList<Thread> threads = new ArrayList<>();
-
- for (int i = 0; i < numThreads; i++) {
- Runnable r = new Runnable() {
- @Override
- public void run() {
-
- int inserts = 0;
- int removes = 0;
-
- for (int i = 0; i < mapsPerThread; i++) {
-
- NativeMap nm = new NativeMap();
-
- for (int j = 0; j < insertsPerMapPerThread; j++) {
- String row = String.format("r%08d", j % rowRange);
- String val = row + "v";
- put(nm, row, val, j);
- inserts++;
- }
-
- if (doRemoves) {
- Iterator<Entry<Key,Value>> iter = nm.iterator();
- while (iter.hasNext()) {
- iter.next();
- iter.remove();
- removes++;
- }
- }
-
- nm.delete();
- }
-
- System.out.println("inserts " + inserts + " removes " + removes + " "
- + Thread.currentThread().getName());
- }
- };
-
- Thread t = new Thread(r);
- t.start();
-
- threads.add(t);
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '{}'.", thread.getName(), e);
- throw new RuntimeException(e);
- }
- }
- }
-
- private static void testLotsOfOverwrites() {
- final Map<Integer,NativeMap> nativeMaps = new HashMap<>();
-
- int numThreads = 8;
- final int insertsPerThread = (int) (100000000 / (double) numThreads);
- final int rowRange = 10000;
- final int numMaps = 50;
-
- ArrayList<Thread> threads = new ArrayList<>();
-
- for (int i = 0; i < numThreads; i++) {
- Runnable r = new Runnable() {
- @Override
- public void run() {
- Random r = new SecureRandom();
- int inserts = 0;
-
- for (int i = 0; i < insertsPerThread / 100.0; i++) {
- int map = r.nextInt(numMaps);
-
- NativeMap nm;
-
- synchronized (nativeMaps) {
- nm = nativeMaps.get(map);
- if (nm == null) {
- nm = new NativeMap();
- nativeMaps.put(map, nm);
-
- }
- }
-
- synchronized (nm) {
- for (int j = 0; j < 100; j++) {
- String row = String.format("r%08d", r.nextInt(rowRange));
- String val = row + "v";
- put(nm, row, val, j);
- inserts++;
- }
- }
- }
-
- System.out.println("inserts " + inserts + " " + Thread.currentThread().getName());
- }
- };
-
- Thread t = new Thread(r);
- t.start();
-
- threads.add(t);
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '{}'.", thread.getName(), e);
- throw new RuntimeException(e);
- }
- }
-
- Set<Entry<Integer,NativeMap>> es = nativeMaps.entrySet();
- for (Entry<Integer,NativeMap> entry : es) {
- entry.getValue().delete();
- }
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java b/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java
deleted file mode 100644
index 4e72f1b..0000000
--- a/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.tablet;
-
-import org.apache.accumulo.core.clientImpl.ClientContext;
-import org.apache.accumulo.core.clientImpl.Table;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.rpc.ThriftUtil;
-import org.apache.accumulo.core.tabletserver.thrift.TDurability;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-public class WrongTablet {
-
- static class Opts extends ServerUtilOpts {
- @Parameter(names = "--location", required = true)
- String location;
- }
-
- public static void main(String[] args) {
- final Opts opts = new Opts();
- opts.parseArgs(WrongTablet.class.getName(), args);
-
- final HostAndPort location = HostAndPort.fromString(opts.location);
- final ClientContext context = opts.getClientContext();
-
- try {
- TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context);
-
- Mutation mutation = new Mutation(new Text("row_0003750001"));
- mutation.putDelete(new Text("colf"), new Text("colq"));
- client.update(Tracer.traceInfo(), context.rpcCreds(),
- new KeyExtent(Table.ID.of("!!"), null, new Text("row_0003750000")).toThrift(),
- mutation.toThrift(), TDurability.DEFAULT);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-}