You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2018/12/12 22:41:27 UTC
[GitHub] mikewalch closed pull request #826: Removed unused code
mikewalch closed pull request #826: Removed unused code
URL: https://github.com/apache/accumulo/pull/826
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/test/src/main/java/org/apache/accumulo/test/CreateRFiles.java b/test/src/main/java/org/apache/accumulo/test/CreateRFiles.java
deleted file mode 100644
index 4cefefa08e..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/CreateRFiles.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.cli.Help;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-public class CreateRFiles {
-
- private static final Logger log = LoggerFactory.getLogger(CreateRFiles.class);
-
- static class Opts extends Help {
-
- @Parameter(names = "--output", description = "the destiation directory")
- String outputDirectory;
-
- @Parameter(names = "--numThreads",
- description = "number of threads to use when generating files")
- int numThreads = 4;
-
- @Parameter(names = "--start", description = "the start number for test data")
- long start = 0;
-
- @Parameter(names = "--end", description = "the maximum number for test data")
- long end = 10 * 1000 * 1000;
-
- @Parameter(names = "--splits", description = "the number of splits in the data")
- long numsplits = 4;
- }
-
- public static void main(String[] args) {
- Opts opts = new Opts();
- opts.parseArgs(CreateRFiles.class.getName(), args);
-
- long splitSize = Math.round((opts.end - opts.start) / (double) opts.numsplits);
-
- long currStart = opts.start;
- long currEnd = opts.start + splitSize;
-
- ExecutorService threadPool = Executors.newFixedThreadPool(opts.numThreads);
-
- int count = 0;
- while (currEnd <= opts.end && currStart < currEnd) {
-
- final String tia = String.format(
- "--rfile %s/mf%05d --timestamp 1 --size 50 --random 56 --rows %d --start %d --user root",
- opts.outputDirectory, count, currEnd - currStart, currStart);
-
- Runnable r = new Runnable() {
-
- @Override
- public void run() {
- try {
- TestIngest.main(tia.split(" "));
- } catch (Exception e) {
- log.error("Could not run " + TestIngest.class.getName() + ".main using the input '"
- + tia + "'", e);
- }
- }
-
- };
-
- threadPool.execute(r);
-
- count++;
- currStart = currEnd;
- currEnd = Math.min(opts.end, currStart + splitSize);
- }
-
- threadPool.shutdown();
- while (!threadPool.isTerminated())
- try {
- threadPool.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/CreateRandomRFile.java b/test/src/main/java/org/apache/accumulo/test/CreateRandomRFile.java
deleted file mode 100644
index 0f10bab18e..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/CreateRandomRFile.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.io.IOException;
-import java.security.SecureRandom;
-import java.util.Arrays;
-import java.util.Random;
-
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.crypto.CryptoServiceFactory;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.RFileOperations;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-
-public class CreateRandomRFile {
- private static int num;
- private static String file;
-
- public static byte[] createValue(long rowid, int dataSize) {
- Random r = new SecureRandom();
- r.setSeed(rowid);
- byte value[] = new byte[dataSize];
-
- r.nextBytes(value);
-
- // transform to printable chars
- for (int j = 0; j < value.length; j++) {
- value[j] = (byte) (((0xff & value[j]) % 92) + ' ');
- }
-
- return value;
- }
-
- public static void main(String[] args) {
- if (args.length != 2) {
- System.err.println("Usage CreateRandomRFile <filename> <size>");
- System.exit(-1);
- }
- file = args[0];
- num = Integer.parseInt(args[1]);
- long rands[] = new long[num];
-
- Random r = new SecureRandom();
-
- for (int i = 0; i < rands.length; i++) {
- rands[i] = (r.nextLong() & 0x7fffffffffffffffL) % 10000000000L;
- }
-
- Arrays.sort(rands);
-
- Configuration conf = CachedConfiguration.getInstance();
- FileSKVWriter mfw;
- try {
- FileSystem fs = FileSystem.get(conf);
- mfw = new RFileOperations().newWriterBuilder()
- .forFile(file, fs, conf, CryptoServiceFactory.newDefaultInstance())
- .withTableConfiguration(DefaultConfiguration.getInstance()).build();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- for (int i = 0; i < rands.length; i++) {
- Text row = new Text(String.format("row_%010d", rands[i]));
- Key key = new Key(row);
-
- Value dv = new Value(createValue(rands[i], 40));
-
- try {
- mfw.append(key, dv);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- try {
- mfw.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/FaultyConditionalWriter.java b/test/src/main/java/org/apache/accumulo/test/FaultyConditionalWriter.java
deleted file mode 100644
index 968ced4467..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/FaultyConditionalWriter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.ConditionalWriter;
-import org.apache.accumulo.core.data.ConditionalMutation;
-
-/**
- * A writer that will sometimes return unknown. When it returns unknown the condition may or may not
- * have been written.
- */
-public class FaultyConditionalWriter implements ConditionalWriter {
-
- private ConditionalWriter cw;
- private double up;
- private Random rand;
- private double wp;
-
- public FaultyConditionalWriter(ConditionalWriter cw, double unknownProbability,
- double writeProbability) {
- this.cw = cw;
- this.up = unknownProbability;
- this.wp = writeProbability;
- this.rand = new SecureRandom();
-
- }
-
- @Override
- public Iterator<Result> write(Iterator<ConditionalMutation> mutations) {
- ArrayList<Result> resultList = new ArrayList<>();
- ArrayList<ConditionalMutation> writes = new ArrayList<>();
-
- while (mutations.hasNext()) {
- ConditionalMutation cm = mutations.next();
- if (rand.nextDouble() <= up && rand.nextDouble() > wp)
- resultList.add(new Result(Status.UNKNOWN, cm, null));
- else
- writes.add(cm);
- }
-
- if (writes.size() > 0) {
- Iterator<Result> results = cw.write(writes.iterator());
-
- while (results.hasNext()) {
- Result result = results.next();
-
- if (rand.nextDouble() <= up && rand.nextDouble() <= wp)
- result = new Result(Status.UNKNOWN, result.getMutation(), result.getTabletServer());
- resultList.add(result);
- }
- }
- return resultList.iterator();
- }
-
- @Override
- public Result write(ConditionalMutation mutation) {
- return write(Collections.singleton(mutation).iterator()).next();
- }
-
- @Override
- public void close() {
- cw.close();
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/IMMLGBenchmark.java b/test/src/main/java/org/apache/accumulo/test/IMMLGBenchmark.java
deleted file mode 100644
index 4384e2108a..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/IMMLGBenchmark.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.core.util.Stat;
-import org.apache.hadoop.io.Text;
-
-import com.google.common.collect.Iterators;
-
-public class IMMLGBenchmark {
- public static void main(String[] args) throws Exception {
-
- AccumuloClient client = Accumulo.newClient().to("test16", "localhost").as("root", "secret")
- .build();
-
- int numlg = Integer.parseInt(args[0]);
-
- ArrayList<byte[]> cfset = new ArrayList<>();
-
- for (int i = 0; i < 32; i++) {
- cfset.add(String.format("%04x", i).getBytes());
- }
-
- Map<String,Stat> stats = new TreeMap<>();
-
- for (int i = 0; i < 5; i++) {
- runTest(client, numlg, cfset, i > 1 ? stats : null);
- System.out.println();
- }
-
- for (Entry<String,Stat> entry : stats.entrySet()) {
- System.out.printf("%20s : %6.2f\n", entry.getKey(), entry.getValue().mean());
- }
-
- }
-
- private static void runTest(AccumuloClient client, int numlg, ArrayList<byte[]> cfset,
- Map<String,Stat> stats) throws Exception {
- String table = "immlgb";
-
- try {
- client.tableOperations().delete(table);
- } catch (TableNotFoundException tnfe) {}
-
- client.tableOperations().create(table);
- client.tableOperations().setProperty(table, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(),
- "snappy");
-
- setupLocalityGroups(client, numlg, cfset, table);
-
- addStat(stats, "write", write(client, cfset, table));
- addStat(stats, "scan cf", scan(client, cfset, table, false));
- addStat(stats, "scan cf:cq", scan(client, cfset, table, true));
- // TODO time reading all data
-
- long t1 = System.currentTimeMillis();
- client.tableOperations().flush(table, null, null, true);
- long t2 = System.currentTimeMillis();
-
- addStat(stats, "flush", t2 - t1);
- }
-
- private static void addStat(Map<String,Stat> stats, String s, long wt) {
- System.out.println(s + ":" + wt);
-
- if (stats == null)
- return;
-
- Stat stat = stats.get(s);
- if (stat == null) {
- stat = new Stat();
- stats.put(s, stat);
- }
- stat.addStat(wt);
- }
-
- private static long scan(AccumuloClient client, ArrayList<byte[]> cfset, String table, boolean cq)
- throws TableNotFoundException {
- try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
-
- if (!cq)
- scanner.fetchColumnFamily(new Text(cfset.get(15)));
- else
- scanner.fetchColumn(new Text(cfset.get(15)), new Text(cfset.get(15)));
-
- long t1 = System.currentTimeMillis();
-
- Iterators.size(scanner.iterator());
-
- long t2 = System.currentTimeMillis();
-
- return t2 - t1;
- }
- }
-
- private static long write(AccumuloClient client, ArrayList<byte[]> cfset, String table)
- throws TableNotFoundException, MutationsRejectedException {
- Random rand = new SecureRandom();
-
- byte val[] = new byte[50];
-
- BatchWriter bw = client.createBatchWriter(table, new BatchWriterConfig());
-
- long t1 = System.currentTimeMillis();
-
- for (int i = 0; i < 1 << 15; i++) {
- byte[] row = FastFormat.toZeroPaddedString(abs(rand.nextLong()), 16, 16, new byte[0]);
-
- Mutation m = new Mutation(row);
- for (byte[] cf : cfset) {
- byte[] cq = FastFormat.toZeroPaddedString(rand.nextInt(1 << 16), 4, 16, new byte[0]);
- rand.nextBytes(val);
- m.put(cf, cq, val);
- }
-
- bw.addMutation(m);
- }
-
- bw.close();
-
- long t2 = System.currentTimeMillis();
-
- return t2 - t1;
- }
-
- private static void setupLocalityGroups(AccumuloClient client, int numlg, ArrayList<byte[]> cfset,
- String table) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- if (numlg > 1) {
- int numCF = cfset.size() / numlg;
- int gNum = 0;
-
- Iterator<byte[]> cfiter = cfset.iterator();
- Map<String,Set<Text>> groups = new HashMap<>();
- while (cfiter.hasNext()) {
- HashSet<Text> groupCols = new HashSet<>();
- for (int i = 0; i < numCF && cfiter.hasNext(); i++) {
- groupCols.add(new Text(cfiter.next()));
- }
-
- groups.put("lg" + (gNum++), groupCols);
- }
-
- client.tableOperations().setLocalityGroups(table, groups);
- client.tableOperations().offline(table);
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- client.tableOperations().online(table);
- }
- }
-
- public static long abs(long l) {
- l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE...
- if (l < 0)
- return 0;
- return l;
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/ListTables.java b/test/src/main/java/org/apache/accumulo/test/ListTables.java
deleted file mode 100644
index e52abb7ea2..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/ListTables.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.clientImpl.Table;
-import org.apache.accumulo.core.clientImpl.Tables;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-
-/**
- * This little program is used by the functional test to get a list of table ids.
- */
-public class ListTables {
- public static void main(String[] args) throws Exception {
- ServerUtilOpts opts = new ServerUtilOpts();
- opts.parseArgs(ListTables.class.getName(), args);
- for (Entry<String,Table.ID> table : Tables.getNameToIdMap(opts.getClientContext()).entrySet())
- System.out.println(table.getKey() + " => " + table.getValue());
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/NullBatchWriter.java b/test/src/main/java/org/apache/accumulo/test/NullBatchWriter.java
deleted file mode 100644
index 4158b2c395..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/NullBatchWriter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.data.Mutation;
-
-public class NullBatchWriter implements BatchWriter {
-
- private int mutationsAdded;
- private long startTime;
-
- @Override
- public void addMutation(Mutation m) throws MutationsRejectedException {
- if (mutationsAdded == 0) {
- startTime = System.currentTimeMillis();
- }
- mutationsAdded++;
- m.numBytes();
- }
-
- @Override
- public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
- for (Mutation mutation : iterable) {
- addMutation(mutation);
- }
- }
-
- @Override
- public void close() throws MutationsRejectedException {
- flush();
- }
-
- @Override
- public void flush() throws MutationsRejectedException {
- System.out.printf("Mutation add rate : %,6.2f mutations/sec%n",
- mutationsAdded / ((System.currentTimeMillis() - startTime) / 1000.0));
- mutationsAdded = 0;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/QueryMetadataTable.java b/test/src/main/java/org/apache/accumulo/test/QueryMetadataTable.java
deleted file mode 100644
index 32c5ca2075..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/QueryMetadataTable.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-public class QueryMetadataTable {
- private static final Logger log = LoggerFactory.getLogger(QueryMetadataTable.class);
-
- static String location;
-
- static class MDTQuery implements Runnable {
-
- private AccumuloClient client;
- private Text row;
-
- MDTQuery(AccumuloClient client, Text row) {
- this.client = client;
- this.row = row;
- }
-
- @Override
- public void run() {
- Scanner mdScanner = null;
- try {
- KeyExtent extent = new KeyExtent(row, (Text) null);
-
- mdScanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- Text row = extent.getMetadataEntry();
-
- mdScanner.setRange(new Range(row));
-
- for (Entry<Key,Value> entry : mdScanner) {
- if (!entry.getKey().getRow().equals(row))
- break;
- }
-
- } catch (TableNotFoundException e) {
- log.error("Table '" + MetadataTable.NAME + "' not found.", e);
- throw new RuntimeException(e);
- } finally {
- if (mdScanner != null) {
- mdScanner.close();
- }
- }
- }
- }
-
- static class Opts extends ServerUtilOpts {
- @Parameter(names = "--numQueries", description = "number of queries to run")
- int numQueries = 1;
- @Parameter(names = "--numThreads", description = "number of threads used to run the queries")
- int numThreads = 1;
- }
-
- public static void main(String[] args) throws TableNotFoundException {
- Opts opts = new Opts();
- ScannerOpts scanOpts = new ScannerOpts();
- opts.parseArgs(QueryMetadataTable.class.getName(), args, scanOpts);
-
- try (AccumuloClient accumuloClient = opts.createClient()) {
- HashSet<Text> rowSet = new HashSet<>();
-
- int count = 0;
-
- try (Scanner scanner = accumuloClient.createScanner(MetadataTable.NAME, opts.auths)) {
- scanner.setBatchSize(scanOpts.scanBatchSize);
- Text mdrow = new Text(TabletsSection.getRow(MetadataTable.ID, null));
-
- for (Entry<Key,Value> entry : scanner) {
- System.out.print(".");
- if (count % 72 == 0) {
- System.out.printf(" %,d%n", count);
- }
- if (entry.getKey().compareRow(mdrow) == 0 && entry.getKey().getColumnFamily()
- .compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
- System.out.println(entry.getKey() + " " + entry.getValue());
- location = entry.getValue().toString();
- }
-
- if (!entry.getKey().getRow().toString().startsWith(MetadataTable.ID.canonicalID()))
- rowSet.add(entry.getKey().getRow());
- count++;
- }
- }
-
- System.out.printf(" %,d%n", count);
-
- ArrayList<Text> rows = new ArrayList<>(rowSet);
-
- Random r = new SecureRandom();
-
- ExecutorService tp = Executors.newFixedThreadPool(opts.numThreads);
-
- long t1 = System.currentTimeMillis();
-
- for (int i = 0; i < opts.numQueries; i++) {
- int index = r.nextInt(rows.size());
- MDTQuery mdtq = new MDTQuery(accumuloClient, rows.get(index));
- tp.submit(mdtq);
- }
-
- tp.shutdown();
-
- try {
- tp.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- log.error("Failed while awaiting the ExcecutorService to terminate.", e);
- throw new RuntimeException(e);
- }
-
- long t2 = System.currentTimeMillis();
- double delta = (t2 - t1) / 1000.0;
- System.out.println("time : " + delta + " queries per sec : " + (opts.numQueries / delta));
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationTablesPrinterThread.java b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationTablesPrinterThread.java
deleted file mode 100644
index 59105c2729..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationTablesPrinterThread.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.replication;
-
-import java.io.PrintStream;
-
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.util.Daemon;
-import org.apache.accumulo.server.replication.PrintReplicationRecords;
-
-public class ReplicationTablesPrinterThread extends Daemon {
-
- private PrintStream out;
- private PrintReplicationRecords printer;
-
- public ReplicationTablesPrinterThread(AccumuloClient client, PrintStream out) {
- printer = new PrintReplicationRecords(client, out);
- this.out = out;
- }
-
- @Override
- public void run() {
- while (true) {
- printer.run();
- out.println();
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
- }
- }
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java b/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java
deleted file mode 100644
index 779b170336..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-public class EstimateInMemMapOverhead {
-
- private static void check(int numEntries, int keyLen, int colFamLen, int colQualLen,
- int colVisLen, int dataLen) {
- new IntObjectMemoryUsageCheck(numEntries).run();
- new InMemoryMapMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, colVisLen, dataLen)
- .run();
- new TextMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, dataLen).run();
- new MutationMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, dataLen).run();
- }
-
- public static void main(String[] args) {
- check(10000, 10, 4, 4, 4, 20);
- check(100000, 10, 4, 4, 4, 20);
- check(500000, 10, 4, 4, 4, 20);
- check(1000000, 10, 4, 4, 4, 20);
- check(2000000, 10, 4, 4, 4, 20);
-
- check(10000, 20, 5, 5, 5, 500);
- check(100000, 20, 5, 5, 5, 500);
- check(500000, 20, 5, 5, 5, 500);
- check(1000000, 20, 5, 5, 5, 500);
- check(2000000, 20, 5, 5, 5, 500);
-
- check(10000, 40, 10, 10, 10, 1000);
- check(100000, 40, 10, 10, 10, 1000);
- check(500000, 40, 10, 10, 10, 1000);
- check(1000000, 40, 10, 10, 10, 1000);
- check(2000000, 40, 10, 10, 10, 1000);
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
deleted file mode 100644
index f2c10607ea..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import java.util.Collections;
-
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
-import org.apache.accumulo.test.InMemoryMapIT;
-import org.apache.accumulo.tserver.InMemoryMap;
-import org.apache.hadoop.io.Text;
-
-class InMemoryMapMemoryUsageCheck extends MemoryUsageCheck {
-
- private int keyLen;
- private int colFamLen;
- private int colQualLen;
- private int colVisLen;
- private int dataLen;
-
- private InMemoryMap imm;
- private Text key;
- private Text colf;
- private Text colq;
- private ColumnVisibility colv;
- private int passes;
-
- InMemoryMapMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int colVisLen,
- int dataLen) {
- this.keyLen = keyLen;
- this.colFamLen = colFamLen;
- this.colQualLen = colQualLen;
- this.dataLen = dataLen;
- this.passes = passes;
- this.colVisLen = colVisLen;
-
- }
-
- @Override
- void init() {
- try {
- imm = new InMemoryMap(DefaultConfiguration.getInstance(), InMemoryMapIT.getServerContext());
- } catch (LocalityGroupConfigurationError e) {
- throw new RuntimeException(e);
- }
- key = new Text();
-
- colf = new Text(String.format("%0" + colFamLen + "d", 0));
- colq = new Text(String.format("%0" + colQualLen + "d", 0));
- colv = new ColumnVisibility(String.format("%0" + colVisLen + "d", 0));
- }
-
- @Override
- public void addEntry(int i) {
- key.set(String.format("%0" + keyLen + "d", i));
-
- Mutation m = new Mutation(key);
-
- byte data[] = new byte[dataLen];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) (j % 10 + 65);
- }
- Value idata = new Value(data);
-
- m.put(colf, colq, colv, idata);
-
- imm.mutate(Collections.singletonList(m));
-
- }
-
- @Override
- public int getEstimatedBytesPerEntry() {
- return keyLen + colFamLen + colQualLen + dataLen + 4 + colVisLen;
- }
-
- @Override
- public void clear() {
- imm = null;
- key = null;
- colf = null;
- colq = null;
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
- @Override
- String getName() {
- return "IMM " + keyLen + " " + colFamLen + " " + colQualLen + " " + dataLen;
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java
deleted file mode 100644
index 74016a2b7c..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-class IntObjectMemoryUsageCheck extends MemoryUsageCheck {
-
- private int passes;
- private Object data[];
-
- static class SimpleObject {
- int d;
-
- SimpleObject(int d) {
- this.d = d;
- }
- }
-
- IntObjectMemoryUsageCheck(int numPasses) {
- this.passes = numPasses;
- }
-
- @Override
- void init() {
- data = new Object[passes];
- }
-
- @Override
- void addEntry(int i) {
- data[i] = new SimpleObject(i);
-
- }
-
- @Override
- void clear() {}
-
- @Override
- int getEstimatedBytesPerEntry() {
- return 4;
- }
-
- @Override
- String getName() {
- return "int obj";
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java
deleted file mode 100644
index 946d71f088..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
-abstract class MemoryUsageCheck {
- abstract void addEntry(int i);
-
- abstract int getEstimatedBytesPerEntry();
-
- abstract void clear();
-
- abstract int getNumPasses();
-
- abstract String getName();
-
- abstract void init();
-
- @SuppressFBWarnings(value = "DM_GC", justification = "gc is okay for test")
- public void run() {
- System.gc();
- long usedMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- int count = 0;
- while (usedMem > 1024 * 1024 && count < 10) {
- System.gc();
- usedMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- count++;
- }
-
- init();
-
- for (int i = 0; i < getNumPasses(); i++) {
- addEntry(i);
- }
-
- System.gc();
-
- long memSize = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
- - usedMem;
-
- double actualBytesPerEntry = memSize / (double) getNumPasses();
- double expectedBytesPerEntry = getEstimatedBytesPerEntry();
- double diff = actualBytesPerEntry - expectedBytesPerEntry;
- double ratio = actualBytesPerEntry / expectedBytesPerEntry * 100;
-
- System.out.printf("%30s | %,10d | %6.2fGB | %6.2f | %6.2f | %6.2f | %6.2f%s%n", getName(),
- getNumPasses(), memSize / (1024 * 1024 * 1024.0), actualBytesPerEntry,
- expectedBytesPerEntry, diff, ratio, "%");
-
- clear();
-
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java
deleted file mode 100644
index f60d34705d..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-class MutationMemoryUsageCheck extends MemoryUsageCheck {
-
- private int keyLen;
- private int colFamLen;
- private int colQualLen;
- private int dataLen;
-
- private Mutation[] mutations;
- private Text key;
- private Text colf;
- private Text colq;
- private int passes;
-
- MutationMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) {
- this.keyLen = keyLen;
- this.colFamLen = colFamLen;
- this.colQualLen = colQualLen;
- this.dataLen = dataLen;
- this.passes = passes;
- mutations = new Mutation[passes];
-
- }
-
- @Override
- void init() {
- key = new Text();
-
- colf = new Text(String.format("%0" + colFamLen + "d", 0));
- colq = new Text(String.format("%0" + colQualLen + "d", 0));
-
- byte data[] = new byte[dataLen];
- for (int i = 0; i < data.length; i++) {
- data[i] = (byte) (i % 10 + 65);
- }
- }
-
- @Override
- public void addEntry(int i) {
- key.set(String.format("%0" + keyLen + "d", i));
-
- Mutation m = new Mutation(key);
-
- byte data[] = new byte[dataLen];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) (j % 10 + 65);
- }
- Value idata = new Value(data);
-
- m.put(colf, colq, idata);
-
- mutations[i] = m;
- }
-
- @Override
- public int getEstimatedBytesPerEntry() {
- return keyLen + colFamLen + colQualLen + dataLen;
- }
-
- @Override
- public void clear() {
- key = null;
- colf = null;
- colq = null;
- mutations = null;
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
- @Override
- String getName() {
- return "Mutation " + keyLen + " " + colFamLen + " " + colQualLen + " " + dataLen;
- }
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java
deleted file mode 100644
index 72d8a33fe2..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.memory;
-
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-class TextMemoryUsageCheck extends MemoryUsageCheck {
-
- private int keyLen;
- private int colFamLen;
- private int colQualLen;
- private int dataLen;
- private TreeMap<Text,Value> map;
- private int passes;
-
- TextMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) {
- this.keyLen = keyLen;
- this.colFamLen = colFamLen;
- this.colQualLen = colQualLen;
- this.dataLen = dataLen;
- this.passes = passes;
-
- }
-
- @Override
- void init() {
- map = new TreeMap<>();
- }
-
- @Override
- public void addEntry(int i) {
- Text key = new Text(
- String.format("%0" + keyLen + "d:%0" + colFamLen + "d:%0" + colQualLen + "d", i, 0, 0)
- .getBytes());
- //
- byte data[] = new byte[dataLen];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) (j % 10 + 65);
- }
- Value value = new Value(data);
-
- map.put(key, value);
-
- }
-
- @Override
- public void clear() {
- map.clear();
- map = null;
- }
-
- @Override
- public int getEstimatedBytesPerEntry() {
- return keyLen + colFamLen + colQualLen + dataLen;
- }
-
- @Override
- int getNumPasses() {
- return passes;
- }
-
- @Override
- String getName() {
- return "Text " + keyLen + " " + colFamLen + " " + colQualLen + " " + dataLen;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java b/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java
deleted file mode 100644
index d7fc1a95c1..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.metadata;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.TreeSet;
-import java.util.UUID;
-
-import org.apache.accumulo.core.cli.ClientOpts;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.clientImpl.Table;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.DataFileValue;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.core.util.Stat;
-import org.apache.accumulo.server.master.state.TServerInstance;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Iterators;
-
-/**
- * This little program can be used to write a lot of metadata entries and measure the performance of
- * varying numbers of threads doing metadata lookups using the batch scanner.
- */
-public class MetadataBatchScan {
-
- private static final Logger log = LoggerFactory.getLogger(MetadataBatchScan.class);
-
- public static void main(String[] args) throws Exception {
-
- ClientOpts opts = new ClientOpts();
- opts.parseArgs(MetadataBatchScan.class.getName(), args);
-
- TreeSet<Long> splits = new TreeSet<>();
- Random r = new SecureRandom();
-
- while (splits.size() < 99999) {
- splits.add((r.nextLong() & 0x7fffffffffffffffL) % 1000000000000L);
- }
-
- Table.ID tid = Table.ID.of("8");
- Text per = null;
-
- ArrayList<KeyExtent> extents = new ArrayList<>();
-
- for (Long split : splits) {
- Text er = new Text(String.format("%012d", split));
- KeyExtent ke = new KeyExtent(tid, er, per);
- per = er;
-
- extents.add(ke);
- }
-
- extents.add(new KeyExtent(tid, null, per));
-
- try (AccumuloClient accumuloClient = opts.createClient()) {
- if (args[0].equals("write")) {
-
- BatchWriter bw = accumuloClient.createBatchWriter(MetadataTable.NAME,
- new BatchWriterConfig());
-
- for (KeyExtent extent : extents) {
- Mutation mut = extent.getPrevRowUpdateMutation();
- new TServerInstance(HostAndPort.fromParts("192.168.1.100", 4567), "DEADBEEF")
- .putLocation(mut);
- bw.addMutation(mut);
- }
-
- bw.close();
- } else if (args[0].equals("writeFiles")) {
- BatchWriter bw = accumuloClient.createBatchWriter(MetadataTable.NAME,
- new BatchWriterConfig());
-
- for (KeyExtent extent : extents) {
-
- Mutation mut = new Mutation(extent.getMetadataEntry());
-
- String dir = "/t-" + UUID.randomUUID();
-
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut,
- new Value(dir.getBytes(UTF_8)));
-
- for (int i = 0; i < 5; i++) {
- mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"),
- new DataFileValue(10000, 1000000).encodeAsValue());
- }
-
- bw.addMutation(mut);
- }
-
- bw.close();
- } else if (args[0].equals("scan")) {
-
- int numThreads = Integer.parseInt(args[1]);
- final int numLoop = Integer.parseInt(args[2]);
- int numLookups = Integer.parseInt(args[3]);
-
- HashSet<Integer> indexes = new HashSet<>();
- while (indexes.size() < numLookups) {
- indexes.add(r.nextInt(extents.size()));
- }
-
- final List<Range> ranges = new ArrayList<>();
- for (Integer i : indexes) {
- ranges.add(extents.get(i).toMetadataRange());
- }
-
- Thread threads[] = new Thread[numThreads];
-
- for (int i = 0; i < threads.length; i++) {
- threads[i] = new Thread(() -> {
- try {
- System.out.println(runScanTest(accumuloClient, numLoop, ranges));
- } catch (Exception e) {
- log.error("Exception while running scan test.", e);
- }
- });
- }
-
- long t1 = System.currentTimeMillis();
-
- for (Thread thread : threads) {
- thread.start();
- }
-
- for (Thread thread : threads) {
- thread.join();
- }
-
- long t2 = System.currentTimeMillis();
-
- System.out.printf("tt : %6.2f%n", (t2 - t1) / 1000.0);
-
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- }
-
- private static ScanStats runScanTest(AccumuloClient accumuloClient, int numLoop,
- List<Range> ranges) throws Exception {
- ScanStats stats = new ScanStats();
-
- try (BatchScanner bs = accumuloClient.createBatchScanner(MetadataTable.NAME,
- Authorizations.EMPTY, 1)) {
- bs.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
- TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(bs);
-
- bs.setRanges(ranges);
-
- // System.out.println(ranges);
- for (int i = 0; i < numLoop; i++) {
- ScanStat ss = scan(bs, ranges, null);
- stats.merge(ss);
- }
- }
- return stats;
- }
-
- private static class ScanStat {
- long delta1;
- long delta2;
- int count1;
- int count2;
- }
-
- private static class ScanStats {
- Stat delta1 = new Stat();
- Stat delta2 = new Stat();
- Stat count1 = new Stat();
- Stat count2 = new Stat();
-
- void merge(ScanStat ss) {
- delta1.addStat(ss.delta1);
- delta2.addStat(ss.delta2);
- count1.addStat(ss.count1);
- count2.addStat(ss.count2);
- }
-
- @Override
- public String toString() {
- return "[" + delta1 + "] [" + delta2 + "]";
- }
- }
-
- private static ScanStat scan(BatchScanner bs, List<Range> ranges, Scanner scanner) {
-
- // System.out.println("ranges : "+ranges);
-
- ScanStat ss = new ScanStat();
-
- long t1 = System.currentTimeMillis();
- int count = Iterators.size(bs.iterator());
- bs.close();
- long t2 = System.currentTimeMillis();
-
- ss.delta1 = t2 - t1;
- ss.count1 = count;
-
- count = 0;
- t1 = System.currentTimeMillis();
- /*
- * for (Range range : ranges) { scanner.setRange(range); for (Entry<Key, Value> entry : scanner)
- * { count++; } }
- */
-
- t2 = System.currentTimeMillis();
-
- ss.delta2 = t2 - t1;
- ss.count2 = count;
-
- return ss;
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java
deleted file mode 100644
index 5fe214ebda..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.nativemap;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.tserver.NativeMap;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-
-public class NativeMapConcurrency {
-
- private static final Logger log = LoggerFactory.getLogger(NativeMapConcurrency.class);
-
- private static final byte ROW_PREFIX[] = {'r'};
- private static final byte COL_PREFIX[] = {'c'};
-
- static Mutation nm(int r) {
- return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 6, 10, ROW_PREFIX)));
- }
-
- private static final Text ET = new Text();
-
- private static void pc(Mutation m, int c, Value v) {
- m.put(new Text(FastFormat.toZeroPaddedString(c, 3, 10, COL_PREFIX)), ET, v);
- }
-
- static NativeMap create(int numRows, int numCols) {
-
- NativeMap nm = new NativeMap();
-
- populate(0, numRows, numCols, nm);
-
- return nm;
-
- }
-
- private static void populate(int start, int numRows, int numCols, NativeMap nm) {
- long t1 = System.currentTimeMillis();
- int mc = 1;
- for (int i = 0; i < numRows; i++) {
- Mutation m = nm(i + start);
- for (int j = 0; j < numCols; j++) {
- Value val = new Value("test".getBytes(UTF_8));
- pc(m, j, val);
- }
- nm.mutate(m, mc++);
- }
- long t2 = System.currentTimeMillis();
-
- System.out.printf("inserted %,d in %,d %,d %,6.2f%n", (numRows * numCols), (t2 - t1), nm.size(),
- rate((numRows * numCols), (t2 - t1)));
- }
-
- private static double rate(int num, long ms) {
- return num / (ms / 1000.0);
- }
-
- static class Opts {
- @Parameter(names = "--rows", description = "rows", required = true)
- int rows = 0;
- @Parameter(names = "--cols", description = "cols")
- int cols = 1;
- @Parameter(names = "--threads", description = "threads")
- int threads = 1;
- @Parameter(names = "--writeThreads", description = "write threads")
- int writeThreads = 1;
- @Parameter(names = "-help", help = true)
- boolean help = false;
- }
-
- public static void main(String[] args) {
- Opts opts = new Opts();
- JCommander jc = new JCommander(opts);
- jc.setProgramName(NativeMapConcurrency.class.getName());
- jc.parse(args);
- if (opts.help) {
- jc.usage();
- return;
- }
- NativeMap nm = create(opts.rows, opts.cols);
- runTest(nm, opts.rows, opts.cols, opts.threads, opts.writeThreads);
- nm.delete();
- }
-
- static class ScanTask implements Runnable {
-
- private NativeMap nm;
-
- ScanTask(NativeMap nm) {
- this.nm = nm;
- }
-
- @Override
- public void run() {
-
- for (int i = 0; i < 10; i++) {
-
- Iterator<Entry<Key,Value>> iter = nm.iterator();
-
- long t1 = System.currentTimeMillis();
-
- int count = 0;
-
- while (iter.hasNext()) {
- count++;
- iter.next();
- }
-
- long t2 = System.currentTimeMillis();
-
- System.out.printf("%d %,d %,d %,d %,d %,6.2f%n", Thread.currentThread().getId(), (t2 - t1),
- t1, t2, count, rate(count, (t2 - t1)));
- }
- }
-
- }
-
- static class WriteTask implements Runnable {
-
- private int start;
- private int rows;
- private int cols;
- private NativeMap nm;
-
- WriteTask(int start, int rows, int cols, NativeMap nm) {
- this.start = start;
- this.rows = rows;
- this.cols = cols;
- this.nm = nm;
- }
-
- @Override
- public void run() {
- populate(start, rows, cols, nm);
- }
-
- }
-
- private static void runTest(NativeMap nm, int rows, int cols, int numReadThreads,
- int writeThreads) {
-
- Thread threads[] = new Thread[numReadThreads + writeThreads];
-
- for (int i = 0; i < numReadThreads; i++) {
- threads[i] = new Thread(new ScanTask(nm));
- }
-
- int start = 0;
- for (int i = numReadThreads; i < writeThreads + numReadThreads; i++) {
- threads[i] = new Thread(new WriteTask(start, rows, cols, nm));
- // start += rows;
- }
-
- for (Thread thread : threads) {
- thread.start();
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '" + thread.getName() + "'", e);
- }
- }
-
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java
deleted file mode 100644
index c9b05af8e7..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.nativemap;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
-
-import java.security.SecureRandom;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.tserver.NativeMap;
-import org.apache.hadoop.io.Text;
-
-public class NativeMapPerformance {
-
- private static final byte ROW_PREFIX[] = {'r'};
- private static final byte COL_PREFIX[] = {'c'};
-
- static Key newKey(int r, int c) {
- return new Key(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)),
- new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)));
- }
-
- static Mutation newMutation(int r) {
- return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)));
- }
-
- static Text ET = new Text();
-
- private static void pc(Mutation m, int c, Value v) {
- m.put(new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)), ET, Long.MAX_VALUE, v);
- }
-
- static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
-
- SortedMap<Key,Value> tm = null;
- NativeMap nm = null;
-
- if (mapType.equals("SKIP_LIST"))
- tm = new ConcurrentSkipListMap<>();
- else if (mapType.equals("TREE_MAP"))
- tm = Collections.synchronizedSortedMap(new TreeMap<>());
- else if (mapType.equals("NATIVE_MAP"))
- nm = new NativeMap();
- else
- throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
-
- Random rand = new SecureRandom();
-
- // puts
- long tps = System.currentTimeMillis();
-
- if (nm != null) {
- for (int i = 0; i < numRows; i++) {
- int row = rand.nextInt(1000000000);
- Mutation m = newMutation(row);
- for (int j = 0; j < numCols; j++) {
- int col = rand.nextInt(1000000);
- Value val = new Value("test".getBytes(UTF_8));
- pc(m, col, val);
- }
- nm.mutate(m, i);
- }
- } else {
- for (int i = 0; i < numRows; i++) {
- int row = rand.nextInt(1000000000);
- for (int j = 0; j < numCols; j++) {
- int col = rand.nextInt(1000000);
- Key key = newKey(row, col);
- Value val = new Value("test".getBytes(UTF_8));
- tm.put(key, val);
- }
- }
- }
-
- long tpe = System.currentTimeMillis();
-
- // Iteration
- Iterator<Entry<Key,Value>> iter;
- if (nm != null) {
- iter = nm.iterator();
- } else {
- iter = tm.entrySet().iterator();
- }
-
- long tis = System.currentTimeMillis();
-
- while (iter.hasNext()) {
- iter.next();
- }
-
- long tie = System.currentTimeMillis();
-
- rand = new SecureRandom();
- int rowsToLookup[] = new int[numLookups];
- int colsToLookup[] = new int[numLookups];
- for (int i = 0; i < Math.min(numLookups, numRows); i++) {
- int row = rand.nextInt(1000000000);
- int col = -1;
- for (int j = 0; j < numCols; j++) {
- col = rand.nextInt(1000000);
- }
-
- rowsToLookup[i] = row;
- colsToLookup[i] = col;
- }
-
- // get
-
- long tgs = System.currentTimeMillis();
- if (nm != null) {
- for (int i = 0; i < numLookups; i++) {
- Key key = newKey(rowsToLookup[i], colsToLookup[i]);
- if (nm.get(key) == null) {
- throw new RuntimeException(
- "Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
- }
- }
- } else {
- for (int i = 0; i < numLookups; i++) {
- Key key = newKey(rowsToLookup[i], colsToLookup[i]);
- if (tm.get(key) == null) {
- throw new RuntimeException(
- "Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
- }
- }
- }
- long tge = System.currentTimeMillis();
-
- long memUsed = 0;
- if (nm != null) {
- memUsed = nm.getMemoryUsed();
- }
-
- int size = (nm == null ? tm.size() : nm.size());
-
- // delete
- long tds = System.currentTimeMillis();
-
- if (nm != null)
- nm.delete();
-
- long tde = System.currentTimeMillis();
-
- if (tm != null)
- tm.clear();
-
- System.gc();
- System.gc();
- System.gc();
- System.gc();
-
- sleepUninterruptibly(3, TimeUnit.SECONDS);
-
- System.out.printf(
- "mapType:%10s put rate:%,6.2f scan rate:%,6.2f get"
- + " rate:%,6.2f delete time : %6.2f mem : %,d%n",
- "" + mapType, (numRows * numCols) / ((tpe - tps) / 1000.0), (size) / ((tie - tis) / 1000.0),
- numLookups / ((tge - tgs) / 1000.0), (tde - tds) / 1000.0, memUsed);
-
- }
-
- public static void main(String[] args) {
-
- if (args.length != 3) {
- throw new IllegalArgumentException(
- "Usage : " + NativeMapPerformance.class.getName() + " <map type> <rows> <columns>");
- }
-
- String mapType = args[0];
- int rows = Integer.parseInt(args[1]);
- int cols = Integer.parseInt(args[2]);
-
- runPerformanceTest(rows, cols, 10000, mapType);
- runPerformanceTest(rows, cols, 10000, mapType);
- runPerformanceTest(rows, cols, 10000, mapType);
-
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java
deleted file mode 100644
index 15c1c5af22..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.nativemap;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.OpTimer;
-import org.apache.accumulo.tserver.NativeMap;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class NativeMapStress {
-
- private static final Logger log = LoggerFactory.getLogger(NativeMapStress.class);
-
- public static void main(String[] args) {
- testLotsOfMapDeletes(true);
- testLotsOfMapDeletes(false);
- testLotsOfOverwrites();
- testLotsOfGetsAndScans();
- }
-
- private static void put(NativeMap nm, String row, String val, int mc) {
- Mutation m = new Mutation(new Text(row));
- m.put(new Text(), new Text(), Long.MAX_VALUE, new Value(val.getBytes(UTF_8)));
- nm.mutate(m, mc);
- }
-
- private static void testLotsOfGetsAndScans() {
-
- ArrayList<Thread> threads = new ArrayList<>();
-
- final int numThreads = 8;
- final int totalGets = 100000000;
- final int mapSizePerThread = (int) (4000000 / (double) numThreads);
- final int getsPerThread = (int) (totalGets / (double) numThreads);
-
- for (int tCount = 0; tCount < numThreads; tCount++) {
- Runnable r = new Runnable() {
- @Override
- public void run() {
- NativeMap nm = new NativeMap();
-
- Random r = new SecureRandom();
-
- OpTimer timer = null;
- AtomicLong nextOpid = new AtomicLong();
-
- if (log.isInfoEnabled()) {
- log.info("tid={} oid={} Creating map of size {}", Thread.currentThread().getId(),
- nextOpid.get(), mapSizePerThread);
- timer = new OpTimer().start();
- }
-
- for (int i = 0; i < mapSizePerThread; i++) {
- String row = String.format("r%08d", i);
- String val = row + "v";
- put(nm, row, val, i);
- }
-
- if (timer != null) {
-
- // stop and log created elapsed time
- timer.stop();
- log.info("tid={} oid={} Created map of size {} in {}", Thread.currentThread().getId(),
- nextOpid.getAndIncrement(), nm.size(),
- String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
-
- // start timer for gets
- log.info("tid={} oid={} Doing {} gets()", Thread.currentThread().getId(),
- nextOpid.get(), getsPerThread);
- timer.reset().start();
- }
-
- for (int i = 0; i < getsPerThread; i++) {
- String row = String.format("r%08d", r.nextInt(mapSizePerThread));
- String val = row + "v";
-
- Value value = nm.get(new Key(new Text(row)));
- if (value == null || !value.toString().equals(val)) {
- log.error("nm.get({}) failed", row);
- }
- }
-
- if (timer != null) {
-
- // stop and log created elapsed time
- timer.stop();
- log.info("tid={} oid={} Finished {} gets in {}", Thread.currentThread().getId(),
- nextOpid.getAndIncrement(), getsPerThread,
- String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
-
- // start timer for random iterations
- log.info("tid={} oid={} Doing {} random iterations", Thread.currentThread().getId(),
- nextOpid.get(), getsPerThread);
- timer.reset().start();
- }
-
- int scanned = 0;
-
- for (int i = 0; i < getsPerThread; i++) {
- int startRow = r.nextInt(mapSizePerThread);
- String row = String.format("r%08d", startRow);
-
- Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
-
- int count = 0;
-
- while (iter.hasNext() && count < 10) {
- String row2 = String.format("r%08d", startRow + count);
- String val2 = row2 + "v";
-
- Entry<Key,Value> entry = iter.next();
- if (!entry.getValue().toString().equals(val2)
- || !entry.getKey().equals(new Key(new Text(row2)))) {
- log.error("nm.iter({}) failed row = {} count = {} row2 = {} val2 = {}", row2, row,
- count, row, val2);
- }
-
- count++;
- }
-
- scanned += count;
- }
-
- if (timer != null) {
-
- // stop and log created elapsed time
- timer.stop();
- log.info("tid={} oid={} Finished {} random iterations (scanned = {}) in {}",
- Thread.currentThread().getId(), nextOpid.getAndIncrement(), getsPerThread, scanned,
- String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
-
- }
-
- nm.delete();
- }
- };
-
- Thread t = new Thread(r);
- t.start();
-
- threads.add(t);
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '{}'.", thread.getName(), e);
- throw new RuntimeException(e);
- }
- }
- }
-
- private static void testLotsOfMapDeletes(final boolean doRemoves) {
- final int numThreads = 8;
- final int rowRange = 10000;
- final int mapsPerThread = 50;
- final int totalInserts = 100000000;
- final int insertsPerMapPerThread = (int) (totalInserts / (double) numThreads / mapsPerThread);
-
- System.out.println("insertsPerMapPerThread " + insertsPerMapPerThread);
-
- ArrayList<Thread> threads = new ArrayList<>();
-
- for (int i = 0; i < numThreads; i++) {
- Runnable r = new Runnable() {
- @Override
- public void run() {
-
- int inserts = 0;
- int removes = 0;
-
- for (int i = 0; i < mapsPerThread; i++) {
-
- NativeMap nm = new NativeMap();
-
- for (int j = 0; j < insertsPerMapPerThread; j++) {
- String row = String.format("r%08d", j % rowRange);
- String val = row + "v";
- put(nm, row, val, j);
- inserts++;
- }
-
- if (doRemoves) {
- Iterator<Entry<Key,Value>> iter = nm.iterator();
- while (iter.hasNext()) {
- iter.next();
- iter.remove();
- removes++;
- }
- }
-
- nm.delete();
- }
-
- System.out.println("inserts " + inserts + " removes " + removes + " "
- + Thread.currentThread().getName());
- }
- };
-
- Thread t = new Thread(r);
- t.start();
-
- threads.add(t);
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '{}'.", thread.getName(), e);
- throw new RuntimeException(e);
- }
- }
- }
-
- private static void testLotsOfOverwrites() {
- final Map<Integer,NativeMap> nativeMaps = new HashMap<>();
-
- int numThreads = 8;
- final int insertsPerThread = (int) (100000000 / (double) numThreads);
- final int rowRange = 10000;
- final int numMaps = 50;
-
- ArrayList<Thread> threads = new ArrayList<>();
-
- for (int i = 0; i < numThreads; i++) {
- Runnable r = new Runnable() {
- @Override
- public void run() {
- Random r = new SecureRandom();
- int inserts = 0;
-
- for (int i = 0; i < insertsPerThread / 100.0; i++) {
- int map = r.nextInt(numMaps);
-
- NativeMap nm;
-
- synchronized (nativeMaps) {
- nm = nativeMaps.get(map);
- if (nm == null) {
- nm = new NativeMap();
- nativeMaps.put(map, nm);
-
- }
- }
-
- synchronized (nm) {
- for (int j = 0; j < 100; j++) {
- String row = String.format("r%08d", r.nextInt(rowRange));
- String val = row + "v";
- put(nm, row, val, j);
- inserts++;
- }
- }
- }
-
- System.out.println("inserts " + inserts + " " + Thread.currentThread().getName());
- }
- };
-
- Thread t = new Thread(r);
- t.start();
-
- threads.add(t);
- }
-
- for (Thread thread : threads) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- log.error("Could not join thread '{}'.", thread.getName(), e);
- throw new RuntimeException(e);
- }
- }
-
- Set<Entry<Integer,NativeMap>> es = nativeMaps.entrySet();
- for (Entry<Integer,NativeMap> entry : es) {
- entry.getValue().delete();
- }
- }
-
-}
diff --git a/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java b/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java
deleted file mode 100644
index 4e72f1b11c..0000000000
--- a/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.util.tablet;
-
-import org.apache.accumulo.core.clientImpl.ClientContext;
-import org.apache.accumulo.core.clientImpl.Table;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.rpc.ThriftUtil;
-import org.apache.accumulo.core.tabletserver.thrift.TDurability;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-public class WrongTablet {
-
- static class Opts extends ServerUtilOpts {
- @Parameter(names = "--location", required = true)
- String location;
- }
-
- public static void main(String[] args) {
- final Opts opts = new Opts();
- opts.parseArgs(WrongTablet.class.getName(), args);
-
- final HostAndPort location = HostAndPort.fromString(opts.location);
- final ClientContext context = opts.getClientContext();
-
- try {
- TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context);
-
- Mutation mutation = new Mutation(new Text("row_0003750001"));
- mutation.putDelete(new Text("colf"), new Text("colq"));
- client.update(Tracer.traceInfo(), context.rpcCreds(),
- new KeyExtent(Table.ID.of("!!"), null, new Text("row_0003750000")).toThrift(),
- mutation.toThrift(), TDurability.DEFAULT);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services