You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by do...@apache.org on 2021/09/10 19:07:06 UTC
[accumulo-testing] branch main updated: Refactor out Non API code
(#150)
This is an automated email from the ASF dual-hosted git repository.
domgarguilo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new e89cd6d Refactor out Non API code (#150)
e89cd6d is described below
commit e89cd6dbffe6a5031ae65c52035ed818bff1fa38
Author: Dom G <47...@users.noreply.github.com>
AuthorDate: Fri Sep 10 15:06:59 2021 -0400
Refactor out Non API code (#150)
Removed or replaced usages of non-API code
---
contrib/import-control.xml | 12 ----
.../org/apache/accumulo/testing/TestProps.java | 4 ++
.../testing/continuous/ContinuousIngest.java | 2 +-
.../apache/accumulo/testing/ingest/TestIngest.java | 6 +-
.../accumulo/testing/ingest/VerifyIngest.java | 13 +----
.../apache/accumulo/testing/merkle/MerkleTree.java | 2 +-
.../performance/tests/HighSplitCreationPT.java | 4 +-
.../testing/performance/tests/ScanExecutorPT.java | 3 +-
.../performance/tests/SplitBalancingPT.java | 5 +-
.../testing/performance/util/TestData.java | 2 +-
.../testing/randomwalk/concurrent/Replication.java | 8 +--
.../apache/accumulo/testing/util/FastFormat.java | 67 ++++++++++++++++++++++
12 files changed, 88 insertions(+), 40 deletions(-)
diff --git a/contrib/import-control.xml b/contrib/import-control.xml
index 840881f..0ca22cc 100644
--- a/contrib/import-control.xml
+++ b/contrib/import-control.xml
@@ -37,18 +37,6 @@
<allow pkg="org.apache.accumulo.core.conf"/>
<allow class="org.apache.accumulo.core.util.threads.ThreadPools"/>
- <!-- TODO refactor code to remove the following exceptions -->
- <allow class="org.apache.accumulo.core.metadata.MetadataTable"/>
- <allow class="org.apache.accumulo.core.replication.ReplicationTable"/>
- <allow class="org.apache.accumulo.core.spi.scan.HintScanPrioritizer"/>
- <allow class="org.apache.accumulo.core.clientImpl.TabletServerBatchWriter"/>
- <allow class="org.apache.accumulo.core.util.FastFormat"/>
- <allow class="org.apache.accumulo.core.util.Pair"/>
- <allow class="org.apache.accumulo.core.trace.Trace"/>
- <allow class="org.apache.accumulo.core.trace.TraceSamplers"/>
- <allow class="org.apache.accumulo.core.trace.Span"/>
- <allow class="org.apache.accumulo.core.trace.DistributedTrace"/>
- <!-- End TODO section -->
<!-- disallow everything else coming from accumulo -->
<disallow pkg="org.apache.accumulo"/>
diff --git a/src/main/java/org/apache/accumulo/testing/TestProps.java b/src/main/java/org/apache/accumulo/testing/TestProps.java
index f15ca31..c4c8948 100644
--- a/src/main/java/org/apache/accumulo/testing/TestProps.java
+++ b/src/main/java/org/apache/accumulo/testing/TestProps.java
@@ -46,6 +46,10 @@ public class TestProps {
public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb";
// Number of cores given to each YARN container
public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores";
+ // Name of metadata table
+ public static final String METADATA_TABLE_NAME = "accumulo.metadata";
+ // Name of replication table
+ public static final String REPLICATION_TABLE_NAME = "accumulo.replication";
/** Continuous ingest test properties **/
/** Common **/
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
index 0f6b04d..66f5152 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
@@ -38,8 +38,8 @@ import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.TestProps;
+import org.apache.accumulo.testing.util.FastFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java
index 4fb90ee..0cd0024 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java
@@ -35,7 +35,6 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.rfile.RFileWriter;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
-import org.apache.accumulo.core.clientImpl.TabletServerBatchWriter;
import org.apache.accumulo.core.conf.ClientProperty;
import org.apache.accumulo.core.data.ConstraintViolationSummary;
import org.apache.accumulo.core.data.Key;
@@ -44,8 +43,8 @@ import org.apache.accumulo.core.data.TabletId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.cli.ClientOpts;
+import org.apache.accumulo.testing.util.FastFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
@@ -184,7 +183,8 @@ public class TestIngest {
try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) {
if (opts.debug)
- Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE);
+ Logger.getLogger("org.apache.accumulo.core.clientImpl.TabletServerBatchWriter")
+ .setLevel(Level.TRACE);
// test batch update
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
index b783e8c..d7c36de 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
@@ -32,8 +32,6 @@ import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-// import org.apache.accumulo.core.trace.DistributedTrace;
-// import org.apache.accumulo.core.trace.Trace;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,18 +60,9 @@ public class VerifyIngest {
opts.parseArgs(VerifyIngest.class.getName(), args);
try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) {
if (opts.trace) {
- /*
- * String name = VerifyIngest.class.getSimpleName(); DistributedTrace.enable();
- * Trace.on(name); Trace.data("cmdLine", Arrays.asList(args).toString());
- */
+ throw new UnsupportedOperationException("Tracing is enabled but not currently supported");
}
-
verifyIngest(client, opts);
-
- } finally {
- /*
- * Trace.off(); DistributedTrace.disable();
- */
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java b/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java
index cc7f810..a9e213a 100644
--- a/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java
+++ b/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java
@@ -21,7 +21,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.math3.util.Pair;
import com.google.common.collect.Iterables;
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java
index 0678c1f..186fbbf 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java
@@ -22,7 +22,7 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
import org.apache.accumulo.testing.performance.Report;
@@ -53,7 +53,7 @@ public class HighSplitCreationPT implements PerformanceTest {
AccumuloClient client = env.getClient();
client.tableOperations().create(TABLE_NAME);
- client.tableOperations().addSplits(MetadataTable.NAME, getMetadataTableSplits());
+ client.tableOperations().addSplits(TestProps.METADATA_TABLE_NAME, getMetadataTableSplits());
SortedSet<Text> splits = getTestTableSplits();
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
index ae9b8c0..e59d480 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
@@ -35,7 +35,6 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.spi.scan.HintScanPrioritizer;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
import org.apache.accumulo.testing.performance.Report;
@@ -57,7 +56,7 @@ public class ScanExecutorPT implements PerformanceTest {
private static final int NUM_QUALS = 10;
private static final String SCAN_EXECUTOR_THREADS = "2";
- private static final String SCAN_PRIORITIZER = HintScanPrioritizer.class.getName();
+ private static final String SCAN_PRIORITIZER = "org.apache.accumulo.core.spi.scan.HintScanPrioritizer";
private static final String TEST_DESC = "Scan Executor Test. Test running lots of short scans "
+ "while long scans are running in the background. Each short scan reads a random row and "
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java
index c88b241..e36c62e 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java
@@ -27,8 +27,8 @@ import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
import org.apache.accumulo.testing.performance.Report;
@@ -97,7 +97,8 @@ public class SplitBalancingPT implements PerformanceTest {
private Map<String,Integer> getTablets(final AccumuloClient client) {
Map<String,Integer> tablets = new HashMap<>();
- try (Scanner scanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ try (Scanner scanner = client.createScanner(TestProps.METADATA_TABLE_NAME,
+ Authorizations.EMPTY)) {
scanner.fetchColumnFamily(TSERVER_ASSIGNED_TABLETS_COL_FAM);
Range range = new Range(null, false, RESERVED_PREFIX, false);
scanner.setRange(range);
diff --git a/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java b/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java
index 66a7f7e..e17106e 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java
@@ -20,7 +20,7 @@ package org.apache.accumulo.testing.performance.util;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.testing.util.FastFormat;
public class TestData {
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java
index 71d8f65..fb775b4 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java
@@ -46,8 +46,8 @@ import org.apache.accumulo.core.conf.ClientProperty;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.TestProps;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.randomwalk.State;
import org.apache.accumulo.testing.randomwalk.Test;
@@ -80,13 +80,13 @@ public class Replication extends Test {
iOps.setProperty(REPLICATION_WORK_PROCESSOR_PERIOD.getKey(), "1s");
// Ensure the replication table is online
- ReplicationTable.setOnline(c);
- boolean online = ReplicationTable.isOnline(c);
+ tOps.online(TestProps.REPLICATION_TABLE_NAME, true);
+ boolean online = tOps.isOnline(TestProps.REPLICATION_TABLE_NAME);
for (int i = 0; i < 10; i++) {
if (online)
break;
sleepUninterruptibly(2, TimeUnit.SECONDS);
- online = ReplicationTable.isOnline(c);
+ online = tOps.isOnline(TestProps.REPLICATION_TABLE_NAME);
}
assertTrue("Replication table was not online", online);
diff --git a/src/main/java/org/apache/accumulo/testing/util/FastFormat.java b/src/main/java/org/apache/accumulo/testing/util/FastFormat.java
new file mode 100644
index 0000000..541dc7c
--- /dev/null
+++ b/src/main/java/org/apache/accumulo/testing/util/FastFormat.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.testing.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Preconditions;
+
+public class FastFormat {
+ // Copied over from accumulo
+ // this 7 to 8 times faster than String.format("%s%06d",prefix, num)
+ public static byte[] toZeroPaddedString(long num, int width, int radix, byte[] prefix) {
+ Preconditions.checkArgument(num >= 0);
+ String strNum = Long.toString(num, radix);
+ byte[] ret = new byte[Math.max(strNum.length(), width) + prefix.length];
+ if (toZeroPaddedString(ret, 0, strNum, width, prefix) != ret.length)
+ throw new RuntimeException(" Did not format to expected width " + num + " " + width + " "
+ + radix + " " + new String(prefix, UTF_8));
+ return ret;
+ }
+
+ public static int toZeroPaddedString(byte[] output, int outputOffset, long num, int width,
+ int radix, byte[] prefix) {
+ Preconditions.checkArgument(num >= 0);
+
+ String strNum = Long.toString(num, radix);
+
+ return toZeroPaddedString(output, outputOffset, strNum, width, prefix);
+ }
+
+ private static int toZeroPaddedString(byte[] output, int outputOffset, String strNum, int width,
+ byte[] prefix) {
+
+ int index = outputOffset;
+
+ for (byte b : prefix) {
+ output[index++] = b;
+ }
+
+ int end = width - strNum.length() + index;
+
+ while (index < end)
+ output[index++] = '0';
+
+ for (int i = 0; i < strNum.length(); i++) {
+ output[index++] = (byte) strNum.charAt(i);
+ }
+
+ return index - outputOffset;
+ }
+}