You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by do...@apache.org on 2022/02/22 14:08:09 UTC
[accumulo-testing] branch main updated: Use `NewTableConfiguration().withSplits()` instead of `addSplits()` (#193)
This is an automated email from the ASF dual-hosted git repository.
domgarguilo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new cce415d Use `NewTableConfiguration().withSplits()` instead of `addSplits()` (#193)
cce415d is described below
commit cce415d96457e438162515d3845d1447efef1bf8
Author: Dom G <do...@gmail.com>
AuthorDate: Tue Feb 22 09:08:02 2022 -0500
Use `NewTableConfiguration().withSplits()` instead of `addSplits()` (#193)
* Convert to use NTC
---
.../apache/accumulo/testing/ingest/TestIngest.java | 18 +++++---
.../testing/performance/tests/RollWALPT.java | 7 ++-
.../testing/randomwalk/image/ImageFixture.java | 5 ++-
.../testing/randomwalk/multitable/CopyTable.java | 6 +--
.../testing/randomwalk/multitable/CreateTable.java | 12 +++---
.../testing/randomwalk/shard/ShardFixture.java | 50 ++++++++++------------
.../accumulo/testing/scalability/Ingest.java | 9 ++--
7 files changed, 54 insertions(+), 53 deletions(-)
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java
index 878c8dc..30f0770 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.rfile.RFileWriter;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
@@ -112,13 +113,16 @@ public class TestIngest {
TreeSet<Text> splits = getSplitPoints(args.startRow, args.startRow + args.rows,
args.numsplits);
- if (!client.tableOperations().exists(args.tableName))
- client.tableOperations().create(args.tableName);
- try {
- client.tableOperations().addSplits(args.tableName, splits);
- } catch (TableNotFoundException ex) {
- // unlikely
- throw new RuntimeException(ex);
+ if (!client.tableOperations().exists(args.tableName)) {
+ client.tableOperations().create(args.tableName,
+ new NewTableConfiguration().withSplits(splits));
+ } else {
+ try {
+ client.tableOperations().addSplits(args.tableName, splits);
+ } catch (TableNotFoundException ex) {
+ // unlikely
+ throw new RuntimeException(ex);
+ }
}
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/RollWALPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/RollWALPT.java
index 4424cd6..37f34d9 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/RollWALPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/RollWALPT.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -115,10 +116,8 @@ public class RollWALPT implements PerformanceTest {
}
private void initTable(final String tableName, final AccumuloClient client)
- throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
- TableExistsException {
- client.tableOperations().create(tableName);
- client.tableOperations().addSplits(tableName, getSplits());
+ throws AccumuloSecurityException, AccumuloException, TableExistsException {
+ client.tableOperations().create(tableName, new NewTableConfiguration().withSplits(getSplits()));
client.instanceOperations().waitForBalance();
}
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/image/ImageFixture.java b/src/main/java/org/apache/accumulo/testing/randomwalk/image/ImageFixture.java
index 28cf33e..6bc3c2b 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/image/ImageFixture.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/image/ImageFixture.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.testing.randomwalk.Fixture;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.randomwalk.State;
@@ -59,8 +60,8 @@ public class ImageFixture extends Fixture {
state.set("indexTableName", indexTableName);
try {
- client.tableOperations().create(imageTableName);
- client.tableOperations().addSplits(imageTableName, splits);
+ client.tableOperations().create(imageTableName,
+ new NewTableConfiguration().withSplits(splits));
log.debug("Created table " + imageTableName + " (id:"
+ client.tableOperations().tableIdMap().get(imageTableName) + ")");
} catch (TableExistsException e) {
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTable.java b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTable.java
index eaffa93..bf96a16 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTable.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTable.java
@@ -21,6 +21,7 @@ import java.util.Properties;
import java.util.Random;
import java.util.TreeSet;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.randomwalk.State;
import org.apache.accumulo.testing.randomwalk.Test;
@@ -66,9 +67,8 @@ public class CopyTable extends Test {
log.debug("copying " + srcTableName + " to " + dstTableName);
- env.getAccumuloClient().tableOperations().create(dstTableName);
-
- env.getAccumuloClient().tableOperations().addSplits(dstTableName, splits);
+ env.getAccumuloClient().tableOperations().create(dstTableName,
+ new NewTableConfiguration().withSplits(splits));
if (ToolRunner.run(env.getHadoopConfiguration(), new CopyTool(), args) != 0) {
log.error("Failed to run map/red verify");
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
index 3ef2d27..ec0ad96 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
@@ -22,6 +22,7 @@ import java.util.TreeSet;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.randomwalk.State;
import org.apache.accumulo.testing.randomwalk.Test;
@@ -48,14 +49,11 @@ public class CreateTable extends Test {
int nextId = (Integer) state.get("nextId");
String tableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId);
try {
- client.tableOperations().create(tableName);
- // Add some splits to make the server's life easier
- client.tableOperations().addSplits(tableName, splits);
+ // Create table and add some splits to make the server's life easier
+ client.tableOperations().create(tableName, new NewTableConfiguration().withSplits(splits));
String tableId = client.tableOperations().tableIdMap().get(tableName);
- log.debug("created " + tableName + " (id:" + tableId + ")");
- // Add some splits to make the server's life easier
- client.tableOperations().addSplits(tableName, splits);
- log.debug("created " + splits.size() + " splits on " + tableName);
+ log.debug("created table {} (id:{}) with {} splits", tableName, tableId, splits.size());
+
@SuppressWarnings("unchecked")
List<String> tables = (List<String>) state.get("tableList");
tables.add(tableName);
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/ShardFixture.java b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/ShardFixture.java
index cdcd0e6..d096d6f 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/ShardFixture.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/ShardFixture.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.testing.randomwalk.shard;
import java.net.InetAddress;
+import java.util.Map;
import java.util.Random;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -24,6 +25,7 @@ import java.util.TreeSet;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.testing.randomwalk.Fixture;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
@@ -53,26 +55,24 @@ public class ShardFixture extends Fixture {
throws Exception {
AccumuloClient client = env.getAccumuloClient();
String name = state.get("indexTableName") + suffix;
- int numPartitions = (Integer) state.get("numPartitions");
- boolean enableCache = (Boolean) state.get("cacheIndex");
- client.tableOperations().create(name);
- String tableId = client.tableOperations().tableIdMap().get(name);
- log.info("Created index table " + name + "(id:" + tableId + ")");
+ NewTableConfiguration ntc = new NewTableConfiguration();
+ int numPartitions = state.getInteger("numPartitions");
SortedSet<Text> splits = genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
- client.tableOperations().addSplits(name, splits);
-
- log.info("Added " + splits.size() + " splits to " + name);
+ ntc.withSplits(splits);
- if (enableCache) {
- client.tableOperations().setProperty(name, Property.TABLE_INDEXCACHE_ENABLED.getKey(),
- "true");
- client.tableOperations().setProperty(name, Property.TABLE_BLOCKCACHE_ENABLED.getKey(),
- "true");
+ if ((Boolean) state.get("cacheIndex")) {
+ ntc.setProperties(Map.of(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true",
+ Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true"));
- log.info("Enabled caching for table " + name);
+ log.info("Enabling caching for table " + name);
}
+
+ client.tableOperations().create(name, ntc);
+
+ String tableId = client.tableOperations().tableIdMap().get(name);
+ log.info("Created index table {} (id:{}) with {} splits", name, tableId, splits.size());
}
@Override
@@ -97,22 +97,18 @@ public class ShardFixture extends Fixture {
createIndexTable(this.log, state, env, "", rand);
- String docTableName = (String) state.get("docTableName");
- client.tableOperations().create(docTableName);
-
- String tableId = client.tableOperations().tableIdMap().get(docTableName);
- log.info("Created doc table " + docTableName + " (id:" + tableId + ")");
-
+ String docTableName = state.getString("docTableName");
+ NewTableConfiguration ntc = new NewTableConfiguration();
SortedSet<Text> splits = genSplits(0xff, rand.nextInt(32) + 1, "%02x");
- client.tableOperations().addSplits(docTableName, splits);
-
- log.info("Added " + splits.size() + " splits to " + docTableName);
-
+ ntc.withSplits(splits);
if (rand.nextDouble() < .5) {
- client.tableOperations().setProperty((String) state.get("docTableName"),
- Property.TABLE_BLOOM_ENABLED.getKey(), "true");
- log.info("Enabled bloom filters for table " + (String) state.get("docTableName"));
+ ntc.setProperties(Map.of(Property.TABLE_BLOOM_ENABLED.getKey(), "true"));
+ log.info("Enabling bloom filters for table {}", docTableName);
}
+
+ client.tableOperations().create(docTableName, ntc);
+ String tableId = client.tableOperations().tableIdMap().get(docTableName);
+ log.info("Created doc table {} (id:{}) with {} splits", docTableName, tableId, splits.size());
}
@Override
diff --git a/src/main/java/org/apache/accumulo/testing/scalability/Ingest.java b/src/main/java/org/apache/accumulo/testing/scalability/Ingest.java
index aa5753e..5fe0359 100644
--- a/src/main/java/org/apache/accumulo/testing/scalability/Ingest.java
+++ b/src/main/java/org/apache/accumulo/testing/scalability/Ingest.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.testing.scalability;
import static java.nio.charset.StandardCharsets.UTF_8;
+import java.util.Map;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -27,6 +28,7 @@ import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.testing.continuous.ContinuousIngest;
@@ -54,10 +56,11 @@ public class Ingest extends ScaleTest {
}
// create table
+ NewTableConfiguration ntc = new NewTableConfiguration();
+ ntc.withSplits(calculateSplits());
+ ntc.setProperties(Map.of("table.split.threshold", "256M"));
try {
- client.tableOperations().create(tableName);
- client.tableOperations().addSplits(tableName, calculateSplits());
- client.tableOperations().setProperty(tableName, "table.split.threshold", "256M");
+ client.tableOperations().create(tableName, ntc);
} catch (Exception e) {
log.error("Failed to create table '" + tableName + "'.", e);
}