You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2018/02/07 14:59:54 UTC
[accumulo] branch master updated: ACCUMULO-4772 Update shell to use
NewTableConfiguration methods (#370)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 4c0d7de ACCUMULO-4772 Update shell to use NewTableConfiguration methods (#370)
4c0d7de is described below
commit 4c0d7dedf25eb1bb34bc5e1184978bbda00e86c3
Author: Mark Owens <jm...@gmail.com>
AuthorDate: Wed Feb 7 09:59:51 2018 -0500
ACCUMULO-4772 Update shell to use NewTableConfiguration methods (#370)
Update Accumulo shell to utilize new NewTableConfiguration methods.
Added capability to pre-configure iterators and locality groups when creating
new tables in Accumulo. Updated CreateTableCommand.java to accept locality group
and iterator data. Updated ShellServerIT.java to test the new capabilities.
---
.../shell/commands/CreateTableCommand.java | 126 +++++++++++++++-
.../org/apache/accumulo/test/ShellServerIT.java | 158 +++++++++++++++++++++
2 files changed, 281 insertions(+), 3 deletions(-)
diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
index eac16fa..5a66de0 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
@@ -17,7 +17,11 @@
package org.apache.accumulo.shell.commands;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -26,6 +30,7 @@ import java.util.TreeSet;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
@@ -34,6 +39,7 @@ import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.constraints.VisibilityConstraint;
import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.shell.Shell;
import org.apache.accumulo.shell.Shell.Command;
import org.apache.accumulo.shell.ShellUtil;
@@ -56,6 +62,8 @@ public class CreateTableCommand extends Command {
private Option base64Opt;
private Option createTableOptFormatter;
private Option createTableOptInitProp;
+ private Option createTableOptLocalityProps;
+ private Option createTableOptIteratorProps;
@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException,
@@ -63,6 +71,7 @@ public class CreateTableCommand extends Command {
final String testTableName = cl.getArgs()[0];
final HashMap<String,String> props = new HashMap<>();
+ NewTableConfiguration ntc = new NewTableConfiguration();
if (!testTableName.matches(Tables.VALID_NAME_REGEX)) {
shellState.getReader().println("Only letters, numbers and underscores are allowed for use in table names.");
@@ -106,8 +115,18 @@ public class CreateTableCommand extends Command {
}
}
+ // Set iterator if supplied
+ if (cl.hasOption(createTableOptIteratorProps.getOpt())) {
+ ntc = attachIteratorToNewTable(cl, shellState, ntc);
+ }
+
+ // Set up locality groups, if supplied
+ if (cl.hasOption(createTableOptLocalityProps.getOpt())) {
+ ntc = setLocalityForNewTable(cl, ntc);
+ }
+
// create table
- shellState.getConnector().tableOperations().create(tableName, new NewTableConfiguration().setTimeType(timeType).setProperties(props));
+ shellState.getConnector().tableOperations().create(tableName, ntc.setTimeType(timeType).setProperties(props));
if (partitions.size() > 0) {
shellState.getConnector().tableOperations().addSplits(tableName, partitions);
}
@@ -150,9 +169,100 @@ public class CreateTableCommand extends Command {
return 0;
}
+ /**
+ * Add supplied locality groups information to a NewTableConfiguration object.
+ *
+ * Used in conjunction with createtable shell command to allow locality groups to be configured upon table creation.
+ */
+ private NewTableConfiguration setLocalityForNewTable(CommandLine cl, NewTableConfiguration ntc) {
+ HashMap<String,Set<Text>> localityGroupMap = new HashMap<>();
+ String[] options = cl.getOptionValues(createTableOptLocalityProps.getOpt());
+ for (String localityInfo : options) {
+ final String parts[] = localityInfo.split("=", 2);
+ if (parts.length < 2)
+ throw new IllegalArgumentException("Missing '=' or there are spaces between entries");
+ final String groupName = parts[0];
+ final HashSet<Text> colFams = new HashSet<>();
+ for (String family : parts[1].split(","))
+ colFams.add(new Text(family.getBytes(Shell.CHARSET)));
+ // check that group names are not duplicated on usage line
+ if (localityGroupMap.put(groupName, colFams) != null)
+ throw new IllegalArgumentException("Duplicate locality group name found. Group names must be unique");
+ }
+ ntc.setLocalityGroups(localityGroupMap);
+ return ntc;
+ }
+
+ /**
+ * Add supplied iterator information to NewTableConfiguration object.
+ *
+ * Used in conjunction with createtable shell command to allow an iterator to be configured upon table creation.
+ */
+ private NewTableConfiguration attachIteratorToNewTable(final CommandLine cl, final Shell shellState, NewTableConfiguration ntc) {
+ EnumSet<IteratorScope> scopeEnumSet;
+ IteratorSetting iteratorSetting;
+ if (shellState.iteratorProfiles.size() == 0)
+ throw new IllegalArgumentException("No shell iterator profiles have been created.");
+ String[] options = cl.getOptionValues(createTableOptIteratorProps.getOpt());
+ for (String profileInfo : options) {
+ String[] parts = profileInfo.split(":", 2);
+ String profileName = parts[0];
+ // The iteratorProfiles.get calls below will throw an NPE if the profile does not exist
+ // This can occur when the profile actually does not exist or if there is
+ // extraneous spacing in the iterator profile argument list causing the parser to read a scope as a profile.
+ try {
+ iteratorSetting = shellState.iteratorProfiles.get(profileName).get(0);
+ } catch (NullPointerException ex) {
+ throw new IllegalArgumentException("invalid iterator argument. Either profile does not exist or unexpected spaces in argument list.", ex);
+ }
+ // handle case where only the profile is supplied. Use all scopes by default if no scope args are provided.
+ if (parts.length == 1) {
+ // add all scopes to enum set
+ scopeEnumSet = EnumSet.allOf(IteratorScope.class);
+ } else {
+ // user provided scope arguments exist, parse them
+ List<String> scopeArgs = Arrays.asList(parts[1].split(","));
+ // there are only three allowable scope values
+ if (scopeArgs.size() > 3)
+ throw new IllegalArgumentException("Too many scope arguments supplied");
+ // handle the 'all' argument separately since it is not an allowable enum value for IteratorScope
+ // if 'all' is used, it should be the only scope provided
+ if (scopeArgs.contains("all")) {
+ if (scopeArgs.size() > 1)
+ throw new IllegalArgumentException("Cannot use 'all' in conjunction with other scopes");
+ scopeEnumSet = EnumSet.allOf(IteratorScope.class);
+ } else {
+ // 'all' is not involved, examine the scope arguments and populate iterator scope EnumSet
+ scopeEnumSet = validateScopes(scopeArgs);
+ }
+ }
+ ntc.attachIterator(iteratorSetting, scopeEnumSet);
+ }
+ return ntc;
+ }
+
+ /**
+ * Validate that the provided scope arguments are valid iterator scope settings. Checking for duplicate entries and invalid scope values.
+ *
+ * Returns an EnumSet of scopes to be set.
+ */
+ private EnumSet<IteratorScope> validateScopes(final List<String> scopeList) {
+ EnumSet<IteratorScope> scopes = EnumSet.noneOf(IteratorScope.class);
+ for (String scopeStr : scopeList) {
+ try {
+ IteratorScope scope = IteratorScope.valueOf(scopeStr);
+ if (!scopes.add(scope))
+ throw new IllegalArgumentException("duplicate scope arguments found");
+ } catch (IllegalArgumentException ex) {
+ throw new IllegalArgumentException("illegal scope arguments: " + ex.getMessage(), ex);
+ }
+ }
+ return scopes;
+ }
+
@Override
public String description() {
- return "creates a new table, with optional aggregators and optionally pre-split";
+ return "creates a new table, with optional aggregators, iterators, locality groups and optionally pre-split";
}
@Override
@@ -174,13 +284,21 @@ public class CreateTableCommand extends Command {
"prevent users from writing data they cannot read. When enabling this, consider disabling bulk import and alter table.");
createTableOptFormatter = new Option("f", "formatter", true, "default formatter to set");
createTableOptInitProp = new Option("prop", "init-properties", true, "user defined initial properties");
-
createTableOptCopyConfig.setArgName("table");
createTableOptCopySplits.setArgName("table");
createTableOptSplit.setArgName("filename");
createTableOptFormatter.setArgName("className");
createTableOptInitProp.setArgName("properties");
+ createTableOptLocalityProps = new Option("l", "locality", true, "create locality groups at table creation");
+ createTableOptLocalityProps.setArgName("group=col_fam[,col_fam]");
+ createTableOptLocalityProps.setArgs(Option.UNLIMITED_VALUES);
+
+ createTableOptIteratorProps = new Option("i", "iter", true,
+ "initialize iterator at table creation using profile. If no scope supplied, all scopes are activated.");
+ createTableOptIteratorProps.setArgName("profile[:[all]|[scan[,]][minc[,]][majc]]");
+ createTableOptIteratorProps.setArgs(Option.UNLIMITED_VALUES);
+
// Splits and CopySplits are put in an optionsgroup to make them
// mutually exclusive
final OptionGroup splitOrCopySplit = new OptionGroup();
@@ -202,6 +320,8 @@ public class CreateTableCommand extends Command {
o.addOption(createTableOptEVC);
o.addOption(createTableOptFormatter);
o.addOption(createTableOptInitProp);
+ o.addOption(createTableOptLocalityProps);
+ o.addOption(createTableOptIteratorProps);
return o;
}
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index 0a91cc1..3e10909 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -34,11 +34,13 @@ import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.Constants;
@@ -2020,4 +2022,160 @@ public class ShellServerIT extends SharedMiniClusterBase {
// check that there are two files, with none having extra summary info
assertMatches(output, "(?sm).*^.*total[:]2[,]\\s+missing[:]0[,]\\s+extra[:]0.*$.*");
}
+
+ @Test
+ public void testCreateTableWithLocalityGroups() throws Exception {
+ final String table = name.getMethodName();
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2", true);
+ Connector connector = getConnector();
+ Map<String,Set<Text>> lMap = connector.tableOperations().getLocalityGroups(table);
+ Set<Text> expectedColFams = new HashSet<>(Arrays.asList(new Text("fam1"), new Text("fam2")));
+ for (Entry<String,Set<Text>> entry : lMap.entrySet()) {
+ Assert.assertTrue(entry.getKey().equals("locg1"));
+ Assert.assertTrue(entry.getValue().containsAll(expectedColFams));
+ }
+ ts.exec("deletetable -f " + table);
+ }
+
+ /**
+ * Due to the existing complexity of the createtable command, the createtable help only displays an example of setting one locality group. It is possible to
+ * set multiple groups if needed. This test verifies that capability.
+ */
+ @Test
+ public void testCreateTableWithMultipleLocalityGroups() throws Exception {
+ final String table = name.getMethodName();
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=colfam1", true);
+ Connector connector = getConnector();
+ Map<String,Set<Text>> lMap = connector.tableOperations().getLocalityGroups(table);
+ Assert.assertTrue(lMap.keySet().contains("locg1"));
+ Assert.assertTrue(lMap.keySet().contains("locg2"));
+ Set<Text> expectedColFams1 = new HashSet<>(Arrays.asList(new Text("fam1"), new Text("fam2")));
+ Set<Text> expectedColFams2 = new HashSet<>(Arrays.asList(new Text("colfam1")));
+ Assert.assertTrue(lMap.get("locg1").containsAll(expectedColFams1));
+ Assert.assertTrue(lMap.get("locg2").containsAll(expectedColFams2));
+ ts.exec("deletetable -f " + table);
+ }
+
+ @Test
+ public void testCreateTableWithLocalityGroupsBadArguments() throws IOException {
+ final String table = name.getMethodName();
+ ts.exec("createtable " + table + " -l locg1 fam1,fam2", false);
+ ts.exec("createtable " + table + "-l", false);
+ ts.exec("createtable " + table + " -l locg1 = fam1,fam2", false);
+ ts.exec("createtable " + table + " -l locg1=fam1 ,fam2", false);
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg1=fam3,fam4", false);
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=fam1", false);
+ ts.exec("createtable " + table + " -l locg1", false);
+ ts.exec("createtable " + table + " group=fam1", false);
+ ts.exec("createtable " + table + "-l fam1,fam2", false);
+ ts.exec("createtable " + table + " -local locg1=fam1,fam2", false);
+ }
+
+ @Test
+ public void testCreateTableWithIterators() throws Exception {
+ final String tmpTable = "tmpTable";
+ final String table = name.getMethodName();
+
+ // create iterator profile
+ // Will use tmpTable for creating profile since setshelliter is requiring a table
+ // even though its command line help indicates that it is optional. Likely due to
+ // the fact that setshelliter extends setiter, which does require a table argument.
+ ts.exec("createtable " + tmpTable, true);
+ String output = ts.exec("tables");
+ Assert.assertTrue(output.contains(tmpTable));
+
+ ts.input.set("\n5000\n\n");
+ ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+ output = ts.exec("listshelliter");
+ Assert.assertTrue(output.contains("Profile : profile1"));
+
+ // create table making use of the iterator profile
+ ts.exec("createtable " + table + " -i profile1:scan,minc", true);
+ ts.exec("insert foo a b c", true);
+ ts.exec("scan", true, "foo a:b [] c");
+ ts.exec("sleep 6", true);
+ ts.exec("scan", true, "", true);
+ ts.exec("deletetable -f " + table);
+ ts.exec("deletetable -f " + tmpTable);
+ }
+
+ /**
+ * Due to the existing complexity of the createtable command, the createtable help only displays an example of setting one iterator upon table creation. It is
+ * possible to set multiple if needed. This test verifies that capability.
+ */
+ @Test
+ public void testCreateTableWithMultipleIterators() throws Exception {
+ final String tmpTable = "tmpTable";
+ final String table = name.getMethodName();
+
+ // create iterator profile
+ // Will use tmpTable for creating profile since setshelliter is requiring a table
+ // even though its command line help indicates that it is optional. Likely due to
+ // the fact that setshelliter extends setiter, which does require a table argument.
+ ts.exec("createtable " + tmpTable, true);
+ String output = ts.exec("tables");
+ Assert.assertTrue(output.contains(tmpTable));
+
+ ts.input.set("\n5000\n\n");
+ ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+ output = ts.exec("listshelliter");
+ Assert.assertTrue(output.contains("Profile : profile1"));
+
+ ts.input.set("2\n");
+ ts.exec("setshelliter -n iter2 -p 11 -pn profile2 -vers -t " + tmpTable, true);
+ output = ts.exec("listshelliter");
+ Assert.assertTrue(output.contains("Profile : profile2"));
+
+ // create table making use of the iterator profiles
+ ts.exec("createtable " + table + " -i profile1:scan,minc profile2:all ", true);
+ ts.exec("insert foo a b c", true);
+ ts.exec("scan", true, "foo a:b [] c");
+ ts.exec("sleep 6", true);
+ ts.exec("scan", true, "", true);
+ output = ts.exec("listiter -t " + table + " -all");
+ Assert.assertTrue(output.contains("Iterator itname, scan scope options"));
+ Assert.assertTrue(output.contains("Iterator itname, minc scope options"));
+ Assert.assertFalse(output.contains("Iterator itname, majc scope options"));
+ Assert.assertTrue(output.contains("Iterator iter2, scan scope options"));
+ Assert.assertTrue(output.contains("Iterator iter2, minc scope options"));
+ Assert.assertTrue(output.contains("Iterator iter2, majc scope options"));
+ ts.exec("deletetable -f " + table);
+ ts.exec("deletetable -f " + tmpTable);
+ }
+
+ @Test
+ public void testCreateTableWithIteratorsBadArguments() throws IOException {
+ final String tmpTable = "tmpTable";
+ final String table = name.getMethodName();
+ ts.exec("createtable " + tmpTable, true);
+ String output = ts.exec("tables");
+ Assert.assertTrue(output.contains(tmpTable));
+ ts.input.set("\n5000\n\n");
+ ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+ output = ts.exec("listshelliter");
+ Assert.assertTrue(output.contains("Profile : profile1"));
+ // test various bad argument calls
+ ts.exec("createtable " + table + " -i noprofile:scan,minc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,minc,all,majc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,all,majc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,min,majc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,max,all", false);
+ ts.exec("createtable " + table + " -i profile1:", false);
+ ts.exec("createtable " + table + " -i profile1: ", false);
+ ts.exec("createtable " + table + " -i profile1:-scan", false);
+ ts.exec("createtable " + table + " profile1:majc", false);
+ ts.exec("createtable " + table + " -i profile1: all", false);
+ ts.exec("createtable " + table + " -i profile1: All", false);
+ ts.exec("createtable " + table + " -i profile1: scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc,Scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc, scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc,,scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc,minc", false);
+ ts.exec("createtable " + table + " -i profile1:minc,Minc", false);
+ ts.exec("createtable " + table + " -i profile1:minc, ,scan", false);
+ ts.exec("createtable " + table + "-i", false);
+ ts.exec("createtable " + table + "-i ", false);
+ ts.exec("deletetable -f " + tmpTable);
+ }
}
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.