You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2018/02/07 14:59:56 UTC

[GitHub] keith-turner closed pull request #370: ACCUMULO-4772 Update shell to use NewTableConfiguration methods

keith-turner closed pull request #370: ACCUMULO-4772 Update shell to use NewTableConfiguration methods
URL: https://github.com/apache/accumulo/pull/370
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
index eac16fa73d..5a66de0c6f 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
@@ -17,7 +17,11 @@
 package org.apache.accumulo.shell.commands;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -26,6 +30,7 @@
 
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.admin.NewTableConfiguration;
@@ -34,6 +39,7 @@
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.constraints.VisibilityConstraint;
 import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.shell.Shell;
 import org.apache.accumulo.shell.Shell.Command;
 import org.apache.accumulo.shell.ShellUtil;
@@ -56,6 +62,8 @@
   private Option base64Opt;
   private Option createTableOptFormatter;
   private Option createTableOptInitProp;
+  private Option createTableOptLocalityProps;
+  private Option createTableOptIteratorProps;
 
   @Override
   public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException,
@@ -63,6 +71,7 @@ public int execute(final String fullCommand, final CommandLine cl, final Shell s
 
     final String testTableName = cl.getArgs()[0];
     final HashMap<String,String> props = new HashMap<>();
+    NewTableConfiguration ntc = new NewTableConfiguration();
 
     if (!testTableName.matches(Tables.VALID_NAME_REGEX)) {
       shellState.getReader().println("Only letters, numbers and underscores are allowed for use in table names.");
@@ -106,8 +115,18 @@ public int execute(final String fullCommand, final CommandLine cl, final Shell s
       }
     }
 
+    // Set iterator if supplied
+    if (cl.hasOption(createTableOptIteratorProps.getOpt())) {
+      ntc = attachIteratorToNewTable(cl, shellState, ntc);
+    }
+
+    // Set up locality groups, if supplied
+    if (cl.hasOption(createTableOptLocalityProps.getOpt())) {
+      ntc = setLocalityForNewTable(cl, ntc);
+    }
+
     // create table
-    shellState.getConnector().tableOperations().create(tableName, new NewTableConfiguration().setTimeType(timeType).setProperties(props));
+    shellState.getConnector().tableOperations().create(tableName, ntc.setTimeType(timeType).setProperties(props));
     if (partitions.size() > 0) {
       shellState.getConnector().tableOperations().addSplits(tableName, partitions);
     }
@@ -150,9 +169,100 @@ public int execute(final String fullCommand, final CommandLine cl, final Shell s
     return 0;
   }
 
+  /**
+   * Add supplied locality groups information to a NewTableConfiguration object.
+   *
+   * Used in conjunction with createtable shell command to allow locality groups to be configured upon table creation.
+   */
+  private NewTableConfiguration setLocalityForNewTable(CommandLine cl, NewTableConfiguration ntc) {
+    HashMap<String,Set<Text>> localityGroupMap = new HashMap<>();
+    String[] options = cl.getOptionValues(createTableOptLocalityProps.getOpt());
+    for (String localityInfo : options) {
+      final String parts[] = localityInfo.split("=", 2);
+      if (parts.length < 2)
+        throw new IllegalArgumentException("Missing '=' or there are spaces between entries");
+      final String groupName = parts[0];
+      final HashSet<Text> colFams = new HashSet<>();
+      for (String family : parts[1].split(","))
+        colFams.add(new Text(family.getBytes(Shell.CHARSET)));
+      // check that group names are not duplicated on usage line
+      if (localityGroupMap.put(groupName, colFams) != null)
+        throw new IllegalArgumentException("Duplicate locality group name found. Group names must be unique");
+    }
+    ntc.setLocalityGroups(localityGroupMap);
+    return ntc;
+  }
+
+  /**
+   * Add supplied iterator information to NewTableConfiguration object.
+   *
+   * Used in conjunction with createtable shell command to allow an iterator to be configured upon table creation.
+   */
+  private NewTableConfiguration attachIteratorToNewTable(final CommandLine cl, final Shell shellState, NewTableConfiguration ntc) {
+    EnumSet<IteratorScope> scopeEnumSet;
+    IteratorSetting iteratorSetting;
+    if (shellState.iteratorProfiles.size() == 0)
+      throw new IllegalArgumentException("No shell iterator profiles have been created.");
+    String[] options = cl.getOptionValues(createTableOptIteratorProps.getOpt());
+    for (String profileInfo : options) {
+      String[] parts = profileInfo.split(":", 2);
+      String profileName = parts[0];
+      // The iteratorProfiles.get calls below will throw an NPE if the profile does not exist
+      // This can occur when the profile actually does not exist or if there is
+      // extraneous spacing in the iterator profile argument list causing the parser to read a scope as a profile.
+      try {
+        iteratorSetting = shellState.iteratorProfiles.get(profileName).get(0);
+      } catch (NullPointerException ex) {
+        throw new IllegalArgumentException("invalid iterator argument. Either profile does not exist or unexpected spaces in argument list.", ex);
+      }
+      // handle case where only the profile is supplied. Use all scopes by default if no scope args are provided.
+      if (parts.length == 1) {
+        // add all scopes to enum set
+        scopeEnumSet = EnumSet.allOf(IteratorScope.class);
+      } else {
+        // user provided scope arguments exist, parse them
+        List<String> scopeArgs = Arrays.asList(parts[1].split(","));
+        // there are only three allowable scope values
+        if (scopeArgs.size() > 3)
+          throw new IllegalArgumentException("Too many scope arguments supplied");
+        // handle the 'all' argument separately since it is not an allowable enum value for IteratorScope
+        // if 'all' is used, it should be the only scope provided
+        if (scopeArgs.contains("all")) {
+          if (scopeArgs.size() > 1)
+            throw new IllegalArgumentException("Cannot use 'all' in conjunction with other scopes");
+          scopeEnumSet = EnumSet.allOf(IteratorScope.class);
+        } else {
+          // 'all' is not involved, examine the scope arguments and populate iterator scope EnumSet
+          scopeEnumSet = validateScopes(scopeArgs);
+        }
+      }
+      ntc.attachIterator(iteratorSetting, scopeEnumSet);
+    }
+    return ntc;
+  }
+
+  /**
+   * Validate that the provided scope arguments are valid iterator scope settings. Checking for duplicate entries and invalid scope values.
+   *
+   * Returns an EnumSet of scopes to be set.
+   */
+  private EnumSet<IteratorScope> validateScopes(final List<String> scopeList) {
+    EnumSet<IteratorScope> scopes = EnumSet.noneOf(IteratorScope.class);
+    for (String scopeStr : scopeList) {
+      try {
+        IteratorScope scope = IteratorScope.valueOf(scopeStr);
+        if (!scopes.add(scope))
+          throw new IllegalArgumentException("duplicate scope arguments found");
+      } catch (IllegalArgumentException ex) {
+        throw new IllegalArgumentException("illegal scope arguments: " + ex.getMessage(), ex);
+      }
+    }
+    return scopes;
+  }
+
   @Override
   public String description() {
-    return "creates a new table, with optional aggregators and optionally pre-split";
+    return "creates a new table, with optional aggregators, iterators, locality groups and optionally pre-split";
   }
 
   @Override
@@ -174,13 +284,21 @@ public Options getOptions() {
         "prevent users from writing data they cannot read.  When enabling this, consider disabling bulk import and alter table.");
     createTableOptFormatter = new Option("f", "formatter", true, "default formatter to set");
     createTableOptInitProp = new Option("prop", "init-properties", true, "user defined initial properties");
-
     createTableOptCopyConfig.setArgName("table");
     createTableOptCopySplits.setArgName("table");
     createTableOptSplit.setArgName("filename");
     createTableOptFormatter.setArgName("className");
     createTableOptInitProp.setArgName("properties");
 
+    createTableOptLocalityProps = new Option("l", "locality", true, "create locality groups at table creation");
+    createTableOptLocalityProps.setArgName("group=col_fam[,col_fam]");
+    createTableOptLocalityProps.setArgs(Option.UNLIMITED_VALUES);
+
+    createTableOptIteratorProps = new Option("i", "iter", true,
+        "initialize iterator at table creation using profile. If no scope supplied, all scopes are activated.");
+    createTableOptIteratorProps.setArgName("profile[:[all]|[scan[,]][minc[,]][majc]]");
+    createTableOptIteratorProps.setArgs(Option.UNLIMITED_VALUES);
+
     // Splits and CopySplits are put in an optionsgroup to make them
     // mutually exclusive
     final OptionGroup splitOrCopySplit = new OptionGroup();
@@ -202,6 +320,8 @@ public Options getOptions() {
     o.addOption(createTableOptEVC);
     o.addOption(createTableOptFormatter);
     o.addOption(createTableOptInitProp);
+    o.addOption(createTableOptLocalityProps);
+    o.addOption(createTableOptIteratorProps);
 
     return o;
   }
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index 0a91cc11a7..3e10909d41 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -34,11 +34,13 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.Constants;
@@ -2020,4 +2022,160 @@ public void testSummarySelection() throws Exception {
     // check that there are two files, with none having extra summary info
     assertMatches(output, "(?sm).*^.*total[:]2[,]\\s+missing[:]0[,]\\s+extra[:]0.*$.*");
   }
+
+  @Test
+  public void testCreateTableWithLocalityGroups() throws Exception {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2", true);
+    Connector connector = getConnector();
+    Map<String,Set<Text>> lMap = connector.tableOperations().getLocalityGroups(table);
+    Set<Text> expectedColFams = new HashSet<>(Arrays.asList(new Text("fam1"), new Text("fam2")));
+    for (Entry<String,Set<Text>> entry : lMap.entrySet()) {
+      Assert.assertTrue(entry.getKey().equals("locg1"));
+      Assert.assertTrue(entry.getValue().containsAll(expectedColFams));
+    }
+    ts.exec("deletetable -f " + table);
+  }
+
+  /**
+   * Due to the existing complexity of the createtable command, the createtable help only displays an example of setting one locality group. It is possible to
+   * set multiple groups if needed. This test verifies that capability.
+   */
+  @Test
+  public void testCreateTableWithMultipleLocalityGroups() throws Exception {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=colfam1", true);
+    Connector connector = getConnector();
+    Map<String,Set<Text>> lMap = connector.tableOperations().getLocalityGroups(table);
+    Assert.assertTrue(lMap.keySet().contains("locg1"));
+    Assert.assertTrue(lMap.keySet().contains("locg2"));
+    Set<Text> expectedColFams1 = new HashSet<>(Arrays.asList(new Text("fam1"), new Text("fam2")));
+    Set<Text> expectedColFams2 = new HashSet<>(Arrays.asList(new Text("colfam1")));
+    Assert.assertTrue(lMap.get("locg1").containsAll(expectedColFams1));
+    Assert.assertTrue(lMap.get("locg2").containsAll(expectedColFams2));
+    ts.exec("deletetable -f " + table);
+  }
+
+  @Test
+  public void testCreateTableWithLocalityGroupsBadArguments() throws IOException {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table + " -l locg1 fam1,fam2", false);
+    ts.exec("createtable " + table + "-l", false);
+    ts.exec("createtable " + table + " -l locg1 = fam1,fam2", false);
+    ts.exec("createtable " + table + " -l locg1=fam1 ,fam2", false);
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg1=fam3,fam4", false);
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=fam1", false);
+    ts.exec("createtable " + table + " -l locg1", false);
+    ts.exec("createtable " + table + " group=fam1", false);
+    ts.exec("createtable " + table + "-l fam1,fam2", false);
+    ts.exec("createtable " + table + " -local locg1=fam1,fam2", false);
+  }
+
+  @Test
+  public void testCreateTableWithIterators() throws Exception {
+    final String tmpTable = "tmpTable";
+    final String table = name.getMethodName();
+
+    // create iterator profile
+    // Will use tmpTable for creating profile since setshelliter is requiring a table
+    // even though its command line help indicates that it is optional. Likely due to
+    // the fact that setshelliter extends setiter, which does require a table argument.
+    ts.exec("createtable " + tmpTable, true);
+    String output = ts.exec("tables");
+    Assert.assertTrue(output.contains(tmpTable));
+
+    ts.input.set("\n5000\n\n");
+    ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile1"));
+
+    // create table making use of the iterator profile
+    ts.exec("createtable " + table + " -i profile1:scan,minc", true);
+    ts.exec("insert foo a b c", true);
+    ts.exec("scan", true, "foo a:b []    c");
+    ts.exec("sleep 6", true);
+    ts.exec("scan", true, "", true);
+    ts.exec("deletetable -f " + table);
+    ts.exec("deletetable -f " + tmpTable);
+  }
+
+  /**
+   * Due to the existing complexity of the createtable command, the createtable help only displays an example of setting one iterator upon table creation. It is
+   * possible to set multiple if needed. This test verifies that capability.
+   */
+  @Test
+  public void testCreateTableWithMultipleIterators() throws Exception {
+    final String tmpTable = "tmpTable";
+    final String table = name.getMethodName();
+
+    // create iterator profile
+    // Will use tmpTable for creating profile since setshelliter is requiring a table
+    // even though its command line help indicates that it is optional. Likely due to
+    // the fact that setshelliter extends setiter, which does require a table argument.
+    ts.exec("createtable " + tmpTable, true);
+    String output = ts.exec("tables");
+    Assert.assertTrue(output.contains(tmpTable));
+
+    ts.input.set("\n5000\n\n");
+    ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile1"));
+
+    ts.input.set("2\n");
+    ts.exec("setshelliter -n iter2 -p 11 -pn profile2 -vers -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile2"));
+
+    // create table making use of the iterator profiles
+    ts.exec("createtable " + table + " -i profile1:scan,minc profile2:all ", true);
+    ts.exec("insert foo a b c", true);
+    ts.exec("scan", true, "foo a:b []    c");
+    ts.exec("sleep 6", true);
+    ts.exec("scan", true, "", true);
+    output = ts.exec("listiter -t " + table + " -all");
+    Assert.assertTrue(output.contains("Iterator itname, scan scope options"));
+    Assert.assertTrue(output.contains("Iterator itname, minc scope options"));
+    Assert.assertFalse(output.contains("Iterator itname, majc scope options"));
+    Assert.assertTrue(output.contains("Iterator iter2, scan scope options"));
+    Assert.assertTrue(output.contains("Iterator iter2, minc scope options"));
+    Assert.assertTrue(output.contains("Iterator iter2, majc scope options"));
+    ts.exec("deletetable -f " + table);
+    ts.exec("deletetable -f " + tmpTable);
+  }
+
+  @Test
+  public void testCreateTableWithIteratorsBadArguments() throws IOException {
+    final String tmpTable = "tmpTable";
+    final String table = name.getMethodName();
+    ts.exec("createtable " + tmpTable, true);
+    String output = ts.exec("tables");
+    Assert.assertTrue(output.contains(tmpTable));
+    ts.input.set("\n5000\n\n");
+    ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile1"));
+    // test various bad argument calls
+    ts.exec("createtable " + table + " -i noprofile:scan,minc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,minc,all,majc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,all,majc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,min,majc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,max,all", false);
+    ts.exec("createtable " + table + " -i profile1:", false);
+    ts.exec("createtable " + table + " -i profile1: ", false);
+    ts.exec("createtable " + table + " -i profile1:-scan", false);
+    ts.exec("createtable " + table + " profile1:majc", false);
+    ts.exec("createtable " + table + " -i profile1: all", false);
+    ts.exec("createtable " + table + " -i profile1: All", false);
+    ts.exec("createtable " + table + " -i profile1: scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc,Scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc, scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc,,scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc,minc", false);
+    ts.exec("createtable " + table + " -i profile1:minc,Minc", false);
+    ts.exec("createtable " + table + " -i profile1:minc, ,scan", false);
+    ts.exec("createtable " + table + "-i", false);
+    ts.exec("createtable " + table + "-i ", false);
+    ts.exec("deletetable -f " + tmpTable);
+  }
 }


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services