You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2015/01/09 03:45:10 UTC
[66/66] [abbrv] accumulo git commit: Merge branch '1.6'
Merge branch '1.6'
Conflicts:
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/392d9d6a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/392d9d6a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/392d9d6a
Branch: refs/heads/master
Commit: 392d9d6a48b2ddf3c8f9b85f91e8e983f0dfa620
Parents: 83a9623 9ca1ff0
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Jan 8 21:41:48 2015 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Jan 8 21:41:48 2015 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/core/cli/ClientOpts.java | 6 +++---
.../core/client/admin/InstanceOperations.java | 6 +++---
.../apache/accumulo/core/file/rfile/CreateEmpty.java | 4 ++--
.../accumulo/core/file/rfile/bcfile/Utils.java | 1 +
.../org/apache/accumulo/core/iterators/Combiner.java | 2 +-
.../apache/accumulo/core/iterators/LongCombiner.java | 4 ++--
.../accumulo/core/iterators/OptionDescriber.java | 4 ++--
.../accumulo/core/iterators/TypedValueCombiner.java | 10 +++++-----
.../core/iterators/user/SummingArrayCombiner.java | 5 +++--
.../core/security/crypto/CryptoModuleParameters.java | 2 +-
.../NonCachingSecretKeyEncryptionStrategy.java | 3 ++-
.../examples/simple/mapreduce/TableToFile.java | 4 ++--
pom.xml | 2 +-
.../java/org/apache/accumulo/server/Accumulo.java | 4 ++--
.../org/apache/accumulo/server/init/Initialize.java | 3 ++-
.../accumulo/server/rpc/CustomNonBlockingServer.java | 4 ++--
.../accumulo/tserver/tablet/CompactionInfo.java | 5 +++--
.../org/apache/accumulo/shell/ShellOptionsJC.java | 6 +++---
.../apache/accumulo/shell/commands/DUCommand.java | 3 ++-
.../apache/accumulo/shell/commands/EGrepCommand.java | 3 ++-
.../shell/commands/QuotedStringTokenizer.java | 15 ++++++---------
.../main/java/org/apache/accumulo/start/Main.java | 4 ++--
.../start/classloader/AccumuloClassLoader.java | 4 ++--
.../test/randomwalk/security/CreateTable.java | 5 ++---
.../test/randomwalk/security/CreateUser.java | 5 ++---
25 files changed, 58 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/server/base/src/main/java/org/apache/accumulo/server/rpc/CustomNonBlockingServer.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/rpc/CustomNonBlockingServer.java
index 21f55b3,0000000..577b5eb
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/rpc/CustomNonBlockingServer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/CustomNonBlockingServer.java
@@@ -1,268 -1,0 +1,268 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.rpc;
+
+import java.io.IOException;
+import java.net.Socket;
+import java.nio.channels.SelectionKey;
+import java.util.Iterator;
+
+import org.apache.log4j.Logger;
+import org.apache.thrift.server.THsHaServer;
+import org.apache.thrift.server.TNonblockingServer;
+import org.apache.thrift.transport.TNonblockingServerTransport;
+import org.apache.thrift.transport.TNonblockingSocket;
+import org.apache.thrift.transport.TNonblockingTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * This class implements a custom non-blocking thrift server, incorporating the {@link THsHaServer} features, and overriding the underlying
+ * {@link TNonblockingServer} methods, especially {@link org.apache.thrift.server.TNonblockingServer.SelectAcceptThread}, in order to override the
+ * {@link org.apache.thrift.server.AbstractNonblockingServer.FrameBuffer} and {@link org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer} with
+ * one that reveals the client address from its transport.
+ *
+ * <p>
+ * The justification for this is explained in https://issues.apache.org/jira/browse/ACCUMULO-1691, and is needed due to the repeated regressions:
+ * <ul>
+ * <li>https://issues.apache.org/jira/browse/THRIFT-958</li>
+ * <li>https://issues.apache.org/jira/browse/THRIFT-1464</li>
+ * <li>https://issues.apache.org/jira/browse/THRIFT-2173</li>
+ * </ul>
+ *
+ * <p>
+ * This class contains a copy of {@link org.apache.thrift.server.TNonblockingServer.SelectAcceptThread} from Thrift 0.9.1, with the slight modification of
+ * instantiating a custom FrameBuffer, rather than the {@link org.apache.thrift.server.AbstractNonblockingServer.FrameBuffer} and
+ * {@link org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer}. Because of this, any change in the implementation upstream will require a review
+ * of this implementation here, to ensure any new bugfixes/features in the upstream Thrift class are also applied here, at least until
+ * https://issues.apache.org/jira/browse/THRIFT-2173 is implemented. In the meantime, the maven-enforcer-plugin ensures that Thrift remains at version 0.9.1,
+ * which has been reviewed and tested.
+ */
+public class CustomNonBlockingServer extends THsHaServer {
+
+ private static final Logger LOGGER = Logger.getLogger(CustomNonBlockingServer.class);
+ private SelectAcceptThread selectAcceptThread_;
+ private volatile boolean stopped_ = false;
+
+ public CustomNonBlockingServer(Args args) {
+ super(args);
+ }
+
+ @Override
+ protected Runnable getRunnable(final FrameBuffer frameBuffer) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ if (frameBuffer instanceof CustomNonblockingFrameBuffer) {
+ TNonblockingTransport trans = ((CustomNonblockingFrameBuffer) frameBuffer).getTransport();
+ if (trans instanceof TNonblockingSocket) {
+ TNonblockingSocket tsock = (TNonblockingSocket) trans;
+ Socket sock = tsock.getSocketChannel().socket();
+ TServerUtils.clientAddress.set(sock.getInetAddress().getHostAddress() + ":" + sock.getPort());
+ }
+ }
+ frameBuffer.invoke();
+ }
+ };
+ }
+
+ @Override
+ protected boolean startThreads() {
+ // start the selector
+ try {
+ selectAcceptThread_ = new SelectAcceptThread((TNonblockingServerTransport) serverTransport_);
+ selectAcceptThread_.start();
+ return true;
+ } catch (IOException e) {
+ LOGGER.error("Failed to start selector thread!", e);
+ return false;
+ }
+ }
+
+ @Override
+ public void stop() {
+ stopped_ = true;
+ if (selectAcceptThread_ != null) {
+ selectAcceptThread_.wakeupSelector();
+ }
+ }
+
+ @Override
+ public boolean isStopped() {
+ return selectAcceptThread_.isStopped();
+ }
+
+ @Override
+ protected void joinSelector() {
+ // wait until the selector thread exits
+ try {
+ selectAcceptThread_.join();
+ } catch (InterruptedException e) {
+ // for now, just silently ignore. technically this means we'll have less of
+ // a graceful shutdown as a result.
+ }
+ }
+
+ private interface CustomNonblockingFrameBuffer {
+ TNonblockingTransport getTransport();
+ }
+
+ private class CustomAsyncFrameBuffer extends AsyncFrameBuffer implements CustomNonblockingFrameBuffer {
+ private TNonblockingTransport trans;
+
+ public CustomAsyncFrameBuffer(TNonblockingTransport trans, SelectionKey selectionKey, AbstractSelectThread selectThread) {
+ super(trans, selectionKey, selectThread);
+ this.trans = trans;
+ }
+
+ @Override
+ public TNonblockingTransport getTransport() {
+ return trans;
+ }
+ }
+
+ private class CustomFrameBuffer extends FrameBuffer implements CustomNonblockingFrameBuffer {
+ private TNonblockingTransport trans;
+
+ public CustomFrameBuffer(TNonblockingTransport trans, SelectionKey selectionKey, AbstractSelectThread selectThread) {
+ super(trans, selectionKey, selectThread);
+ this.trans = trans;
+ }
+
+ @Override
+ public TNonblockingTransport getTransport() {
+ return trans;
+ }
+ }
+
+ // @formatter:off
+ private class SelectAcceptThread extends AbstractSelectThread {
+
+ // The server transport on which new client transports will be accepted
+ private final TNonblockingServerTransport serverTransport;
+
+ /**
+ * Set up the thread that will handle the non-blocking accepts, reads, and
+ * writes.
+ */
+ public SelectAcceptThread(final TNonblockingServerTransport serverTransport)
+ throws IOException {
+ this.serverTransport = serverTransport;
+ serverTransport.registerSelector(selector);
+ }
+
+ public boolean isStopped() {
+ return stopped_;
+ }
+
+ /**
+ * The work loop. Handles both selecting (all IO operations) and managing
+ * the selection preferences of all existing connections.
+ */
+ @Override
+ public void run() {
+ try {
+ if (eventHandler_ != null) {
+ eventHandler_.preServe();
+ }
+
+ while (!stopped_) {
+ select();
+ processInterestChanges();
+ }
+ for (SelectionKey selectionKey : selector.keys()) {
+ cleanupSelectionKey(selectionKey);
+ }
+ } catch (Throwable t) {
+ LOGGER.error("run() exiting due to uncaught error", t);
+ } finally {
+ stopped_ = true;
+ }
+ }
+
+ /**
+ * Select and process IO events appropriately:
+ * If there are connections to be accepted, accept them.
+ * If there are existing connections with data waiting to be read, read it,
+ * buffering until a whole frame has been read.
+ * If there are any pending responses, buffer them until their target client
+ * is available, and then send the data.
+ */
+ private void select() {
+ try {
+ // wait for io events.
+ selector.select();
+
+ // process the io events we received
+ Iterator<SelectionKey> selectedKeys = selector.selectedKeys().iterator();
+ while (!stopped_ && selectedKeys.hasNext()) {
+ SelectionKey key = selectedKeys.next();
+ selectedKeys.remove();
+
+ // skip if not valid
+ if (!key.isValid()) {
+ cleanupSelectionKey(key);
+ continue;
+ }
+
+ // if the key is marked Accept, then it has to be the server
+ // transport.
+ if (key.isAcceptable()) {
+ handleAccept();
+ } else if (key.isReadable()) {
+ // deal with reads
+ handleRead(key);
+ } else if (key.isWritable()) {
+ // deal with writes
+ handleWrite(key);
+ } else {
+ LOGGER.warn("Unexpected state in select! " + key.interestOps());
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Got an IOException while selecting!", e);
+ }
+ }
+
+ /**
+ * Accept a new connection.
+ */
+ @SuppressWarnings("unused")
+ private void handleAccept() throws IOException {
+ SelectionKey clientKey = null;
+ TNonblockingTransport client = null;
+ try {
+ // accept the connection
+ client = (TNonblockingTransport)serverTransport.accept();
+ clientKey = client.registerSelector(selector, SelectionKey.OP_READ);
+
+ // add this key to the map
- FrameBuffer frameBuffer = processorFactory_.isAsyncProcessor() ?
- new CustomAsyncFrameBuffer(client, clientKey,SelectAcceptThread.this) :
++ FrameBuffer frameBuffer =
++ processorFactory_.isAsyncProcessor() ? new CustomAsyncFrameBuffer(client, clientKey,SelectAcceptThread.this) :
+ new CustomFrameBuffer(client, clientKey,SelectAcceptThread.this);
+
+ clientKey.attach(frameBuffer);
+ } catch (TTransportException tte) {
+ // something went wrong accepting.
+ LOGGER.warn("Exception trying to accept!", tte);
+ tte.printStackTrace();
+ if (clientKey != null) cleanupSelectionKey(clientKey);
+ if (client != null) client.close();
+ }
+ }
+ } // SelectAcceptThread
+ // @formatter:on
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
index 918edf6,0000000..2023d2c
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
@@@ -1,129 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.CompactionReason;
+import org.apache.accumulo.core.tabletserver.thrift.CompactionType;
+import org.apache.accumulo.server.fs.FileRef;
+
+public class CompactionInfo {
+
+ private final Compactor compactor;
+ private final String localityGroup;
+ private final long entriesRead;
+ private final long entriesWritten;
+
+ CompactionInfo(Compactor compactor) {
+ this.localityGroup = compactor.getCurrentLocalityGroup();
+ this.entriesRead = compactor.getEntriesRead();
+ this.entriesWritten = compactor.getEntriesWritten();
+ this.compactor = compactor;
+ }
+
+ public long getID() {
+ return compactor.getCompactorID();
+ }
+
+ public KeyExtent getExtent() {
+ return compactor.getExtent();
+ }
+
+ public long getEntriesRead() {
+ return entriesRead;
+ }
+
+ public long getEntriesWritten() {
+ return entriesWritten;
+ }
+
+ public Thread getThread() {
+ return compactor.thread;
+ }
+
+ public String getOutputFile() {
+ return compactor.getOutputFile();
+ }
+
+ public ActiveCompaction toThrift() {
+
+ CompactionType type;
+
+ if (compactor.hasIMM())
+ if (compactor.getFilesToCompact().size() > 0)
+ type = CompactionType.MERGE;
+ else
+ type = CompactionType.MINOR;
+ else if (!compactor.willPropogateDeletes())
+ type = CompactionType.FULL;
+ else
+ type = CompactionType.MAJOR;
+
+ CompactionReason reason;
+
- if (compactor.hasIMM())
++ if (compactor.hasIMM()) {
+ switch (compactor.getMinCReason()) {
+ case USER:
+ reason = CompactionReason.USER;
+ break;
+ case CLOSE:
+ reason = CompactionReason.CLOSE;
+ break;
+ case SYSTEM:
+ default:
+ reason = CompactionReason.SYSTEM;
+ break;
+ }
- else
++ } else {
+ switch (compactor.getMajorCompactionReason()) {
+ case USER:
+ reason = CompactionReason.USER;
+ break;
+ case CHOP:
+ reason = CompactionReason.CHOP;
+ break;
+ case IDLE:
+ reason = CompactionReason.IDLE;
+ break;
+ case NORMAL:
+ default:
+ reason = CompactionReason.SYSTEM;
+ break;
+ }
++ }
+
+ List<IterInfo> iiList = new ArrayList<IterInfo>();
+ Map<String,Map<String,String>> iterOptions = new HashMap<String,Map<String,String>>();
+
+ for (IteratorSetting iterSetting : compactor.getIterators()) {
+ iiList.add(new IterInfo(iterSetting.getPriority(), iterSetting.getIteratorClass(), iterSetting.getName()));
+ iterOptions.put(iterSetting.getName(), iterSetting.getOptions());
+ }
+ List<String> filesToCompact = new ArrayList<String>();
+ for (FileRef ref : compactor.getFilesToCompact())
+ filesToCompact.add(ref.toString());
+ return new ActiveCompaction(compactor.extent.toThrift(), System.currentTimeMillis() - compactor.getStartTime(), filesToCompact, compactor.getOutputFile(),
+ type, reason, localityGroup, entriesRead, entriesWritten, iiList, iterOptions);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java
index 67c8c40,0000000..875367d
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java
@@@ -1,281 -1,0 +1,281 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.DynamicParameter;
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import com.beust.jcommander.converters.FileConverter;
+
+public class ShellOptionsJC {
+ private static final Logger log = LoggerFactory.getLogger(Shell.class);
+
+ @Parameter(names = {"-u", "--user"}, description = "username (defaults to your OS user)")
+ private String username = System.getProperty("user.name", "root");
+
+ public static class PasswordConverter implements IStringConverter<String> {
+ public static final String STDIN = "stdin";
+
+ private enum KeyType {
+ PASS("pass:"), ENV("env:") {
+ @Override
+ String process(String value) {
+ return System.getenv(value);
+ }
+ },
+ FILE("file:") {
+ @Override
+ String process(String value) {
+ Scanner scanner = null;
+ try {
+ scanner = new Scanner(new File(value));
+ return scanner.nextLine();
+ } catch (FileNotFoundException e) {
+ throw new ParameterException(e);
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ }
+ }
+ },
+ STDIN(PasswordConverter.STDIN) {
+ @Override
+ public boolean matches(String value) {
+ return prefix.equals(value);
+ }
+
+ @Override
+ public String convert(String value) {
+ // Will check for this later
+ return prefix;
+ }
+ };
+
+ String prefix;
+
+ private KeyType(String prefix) {
+ this.prefix = prefix;
+ }
+
+ public boolean matches(String value) {
+ return value.startsWith(prefix);
+ }
+
+ public String convert(String value) {
+ return process(value.substring(prefix.length()));
+ }
+
+ String process(String value) {
+ return value;
+ }
+ };
+
+ @Override
+ public String convert(String value) {
+ for (KeyType keyType : KeyType.values()) {
+ if (keyType.matches(value)) {
+ return keyType.convert(value);
+ }
+ }
+
+ return value;
+ }
+ }
+
+ // Note: Don't use "password = true" because then it will prompt even if we have a token
+ @Parameter(names = {"-p", "--password"}, description = "password (can be specified as 'pass:<password>', 'file:<local file containing the password>', "
+ + "'env:<variable containing the pass>', or stdin)", converter = PasswordConverter.class)
+ private String password;
+
+ public static class TokenConverter implements IStringConverter<AuthenticationToken> {
+ @Override
+ public AuthenticationToken convert(String value) {
+ try {
+ return Class.forName(value).asSubclass(AuthenticationToken.class).newInstance();
+ } catch (Exception e) {
+ // Catching ClassNotFoundException, ClassCastException, InstantiationException and IllegalAccessException
+ log.error("Could not instantiate AuthenticationToken " + value, e);
+ throw new ParameterException(e);
+ }
+ }
+ }
+
+ @Parameter(names = {"-tc", "--tokenClass"}, description = "token type to create, use the -l to pass options", converter = TokenConverter.class)
+ private AuthenticationToken authenticationToken;
+
+ @DynamicParameter(names = {"-l", "--tokenProperty"}, description = "login properties in the format key=value. Reuse -l for each property")
+ private Map<String,String> tokenProperties = new TreeMap<String,String>();
+
+ @Parameter(names = "--disable-tab-completion", description = "disables tab completion (for less overhead when scripting)")
+ private boolean tabCompletionDisabled;
+
+ @Parameter(names = "--debug", description = "enables client debugging")
+ private boolean debugEnabled;
+
+ @Parameter(names = "--fake", description = "fake a connection to accumulo")
+ private boolean fake;
+
+ @Parameter(names = {"-?", "--help"}, help = true, description = "display this help")
+ private boolean helpEnabled;
+
+ @Parameter(names = {"-e", "--execute-command"}, description = "executes a command, and then exits")
+ private String execCommand;
+
+ @Parameter(names = {"-f", "--execute-file"}, description = "executes commands from a file at startup", converter = FileConverter.class)
+ private File execFile;
+
+ @Parameter(names = {"-fv", "--execute-file-verbose"}, description = "executes commands from a file at startup, with commands shown",
+ converter = FileConverter.class)
+ private File execFileVerbose;
+
+ @Parameter(names = {"-h", "--hdfsZooInstance"}, description = "use hdfs zoo instance")
+ private boolean hdfsZooInstance;
+
+ @Parameter(names = {"-z", "--zooKeeperInstance"}, description = "use a zookeeper instance with the given instance name and list of zoo hosts", arity = 2)
+ private List<String> zooKeeperInstance = new ArrayList<String>();
+
+ @Parameter(names = {"--ssl"}, description = "use ssl to connect to accumulo")
+ private boolean useSsl = false;
+
- @Parameter(
- names = "--config-file",
- description = "read the given client config file. If omitted, the path searched can be specified with $ACCUMULO_CLIENT_CONF_PATH, which defaults to ~/.accumulo/config:$ACCUMULO_CONF_DIR/client.conf:/etc/accumulo/client.conf")
++ @Parameter(names = "--config-file", description = "read the given client config file. "
++ + "If omitted, the path searched can be specified with $ACCUMULO_CLIENT_CONF_PATH, "
++ + "which defaults to ~/.accumulo/config:$ACCUMULO_CONF_DIR/client.conf:/etc/accumulo/client.conf")
+ private String clientConfigFile = null;
+
+ @Parameter(names = {"-zi", "--zooKeeperInstanceName"}, description = "use a zookeeper instance with the given instance name")
+ private String zooKeeperInstanceName;
+
+ @Parameter(names = {"-zh", "--zooKeeperHosts"}, description = "use a zookeeper instance with the given list of zoo hosts")
+ private String zooKeeperHosts;
+
+ @Parameter(names = "--auth-timeout", description = "minutes the shell can be idle without re-entering a password")
+ private int authTimeout = 60; // TODO Add validator for positive number
+
+ @Parameter(names = "--disable-auth-timeout", description = "disables requiring the user to re-type a password after being idle")
+ private boolean authTimeoutDisabled;
+
+ @Parameter(hidden = true)
+ private List<String> unrecognizedOptions;
+
+ public String getUsername() {
+ return username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public AuthenticationToken getAuthenticationToken() {
+ return authenticationToken;
+ }
+
+ public Map<String,String> getTokenProperties() {
+ return tokenProperties;
+ }
+
+ public boolean isTabCompletionDisabled() {
+ return tabCompletionDisabled;
+ }
+
+ public boolean isDebugEnabled() {
+ return debugEnabled;
+ }
+
+ public boolean isFake() {
+ return fake;
+ }
+
+ public boolean isHelpEnabled() {
+ return helpEnabled;
+ }
+
+ public String getExecCommand() {
+ return execCommand;
+ }
+
+ public File getExecFile() {
+ return execFile;
+ }
+
+ public File getExecFileVerbose() {
+ return execFileVerbose;
+ }
+
+ public boolean isHdfsZooInstance() {
+ return hdfsZooInstance;
+ }
+
+ public List<String> getZooKeeperInstance() {
+ return zooKeeperInstance;
+ }
+
+ public String getZooKeeperInstanceName() {
+ return zooKeeperInstanceName;
+ }
+
+ public String getZooKeeperHosts() {
+ return zooKeeperHosts;
+ }
+
+ public int getAuthTimeout() {
+ return authTimeout;
+ }
+
+ public boolean isAuthTimeoutDisabled() {
+ return authTimeoutDisabled;
+ }
+
+ public List<String> getUnrecognizedOptions() {
+ return unrecognizedOptions;
+ }
+
+ public boolean useSsl() {
+ return useSsl;
+ }
+
+ public String getClientConfigFile() {
+ return clientConfigFile;
+ }
+
+ public ClientConfiguration getClientConfiguration() throws ConfigurationException, FileNotFoundException {
+ ClientConfiguration clientConfig = clientConfigFile == null ? ClientConfiguration.loadDefault() : new ClientConfiguration(new PropertiesConfiguration(
+ getClientConfigFile()));
+ if (useSsl()) {
+ clientConfig.setProperty(ClientProperty.INSTANCE_RPC_SSL_ENABLED, "true");
+ }
+ return clientConfig;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/shell/src/main/java/org/apache/accumulo/shell/commands/DUCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/DUCommand.java
index 1d9b9f1,0000000..3e851d4
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/DUCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/DUCommand.java
@@@ -1,125 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.DiskUsage;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.accumulo.shell.ShellOptions;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+public class DUCommand extends Command {
+
+ private Option optTablePattern, optHumanReadble, optNamespace;
+
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws IOException, TableNotFoundException,
+ NamespaceNotFoundException {
+
+ final SortedSet<String> tables = new TreeSet<String>(Arrays.asList(cl.getArgs()));
+
+ if (cl.hasOption(ShellOptions.tableOption)) {
+ String tableName = cl.getOptionValue(ShellOptions.tableOption);
+ if (!shellState.getConnector().tableOperations().exists(tableName)) {
+ throw new TableNotFoundException(tableName, tableName, "specified table that doesn't exist");
+ }
+ tables.add(tableName);
+ }
+
+ if (cl.hasOption(optNamespace.getOpt())) {
+ Instance instance = shellState.getInstance();
+ String namespaceId = Namespaces.getNamespaceId(instance, cl.getOptionValue(optNamespace.getOpt()));
+ tables.addAll(Namespaces.getTableNames(instance, namespaceId));
+ }
+
+ boolean prettyPrint = cl.hasOption(optHumanReadble.getOpt()) ? true : false;
+
+ // Add any patterns
+ if (cl.hasOption(optTablePattern.getOpt())) {
+ for (String table : shellState.getConnector().tableOperations().list()) {
+ if (table.matches(cl.getOptionValue(optTablePattern.getOpt()))) {
+ tables.add(table);
+ }
+ }
+ }
+
+ // If we didn't get any tables, and we have a table selected, add the current table
+ if (tables.isEmpty() && !shellState.getTableName().isEmpty()) {
+ tables.add(shellState.getTableName());
+ }
+
+ try {
+ String valueFormat = prettyPrint ? "%9s" : "%,24d";
+ for (DiskUsage usage : shellState.getConnector().tableOperations().getDiskUsage(tables)) {
+ Object value = prettyPrint ? NumUtil.bigNumberForSize(usage.getUsage()) : usage.getUsage();
+ shellState.getReader().println(String.format(valueFormat + " %s", value, usage.getTables()));
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ return 0;
+ }
+
+ @Override
+ public String description() {
- return "prints how much space, in bytes, is used by files referenced by a table. When multiple tables are specified it prints how much space, in bytes, is used by files shared between tables, if any.";
++ return "prints how much space, in bytes, is used by files referenced by a table. "
++ + "When multiple tables are specified it prints how much space, in bytes, is used by files shared between tables, if any.";
+ }
+
+ @Override
+ public Options getOptions() {
+ final Options o = new Options();
+
+ optTablePattern = new Option("p", "pattern", true, "regex pattern of table names");
+ optTablePattern.setArgName("pattern");
+
+ optHumanReadble = new Option("h", "human-readable", false, "format large sizes to human readable units");
+ optHumanReadble.setArgName("human readable output");
+
+ optNamespace = new Option(ShellOptions.namespaceOption, "namespace", true, "name of a namespace");
+ optNamespace.setArgName("namespace");
+
+ o.addOption(OptUtil.tableOpt("table to examine"));
+
+ o.addOption(optTablePattern);
+ o.addOption(optHumanReadble);
+ o.addOption(optNamespace);
+
+ return o;
+ }
+
+ @Override
+ public String usage() {
+ return getName() + " <table>{ <table>}";
+ }
+
+ @Override
+ public int numArgs() {
+ return Shell.NO_FIXED_ARG_LENGTH_CHECK;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/shell/src/main/java/org/apache/accumulo/shell/commands/EGrepCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/EGrepCommand.java
index eeac50c,0000000..5ffdca7
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/EGrepCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/EGrepCommand.java
@@@ -1,59 -1,0 +1,60 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+public class EGrepCommand extends GrepCommand {
+
+ private Option matchSubstringOption;
+
+ @Override
+ protected void setUpIterator(final int prio, final String name, final String term, final BatchScanner scanner, CommandLine cl) throws IOException {
+ if (prio < 0) {
+ throw new IllegalArgumentException("Priority < 0 " + prio);
+ }
+ final IteratorSetting si = new IteratorSetting(prio, name, RegExFilter.class);
+ RegExFilter.setRegexs(si, term, term, term, term, true, cl.hasOption(matchSubstringOption.getOpt()));
+ scanner.addScanIterator(si);
+ }
+
+ @Override
+ public String description() {
- return "searches each row, column family, column qualifier and value, in parallel, on the server side (using a java Matcher, so put .* before and after your term if you're not matching the whole element)";
++ return "searches each row, column family, column qualifier and value, in parallel, on the server side "
++ + "(using a java Matcher, so put .* before and after your term if you're not matching the whole element)";
+ }
+
+ @Override
+ public String usage() {
+ return getName() + " <regex>{ <regex>}";
+ }
+
+ @Override
+ public Options getOptions() {
+ final Options opts = super.getOptions();
+ matchSubstringOption = new Option("g", "global", false, "forces the use of the find() expression matcher, causing substring matches to return true");
+ opts.addOption(matchSubstringOption);
+ return opts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
index 1f3a1ae,0000000..74397de
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
@@@ -1,142 -1,0 +1,139 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.shell.Shell;
+
+/**
+ * A basic tokenizer for generating tokens from a string. It understands quoted strings and escaped quote characters.
+ *
+ * You can use the escape sequence '\' to escape single quotes, double quotes, and spaces only, in addition to the escape character itself.
+ *
+ * The behavior is the same for single and double quoted strings. (i.e. '\'' is the same as "\'")
+ */
+
+public class QuotedStringTokenizer implements Iterable<String> {
+ private ArrayList<String> tokens;
+ private String input;
+
+ public QuotedStringTokenizer(final String t) throws BadArgumentException {
+ tokens = new ArrayList<String>();
+ this.input = t;
+ try {
+ createTokens();
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+
+ public String[] getTokens() {
+ return tokens.toArray(new String[tokens.size()]);
+ }
+
+ private void createTokens() throws BadArgumentException, UnsupportedEncodingException {
+ boolean inQuote = false;
+ boolean inEscapeSequence = false;
+ String hexChars = null;
+ char inQuoteChar = '"';
+
+ final byte[] token = new byte[input.length()];
+ int tokenLength = 0;
+ final byte[] inputBytes = input.getBytes(UTF_8);
+ for (int i = 0; i < input.length(); ++i) {
+ final char ch = input.charAt(i);
+
+ // if I ended up in an escape sequence, check for valid escapable character, and add it as a literal
+ if (inEscapeSequence) {
+ inEscapeSequence = false;
+ if (ch == 'x') {
+ hexChars = "";
+ } else if (ch == ' ' || ch == '\'' || ch == '"' || ch == '\\') {
+ token[tokenLength++] = inputBytes[i];
+ } else {
+ throw new BadArgumentException("can only escape single quotes, double quotes, the space character, the backslash, and hex input", input, i);
+ }
- }
- // in a hex escape sequence
- else if (hexChars != null) {
++ } else if (hexChars != null) {
++ // in a hex escape sequence
+ final int digit = Character.digit(ch, 16);
+ if (digit < 0) {
+ throw new BadArgumentException("expected hex character", input, i);
+ }
+ hexChars += ch;
+ if (hexChars.length() == 2) {
+ byte b;
+ try {
+ b = (byte) (0xff & Short.parseShort(hexChars, 16));
+ if (!Character.isValidCodePoint(0xff & b))
+ throw new NumberFormatException();
+ } catch (NumberFormatException e) {
+ throw new BadArgumentException("unsupported non-ascii character", input, i);
+ }
+ token[tokenLength++] = b;
+ hexChars = null;
+ }
- }
- // in a quote, either end the quote, start escape, or continue a token
- else if (inQuote) {
++ } else if (inQuote) {
++ // in a quote, either end the quote, start escape, or continue a token
+ if (ch == inQuoteChar) {
+ inQuote = false;
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ tokenLength = 0;
+ } else if (ch == '\\') {
+ inEscapeSequence = true;
+ } else {
+ token[tokenLength++] = inputBytes[i];
+ }
- }
- // not in a quote, either enter a quote, end a token, start escape, or continue a token
- else {
++ } else {
++ // not in a quote, either enter a quote, end a token, start escape, or continue a token
+ if (ch == '\'' || ch == '"') {
+ if (tokenLength > 0) {
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ tokenLength = 0;
+ }
+ inQuote = true;
+ inQuoteChar = ch;
+ } else if (ch == ' ' && tokenLength > 0) {
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ tokenLength = 0;
+ } else if (ch == '\\') {
+ inEscapeSequence = true;
+ } else if (ch != ' ') {
+ token[tokenLength++] = inputBytes[i];
+ }
+ }
+ }
+ if (inQuote) {
+ throw new BadArgumentException("missing terminating quote", input, input.length());
+ } else if (inEscapeSequence || hexChars != null) {
+ throw new BadArgumentException("escape sequence not complete", input, input.length());
+ }
+ if (tokenLength > 0) {
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ }
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return tokens.iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/start/src/main/java/org/apache/accumulo/start/Main.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateTable.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateTable.java
index fb4f309,d3870a5..3392e1d
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateTable.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateTable.java
@@@ -45,12 -44,11 +45,11 @@@ public class CreateTable extends Test
if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (hasPermission)
throw new AccumuloException("Got a security exception when I should have had permission.", ae);
- else
- // create table anyway for sake of state
- {
+ else {
+ // create table anyway for sake of state
try {
- state.getConnector().tableOperations().create(tableName);
- WalkingSecurity.get(state).initTable(tableName);
+ env.getConnector().tableOperations().create(tableName);
+ WalkingSecurity.get(state, env).initTable(tableName);
} catch (TableExistsException tee) {
if (exists)
return;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/392d9d6a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateUser.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateUser.java
index eb07c43,1f539ff..5ddd441
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateUser.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/CreateUser.java
@@@ -44,12 -43,11 +44,11 @@@ public class CreateUser extends Test
case PERMISSION_DENIED:
if (hasPermission)
throw new AccumuloException("Got a security exception when I should have had permission.", ae);
- else
- // create user anyway for sake of state
- {
+ else {
+ // create user anyway for sake of state
if (!exists) {
- state.getConnector().securityOperations().createLocalUser(tableUserName, tabUserPass);
- WalkingSecurity.get(state).createUser(tableUserName, tabUserPass);
+ env.getConnector().securityOperations().createLocalUser(tableUserName, tabUserPass);
+ WalkingSecurity.get(state, env).createUser(tableUserName, tabUserPass);
Thread.sleep(1000);
}
return;