You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/12/19 17:25:07 UTC
svn commit: r1423923 [6/8] - in /accumulo/trunk: ./ bin/ core/
core/src/main/java/org/apache/accumulo/core/cli/
core/src/main/java/org/apache/accumulo/core/client/impl/
core/src/main/java/org/apache/accumulo/core/client/mapreduce/
core/src/main/java/or...
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/ZombieTServer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/ZombieTServer.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/ZombieTServer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/ZombieTServer.java Wed Dec 19 16:25:03 2012
@@ -99,7 +99,7 @@ public class ZombieTServer {
TransactionWatcher watcher = new TransactionWatcher();
final ThriftClientHandler tch = new ThriftClientHandler(instance, watcher);
Processor<Iface> processor = new Processor<Iface>(tch);
- ServerPort serverPort = TServerUtils.startTServer(port, processor, "ZombieTServer", "walking dead", 2, 1000);
+ ServerPort serverPort = TServerUtils.startTServer(port, processor, "ZombieTServer", "walking dead", 2, 1000, 10*1024*1024);
InetSocketAddress addr = new InetSocketAddress(InetAddress.getLocalHost(), serverPort.port);
String addressString = AddressUtil.toString(addr);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java Wed Dec 19 16:25:03 2012
@@ -38,9 +38,10 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.ArrayByteSequence;
@@ -77,74 +78,66 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
public class CollectTabletStats {
- public static void main(String[] args) throws Exception {
-
+ private static final Logger log = Logger.getLogger(CollectTabletStats.class);
+
+ static class CollectOptions extends ClientOnRequiredTable {
+ @Parameter(names="--iterations", description="number of iterations")
int iterations = 3;
+ @Parameter(names="-t", description="number of threads")
int numThreads = 1;
- boolean selectLocalTablets = true;
- String columnsTmp[] = new String[] {};
+ @Parameter(names="-f", description="select far tablets, default is to use local tablets")
+ boolean selectFarTablets = false;
+ @Parameter(names="-c", description="comma separated list of columns")
+ String columns;
+ }
+
+ public static void main(String[] args) throws Exception {
- int index = 0;
- String processedArgs[] = new String[8];
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-i"))
- iterations = Integer.parseInt(args[++i]);
- else if (args[i].equals("-t"))
- numThreads = Integer.parseInt(args[++i]);
- else if (args[i].equals("-l"))
- selectLocalTablets = true;
- else if (args[i].equals("-f"))
- selectLocalTablets = false;
- else if (args[i].equals("-c"))
- columnsTmp = args[++i].split(",");
- else
- processedArgs[index++] = args[i];
- }
+ final CollectOptions opts = new CollectOptions();
+ opts.parseArgs(CollectTabletStats.class.getName(), args);
+ String columnsTmp[] = new String[] {};
+ if (opts.columns != null)
+ columnsTmp = opts.columns.split(",");
final String columns[] = columnsTmp;
- if (index != 7) {
- System.err.println("USAGE : " + CollectTabletStats.class
- + " [-i <iterations>] [-t <num threads>] [-l|-f] [-c <column fams>] <instance> <zookeepers> <user> <pass> <table> <auths> <batch size>");
- return;
- }
final FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- String instance = processedArgs[0];
- String zookeepers = processedArgs[1];
- String user = processedArgs[2];
- String pass = processedArgs[3];
- final String tableName = processedArgs[4];
- final String auths[] = processedArgs[5].split(",");
- final int batchSize = Integer.parseInt(processedArgs[6]);
- ZooKeeperInstance zki = new ZooKeeperInstance(instance, zookeepers);
- final ServerConfiguration sconf = new ServerConfiguration(zki);
+ Instance instance = opts.getInstance();
+ final ServerConfiguration sconf = new ServerConfiguration(instance);
- String tableId = Tables.getNameToIdMap(zki).get(tableName);
+ String tableId = Tables.getNameToIdMap(instance).get(opts.tableName);
+ if (tableId == null) {
+ log.error("Unable to find table named " + opts.tableName);
+ System.exit(-1);
+ }
Map<KeyExtent,String> locations = new HashMap<KeyExtent,String>();
- List<KeyExtent> candidates = findTablets(selectLocalTablets, user, pass, tableName, zki, locations);
+ List<KeyExtent> candidates = findTablets(!opts.selectFarTablets, opts.user, opts.getPassword(), opts.tableName, instance, locations);
- if (candidates.size() < numThreads) {
- System.err.println("ERROR : Unable to find " + numThreads + " " + (selectLocalTablets ? "local" : "far") + " tablets");
+ if (candidates.size() < opts.numThreads) {
+ System.err.println("ERROR : Unable to find " + opts.numThreads + " " + (opts.selectFarTablets ? "far" : "local") + " tablets");
System.exit(-1);
}
- List<KeyExtent> tabletsToTest = selectRandomTablets(numThreads, candidates);
+ List<KeyExtent> tabletsToTest = selectRandomTablets(opts.numThreads, candidates);
Map<KeyExtent,List<String>> tabletFiles = new HashMap<KeyExtent,List<String>>();
for (KeyExtent ke : tabletsToTest) {
- List<String> files = getTabletFiles(user, pass, zki, tableId, ke);
+ List<String> files = getTabletFiles(opts.user, opts.getPassword(), opts.getInstance(), tableId, ke);
tabletFiles.put(ke, files);
}
System.out.println();
System.out.println("run location : " + InetAddress.getLocalHost().getHostName() + "/" + InetAddress.getLocalHost().getHostAddress());
- System.out.println("num threads : " + numThreads);
- System.out.println("table : " + tableName);
+ System.out.println("num threads : " + opts.numThreads);
+ System.out.println("table : " + opts.tableName);
System.out.println("table id : " + tableId);
for (KeyExtent ke : tabletsToTest) {
@@ -156,9 +149,9 @@ public class CollectTabletStats {
System.out.println("%n*** RUNNING TEST ***%n");
- ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
+ ExecutorService threadPool = Executors.newFixedThreadPool(opts.numThreads);
- for (int i = 0; i < iterations; i++) {
+ for (int i = 0; i < opts.iterations; i++) {
ArrayList<Test> tests = new ArrayList<Test>();
@@ -174,10 +167,10 @@ public class CollectTabletStats {
tests.add(test);
}
- runTest("read files", tests, numThreads, threadPool);
+ runTest("read files", tests, opts.numThreads, threadPool);
}
- for (int i = 0; i < iterations; i++) {
+ for (int i = 0; i < opts.iterations; i++) {
ArrayList<Test> tests = new ArrayList<Test>();
@@ -185,59 +178,59 @@ public class CollectTabletStats {
final List<String> files = tabletFiles.get(ke);
Test test = new Test(ke) {
public int runTest() throws Exception {
- return readFilesUsingIterStack(fs, sconf, files, auths, ke, columns, false);
+ return readFilesUsingIterStack(fs, sconf, files, opts.auths, ke, columns, false);
}
};
tests.add(test);
}
- runTest("read tablet files w/ system iter stack", tests, numThreads, threadPool);
+ runTest("read tablet files w/ system iter stack", tests, opts.numThreads, threadPool);
}
- for (int i = 0; i < iterations; i++) {
+ for (int i = 0; i < opts.iterations; i++) {
ArrayList<Test> tests = new ArrayList<Test>();
for (final KeyExtent ke : tabletsToTest) {
final List<String> files = tabletFiles.get(ke);
Test test = new Test(ke) {
public int runTest() throws Exception {
- return readFilesUsingIterStack(fs, sconf, files, auths, ke, columns, true);
+ return readFilesUsingIterStack(fs, sconf, files, opts.auths, ke, columns, true);
}
};
tests.add(test);
}
- runTest("read tablet files w/ table iter stack", tests, numThreads, threadPool);
+ runTest("read tablet files w/ table iter stack", tests, opts.numThreads, threadPool);
}
- for (int i = 0; i < iterations; i++) {
+ for (int i = 0; i < opts.iterations; i++) {
ArrayList<Test> tests = new ArrayList<Test>();
- final Connector conn = zki.getConnector(user, pass.getBytes());
+ final Connector conn = opts.getConnector();
for (final KeyExtent ke : tabletsToTest) {
Test test = new Test(ke) {
public int runTest() throws Exception {
- return scanTablet(conn, tableName, auths, batchSize, ke.getPrevEndRow(), ke.getEndRow(), columns);
+ return scanTablet(conn, opts.tableName, opts.auths, opts.scanBatchSize, ke.getPrevEndRow(), ke.getEndRow(), columns);
}
};
tests.add(test);
}
- runTest("read tablet data through accumulo", tests, numThreads, threadPool);
+ runTest("read tablet data through accumulo", tests, opts.numThreads, threadPool);
}
for (final KeyExtent ke : tabletsToTest) {
- final Connector conn = zki.getConnector(user, pass.getBytes());
+ final Connector conn = opts.getConnector();
threadPool.submit(new Runnable() {
public void run() {
try {
- calcTabletStats(conn, tableName, auths, batchSize, ke, columns);
+ calcTabletStats(conn, opts.tableName, opts.auths, opts.scanBatchSize, ke, columns);
} catch (Exception e) {
e.printStackTrace();
}
@@ -345,11 +338,11 @@ public class CollectTabletStats {
}
- private static List<KeyExtent> findTablets(boolean selectLocalTablets, String user, String pass, String table, ZooKeeperInstance zki,
+ private static List<KeyExtent> findTablets(boolean selectLocalTablets, String user, byte[] pass, String table, Instance zki,
Map<KeyExtent,String> locations) throws Exception {
SortedSet<KeyExtent> tablets = new TreeSet<KeyExtent>();
- MetadataTable.getEntries(zki, new AuthInfo(user, ByteBuffer.wrap(pass.getBytes()), zki.getInstanceID()), table, false, locations, tablets);
+ MetadataTable.getEntries(zki, new AuthInfo(user, ByteBuffer.wrap(pass), zki.getInstanceID()), table, false, locations, tablets);
InetAddress localaddress = InetAddress.getLocalHost();
@@ -380,11 +373,11 @@ public class CollectTabletStats {
return tabletsToTest;
}
- private static List<String> getTabletFiles(String user, String pass, ZooKeeperInstance zki, String tableId, KeyExtent ke) {
+ private static List<String> getTabletFiles(String user, byte[] pass, Instance zki, String tableId, KeyExtent ke) {
List<String> files = new ArrayList<String>();
SortedMap<Key,Value> tkv = new TreeMap<Key,Value>();
- MetadataTable.getTabletAndPrevTabletKeyValues(zki, tkv, ke, null, new AuthInfo(user, ByteBuffer.wrap(pass.getBytes()), zki.getInstanceID()));
+ MetadataTable.getTabletAndPrevTabletKeyValues(zki, tkv, ke, null, new AuthInfo(user, ByteBuffer.wrap(pass), zki.getInstanceID()));
Set<Entry<Key,Value>> es = tkv.entrySet();
for (Entry<Key,Value> entry : es) {
@@ -477,7 +470,7 @@ public class CollectTabletStats {
return columnSet;
}
- private static int readFilesUsingIterStack(FileSystem fs, ServerConfiguration aconf, List<String> files, String auths[], KeyExtent ke, String[] columns,
+ private static int readFilesUsingIterStack(FileSystem fs, ServerConfiguration aconf, List<String> files, Authorizations auths, KeyExtent ke, String[] columns,
boolean useTableIterators)
throws Exception {
@@ -492,7 +485,7 @@ public class CollectTabletStats {
List<IterInfo> emptyIterinfo = Collections.emptyList();
Map<String,Map<String,String>> emptySsio = Collections.emptyMap();
TableConfiguration tconf = aconf.getTableConfiguration(ke.getTableId().toString());
- reader = createScanIterator(ke, readers, new Authorizations(auths), new byte[] {}, new HashSet<Column>(), emptyIterinfo, emptySsio, useTableIterators, tconf);
+ reader = createScanIterator(ke, readers,auths, new byte[] {}, new HashSet<Column>(), emptyIterinfo, emptySsio, useTableIterators, tconf);
HashSet<ByteSequence> columnSet = createColumnBSS(columns);
@@ -509,9 +502,9 @@ public class CollectTabletStats {
}
- private static int scanTablet(Connector conn, String table, String[] auths, int batchSize, Text prevEndRow, Text endRow, String[] columns) throws Exception {
+ private static int scanTablet(Connector conn, String table, Authorizations auths, int batchSize, Text prevEndRow, Text endRow, String[] columns) throws Exception {
- Scanner scanner = conn.createScanner(table, new Authorizations(auths));
+ Scanner scanner = conn.createScanner(table, auths);
scanner.setBatchSize(batchSize);
scanner.setRange(new Range(prevEndRow, false, endRow, true));
@@ -529,11 +522,11 @@ public class CollectTabletStats {
return count;
}
- private static void calcTabletStats(Connector conn, String table, String[] auths, int batchSize, KeyExtent ke, String[] columns) throws Exception {
+ private static void calcTabletStats(Connector conn, String table, Authorizations auths, int batchSize, KeyExtent ke, String[] columns) throws Exception {
// long t1 = System.currentTimeMillis();
- Scanner scanner = conn.createScanner(table, new Authorizations(auths));
+ Scanner scanner = conn.createScanner(table, auths);
scanner.setBatchSize(batchSize);
scanner.setRange(new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true));
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java Wed Dec 19 16:25:03 2012
@@ -25,9 +25,12 @@ import java.util.List;
import java.util.Map;
import org.apache.accumulo.cloudtrace.thrift.TInfo;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.thrift.InitialMultiScan;
@@ -64,6 +67,8 @@ import org.apache.accumulo.server.zookee
import org.apache.hadoop.io.Text;
import org.apache.thrift.TException;
+import com.beust.jcommander.Parameter;
+
/**
* The purpose of this class is to server as fake tserver that is a data sink like /dev/null. NullTserver modifies the !METADATA location entries for a table to
@@ -198,28 +203,36 @@ public class NullTserver {
}
}
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="instance name", required=true)
+ String iname = null;
+ @Parameter(names={"-z", "--keepers"}, description="comma-separated list of zookeeper host:ports", required=true)
+ String keepers = null;
+ @Parameter(names="--table", description="table to adopt", required=true)
+ String tableName = null;
+ @Parameter(names="--port", description="port number to use")
+ int port = DefaultConfiguration.getInstance().getPort(Property.TSERV_CLIENTPORT);
+ }
+
public static void main(String[] args) throws Exception {
-
- String iname = args[0];
- String keepers = args[1];
- String tableName = args[2];
- int port = Integer.parseInt(args[3]);
+ Opts opts = new Opts();
+ opts.parseArgs(NullTserver.class.getName(), args);
TransactionWatcher watcher = new TransactionWatcher();
ThriftClientHandler tch = new ThriftClientHandler(HdfsZooInstance.getInstance(), watcher);
Processor<Iface> processor = new Processor<Iface>(tch);
- TServerUtils.startTServer(port, processor, "NullTServer", "null tserver", 2, 1000);
+ TServerUtils.startTServer(opts.port, processor, "NullTServer", "null tserver", 2, 1000, 10*1024*1024);
- InetSocketAddress addr = new InetSocketAddress(InetAddress.getLocalHost(), port);
+ InetSocketAddress addr = new InetSocketAddress(InetAddress.getLocalHost(), opts.port);
// modify !METADATA
- ZooKeeperInstance zki = new ZooKeeperInstance(iname, keepers);
- String tableId = Tables.getTableId(zki, tableName);
+ ZooKeeperInstance zki = new ZooKeeperInstance(opts.iname, opts.keepers);
+ String tableId = Tables.getTableId(zki, opts.tableName);
// read the locations for the table
Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
MetaDataTableScanner s = new MetaDataTableScanner(zki, SecurityConstants.getSystemCredentials(), tableRange);
- long randomSessionID = port;
+ long randomSessionID = opts.port;
TServerInstance instance = new TServerInstance(addr, randomSessionID);
List<Assignment> assignments = new ArrayList<Assignment>();
while (s.hasNext()) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Framework.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Framework.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Framework.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Framework.java Wed Dec 19 16:25:03 2012
@@ -20,9 +20,12 @@ import java.io.File;
import java.io.FileInputStream;
import java.util.HashMap;
import java.util.Properties;
+
import org.apache.log4j.Logger;
import org.apache.log4j.xml.DOMConfigurator;
+import com.beust.jcommander.Parameter;
+
public class Framework {
private static final Logger log = Logger.getLogger(Framework.class);
@@ -92,28 +95,33 @@ public class Framework {
return node;
}
+ static class Opts extends org.apache.accumulo.core.cli.Help {
+ @Parameter(names="--configDir", required=true, description="directory containing the test configuration")
+ String configDir;
+ @Parameter(names="--logDir", required=true, description="location of the local logging directory")
+ String localLogPath;
+ @Parameter(names="--logId", required=true, description="a unique log identifier (like a hostname, or pid)")
+ String logId;
+ @Parameter(names="--module", required=true, description="the name of the module to run")
+ String module;
+ }
+
public static void main(String[] args) throws Exception {
-
- if (args.length != 4) {
- throw new IllegalArgumentException("usage : Framework <configDir> <localLogPath> <logId> <module>");
- }
- String configDir = args[0];
- String localLogPath = args[1];
- String logId = args[2];
- String module = args[3];
-
+ Opts opts = new Opts();
+ opts.parseArgs(Framework.class.getName(), args);
+
Properties props = new Properties();
- FileInputStream fis = new FileInputStream(configDir + "/randomwalk.conf");
+ FileInputStream fis = new FileInputStream(opts.configDir + "/randomwalk.conf");
props.load(fis);
fis.close();
- System.setProperty("localLog", localLogPath + "/" + logId);
- System.setProperty("nfsLog", props.getProperty("NFS_LOGPATH") + "/" + logId);
+ System.setProperty("localLog", opts.localLogPath + "/" + opts.logId);
+ System.setProperty("nfsLog", props.getProperty("NFS_LOGPATH") + "/" + opts.logId);
- DOMConfigurator.configure(configDir + "logger.xml");
+ DOMConfigurator.configure(opts.configDir + "logger.xml");
State state = new State(props);
- int retval = getInstance().run(module, state, configDir);
+ int retval = getInstance().run(opts.module, state, configDir);
System.exit(retval);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Run.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Run.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Run.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Run.java Wed Dec 19 16:25:03 2012
@@ -20,33 +20,39 @@ import java.io.FileInputStream;
import java.util.Properties;
import java.net.InetAddress;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.server.test.scalability.ScaleTest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import com.beust.jcommander.Parameter;
+
public class Run {
+ static class Opts extends Help {
+ @Parameter(names="--testId", required=true)
+ String testId;
+ @Parameter(names="--action", required=true, description="one of 'setup', 'teardown' or 'client'")
+ String action;
+ @Parameter(names="--count", description="number of tablet servers", required=true)
+ int numTabletServers;
+ }
+
public static void main(String[] args) throws Exception {
final String sitePath = "/tmp/scale-site.conf";
final String testPath = "/tmp/scale-test.conf";
-
- // parse command line
- if (args.length != 3) {
- throw new IllegalArgumentException("usage : Run <testId> <action> <numTabletServers>");
- }
- String testId = args[0];
- String action = args[1];
- int numTabletServers = Integer.parseInt(args[2]);
+ Opts opts = new Opts();
+ opts.parseArgs(Run.class.getName(), args);
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs;
fs = FileSystem.get(conf);
fs.copyToLocalFile(new Path("/accumulo-scale/conf/site.conf"), new Path(sitePath));
- fs.copyToLocalFile(new Path(String.format("/accumulo-scale/conf/%s.conf", testId)), new Path(testPath));
+ fs.copyToLocalFile(new Path(String.format("/accumulo-scale/conf/%s.conf", opts.testId)), new Path(testPath));
// load configuration file properties
Properties scaleProps = new Properties();
@@ -62,19 +68,19 @@ public class Run {
e.printStackTrace();
}
- ScaleTest test = (ScaleTest) Class.forName(String.format("accumulo.server.test.scalability.%s", testId)).newInstance();
+ ScaleTest test = (ScaleTest) Class.forName(String.format("accumulo.server.test.scalability.%s", opts.testId)).newInstance();
- test.init(scaleProps, testProps, numTabletServers);
+ test.init(scaleProps, testProps, opts.numTabletServers);
- if (action.equalsIgnoreCase("setup")) {
+ if (opts.action.equalsIgnoreCase("setup")) {
test.setup();
- } else if (action.equalsIgnoreCase("client")) {
+ } else if (opts.action.equalsIgnoreCase("client")) {
InetAddress addr = InetAddress.getLocalHost();
String host = addr.getHostName();
fs.createNewFile(new Path("/accumulo-scale/clients/" + host));
test.client();
fs.copyFromLocalFile(new Path("/tmp/scale.out"), new Path("/accumulo-scale/results/" + host));
- } else if (action.equalsIgnoreCase("teardown")) {
+ } else if (opts.action.equalsIgnoreCase("teardown")) {
test.teardown();
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java Wed Dec 19 16:25:03 2012
@@ -16,14 +16,12 @@
*/
package org.apache.accumulo.server.util;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
@@ -32,10 +30,7 @@ import org.apache.accumulo.core.data.Mut
import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -43,25 +38,32 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
+import com.beust.jcommander.Parameter;
+
public class AddFilesWithMissingEntries {
static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
- static boolean update = false;
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names="-update", description="Make changes to the !METADATA table to include missing files")
+ boolean update = false;
+ }
+
/**
+ * A utility to add files to the !METADATA table that are not listed in the root tablet.
+ * This is a recovery tool for someone who knows what they are doing. It might be better to
+ * save off files, and recover your instance by re-initializing and importing the existing files.
+ *
* @param args
*/
public static void main(String[] args) throws Exception {
- if (args.length > 1 || new HashSet<String>(Arrays.asList(args)).contains("-?")) {
- System.err.println("Usage: bin/accumulo " + AddFilesWithMissingEntries.class.getName() + " [update]");
- System.exit(1);
- }
- update = args.length > 0;
- final AuthInfo creds = SecurityConstants.getSystemCredentials();
- final Connector connector = HdfsZooInstance.getInstance().getConnector(creds.getUser(), creds.getPassword());
+ Opts opts = new Opts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args);
+
final Key rootTableEnd = new Key(Constants.ROOT_TABLET_EXTENT.getEndRow());
final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, Constants.METADATA_RESERVED_KEYSPACE_START_KEY, false);
- final Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ final Scanner scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
scanner.setRange(range);
final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.get(conf);
@@ -71,7 +73,7 @@ public class AddFilesWithMissingEntries
Set<String> knownFiles = new HashSet<String>();
int count = 0;
- final MultiTableBatchWriter writer = connector.createMultiTableBatchWriter(new BatchWriterConfig());
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(opts.getBatchWriterConfig());
// collect the list of known files and the directory for each extent
for (Entry<Key,Value> entry : scanner) {
@@ -81,7 +83,7 @@ public class AddFilesWithMissingEntries
if (!ke.equals(last)) {
if (directory != null) {
// add any files in the directory unknown to the key extent
- count += addUnknownFiles(fs, directory, knownFiles, last, writer);
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
}
directory = null;
knownFiles.clear();
@@ -98,13 +100,13 @@ public class AddFilesWithMissingEntries
}
if (directory != null) {
// catch the last key extent
- count += addUnknownFiles(fs, directory, knownFiles, last, writer);
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
}
log.info("There were " + count + " files that are unknown to the metadata table");
writer.close();
}
- private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer) throws Exception {
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update) throws Exception {
int count = 0;
final String tableId = ke.getTableId().toString();
final Text row = ke.getMetadataEntry();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Admin.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Admin.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Admin.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Admin.java Wed Dec 19 16:25:03 2012
@@ -16,17 +16,16 @@
*/
package org.apache.accumulo.server.util;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-
-import jline.ConsoleReader;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.accumulo.cloudtrace.instrument.Tracer;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.impl.ClientExec;
import org.apache.accumulo.core.client.impl.MasterClient;
import org.apache.accumulo.core.conf.Property;
@@ -34,81 +33,67 @@ import org.apache.accumulo.core.master.t
import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.security.SecurityConstants;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
import org.apache.log4j.Logger;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
public class Admin {
private static final Logger log = Logger.getLogger(Admin.class);
+ static class AdminOpts extends ClientOpts {
+ @Parameter(names={"-f", "--force"}, description="force the given server to stop by removing its lock")
+ boolean force = false;
+ }
+
+ @Parameters(commandDescription="stop the tablet server on the given hosts")
+ static class StopCommand {
+ @Parameter(description="<host> {<host> ... }")
+ List<String> args = new ArrayList<String>();
+ }
+
+ @Parameters(commandDescription="stop the master")
+ static class StopMasterCommand {
+ }
+
+ @Parameters(commandDescription="stop all the servers")
+ static class StopAllCommand {
+ }
+
public static void main(String[] args) {
boolean everything;
+
+ AdminOpts opts = new AdminOpts();
+ JCommander cl = new JCommander(opts);
+ cl.setProgramName(Admin.class.getName());
+ StopCommand stopOpts = new StopCommand();
+ cl.addCommand("stop", stopOpts);
+ StopMasterCommand stopMasterOpts = new StopMasterCommand();
+ cl.addCommand("stopMaster", stopMasterOpts);
+ StopAllCommand stopAllOpts = new StopAllCommand();
+ cl.addCommand("stopAll", stopAllOpts);
+ cl.parse(args);
- CommandLine cl = null;
- Options opts = new Options();
- opts.addOption("u", true, "optional administrator user name");
- opts.addOption("p", true, "optional administrator password");
- opts.addOption("f", "force", false, "force the given server to stop by removing its lock");
- opts.addOption("?", "help", false, "displays the help");
- String user = null;
- byte[] pass = null;
- boolean force = false;
-
- try {
- cl = new BasicParser().parse(opts, args);
- if (cl.hasOption("?"))
- throw new ParseException("help requested");
- args = cl.getArgs();
-
- user = cl.hasOption("u") ? cl.getOptionValue("u") : "root";
- pass = cl.hasOption("p") ? cl.getOptionValue("p").getBytes() : null;
- force = cl.hasOption("f");
-
- if (!((cl.getArgs().length == 1 && (args[0].equalsIgnoreCase("stopMaster") || args[0].equalsIgnoreCase("stopAll"))) || (cl.getArgs().length == 2 && args[0]
- .equalsIgnoreCase("stop"))))
- throw new ParseException("Incorrect arguments");
-
- } catch (ParseException e) {
- // print to the log and to stderr
- if (cl == null || !cl.hasOption("?"))
- log.error(e, e);
- HelpFormatter h = new HelpFormatter();
- StringWriter str = new StringWriter();
- h.printHelp(new PrintWriter(str), h.getWidth(), Admin.class.getName() + " stopMaster | stopAll | stop <tserver>", null, opts, h.getLeftPadding(),
- h.getDescPadding(), null, true);
- if (cl != null && cl.hasOption("?"))
- log.info(str.toString());
- else
- log.error(str.toString());
- h.printHelp(new PrintWriter(System.err), h.getWidth(), Admin.class.getName() + " stopMaster | stopAll | stop <tserver>", null, opts, h.getLeftPadding(),
- h.getDescPadding(), null, true);
- System.exit(3);
+ if (opts.help || cl.getParsedCommand() == null) {
+ cl.usage();
+ return;
}
-
+ Instance instance = opts.getInstance();
+
try {
AuthInfo creds;
- if (args[0].equalsIgnoreCase("stop")) {
- stopTabletServer(args[1], force);
+ if (opts.getPassword() == null) {
+ creds = SecurityConstants.getSystemCredentials();
} else {
- if (!cl.hasOption("u") && !cl.hasOption("p")) {
- creds = SecurityConstants.getSystemCredentials();
- } else {
- if (pass == null) {
- try {
- pass = new ConsoleReader().readLine("Enter current password for '" + user + "': ", '*').getBytes();
- } catch (IOException ioe) {
- log.error("Password not specified and unable to prompt: " + ioe);
- System.exit(4);
- }
- }
- creds = new AuthInfo(user, ByteBuffer.wrap(pass), HdfsZooInstance.getInstance().getInstanceID());
- }
-
- everything = args[0].equalsIgnoreCase("stopAll");
- stopServer(creds, everything);
+ creds = new AuthInfo(opts.user, ByteBuffer.wrap(opts.getPassword()), instance.getInstanceID());
+ }
+
+ if (cl.getParsedCommand().equals("stop")) {
+ stopTabletServer(instance, creds, stopOpts.args, opts.force);
+ } else {
+ everything = cl.getParsedCommand().equals("stopAll");
+ stopServer(instance, creds, everything);
}
} catch (AccumuloException e) {
log.error(e);
@@ -119,7 +104,7 @@ public class Admin {
}
}
- private static void stopServer(final AuthInfo credentials, final boolean tabletServersToo) throws AccumuloException, AccumuloSecurityException {
+ private static void stopServer(Instance instance, final AuthInfo credentials, final boolean tabletServersToo) throws AccumuloException, AccumuloSecurityException {
MasterClient.execute(HdfsZooInstance.getInstance(), new ClientExec<MasterClientService.Client>() {
@Override
public void execute(MasterClientService.Client client) throws Exception {
@@ -128,14 +113,17 @@ public class Admin {
});
}
- private static void stopTabletServer(String server, final boolean force) throws AccumuloException, AccumuloSecurityException {
- InetSocketAddress address = AddressUtil.parseAddress(server, Property.TSERV_CLIENTPORT);
- final String finalServer = org.apache.accumulo.core.util.AddressUtil.toString(address);
- MasterClient.execute(HdfsZooInstance.getInstance(), new ClientExec<MasterClientService.Client>() {
- @Override
- public void execute(MasterClientService.Client client) throws Exception {
- client.shutdownTabletServer(Tracer.traceInfo(), SecurityConstants.getSystemCredentials(), finalServer, force);
- }
- });
+ private static void stopTabletServer(Instance instance, final AuthInfo creds, List<String> servers, final boolean force) throws AccumuloException, AccumuloSecurityException {
+ for (String server : servers) {
+ InetSocketAddress address = AddressUtil.parseAddress(server, Property.TSERV_CLIENTPORT);
+ final String finalServer = org.apache.accumulo.core.util.AddressUtil.toString(address);
+ log.info("Stopping server " + finalServer);
+ MasterClient.execute(HdfsZooInstance.getInstance(), new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.shutdownTabletServer(Tracer.traceInfo(), creds, finalServer, force);
+ }
+ });
+ }
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java Wed Dec 19 16:25:03 2012
@@ -18,9 +18,11 @@ package org.apache.accumulo.server.util;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.UUID;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -29,7 +31,6 @@ import org.apache.accumulo.fate.zookeepe
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -37,26 +38,31 @@ import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
+import com.beust.jcommander.Parameter;
+
public class ChangeSecret {
+ static class Opts extends ClientOpts {
+ @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
+ String oldPass;
+ @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
+ String newPass;
+ }
+
public static void main(String[] args) throws Exception {
- String oldPass = null;
- String newPass = null;
- if (args.length < 2) {
- System.err.println("Usage: accumulo " + ChangeSecret.class.getName() + " oldPass newPass");
- System.exit(-1);
- }
- oldPass = args[0];
- newPass = args[1];
-
+ Opts opts = new Opts();
+ List<String> argsList = Arrays.asList(args);
+ argsList.add("--old");
+ argsList.add("--new");
+ opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- Instance inst = HdfsZooInstance.getInstance();
- if (!verifyAccumuloIsDown(inst, oldPass))
+ Instance inst = opts.getInstance();
+ if (!verifyAccumuloIsDown(inst, opts.oldPass))
System.exit(-1);
- String instanceId = rewriteZooKeeperInstance(inst, oldPass, newPass);
+ String instanceId = rewriteZooKeeperInstance(inst, opts.oldPass, opts.newPass);
updateHdfs(fs, inst, instanceId);
- if (oldPass != null) {
- deleteInstance(inst, oldPass);
+ if (opts.oldPass != null) {
+ deleteInstance(inst, opts.oldPass);
}
System.out.println("New instance id is " + instanceId);
System.out.println("Be sure to put your new secret in accumulo-site.xml");
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java Wed Dec 19 16:25:03 2012
@@ -16,8 +16,6 @@
*/
package org.apache.accumulo.server.util;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -26,33 +24,25 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.impl.ScannerImpl;
import org.apache.accumulo.core.client.impl.Writer;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
public class CheckForMetadataProblems {
- private static String user;
- private static byte[] pass;
- private static boolean fix = false;
- private static boolean offline = false;
-
private static boolean sawProblems = false;
- public static void checkTable(String tablename, TreeSet<KeyExtent> tablets, boolean patch) {
+ public static void checkTable(String tablename, TreeSet<KeyExtent> tablets, Opts opts) {
// sanity check of metadata table entries
// make sure tablets has no holes, and that it starts and ends w/ null
@@ -90,10 +80,10 @@ public class CheckForMetadataProblems {
if (broke) {
everythingLooksGood = false;
}
- if (broke && patch) {
+ if (broke && opts.fix) {
KeyExtent ke = new KeyExtent(tabke);
ke.setPrevEndRow(lastEndRow);
- MetadataTable.updateTabletPrevEndRow(ke, new AuthInfo(user, ByteBuffer.wrap(pass), HdfsZooInstance.getInstance().getInstanceID()));
+ MetadataTable.updateTabletPrevEndRow(ke, opts.getAuthInfo());
System.out.println("KE " + tabke + " has been repaired to " + ke);
}
@@ -105,16 +95,15 @@ public class CheckForMetadataProblems {
sawProblems = true;
}
- public static void checkMetadataTableEntries(ServerConfiguration conf, FileSystem fs, boolean offline, boolean patch) throws Exception {
+ public static void checkMetadataTableEntries(Opts opts, FileSystem fs) throws Exception {
Map<String,TreeSet<KeyExtent>> tables = new HashMap<String,TreeSet<KeyExtent>>();
Scanner scanner;
- if (offline) {
- scanner = new OfflineMetadataScanner(conf.getConfiguration(), fs);
+ if (opts.offline) {
+ scanner = new OfflineMetadataScanner(ServerConfiguration.getSystemConfiguration(opts.getInstance()), fs);
} else {
- scanner = new ScannerImpl(conf.getInstance(), new AuthInfo(user, ByteBuffer.wrap(pass), conf.getInstance().getInstanceID()),
- Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
}
scanner.setRange(Constants.METADATA_KEYSPACE);
@@ -140,7 +129,7 @@ public class CheckForMetadataProblems {
Set<Entry<String,TreeSet<KeyExtent>>> es = tables.entrySet();
for (Entry<String,TreeSet<KeyExtent>> entry2 : es) {
- checkTable(entry2.getKey(), entry2.getValue(), patch);
+ checkTable(entry2.getKey(), entry2.getValue(), opts);
}
tables.clear();
@@ -157,8 +146,8 @@ public class CheckForMetadataProblems {
if (justLoc) {
System.out.println("Problem at key " + entry.getKey());
sawProblems = true;
- if (patch) {
- Writer t = MetadataTable.getMetadataTable(new AuthInfo(user, ByteBuffer.wrap(pass), HdfsZooInstance.getInstance().getInstanceID()));
+ if (opts.fix) {
+ Writer t = MetadataTable.getMetadataTable(opts.getAuthInfo());
Key k = entry.getKey();
Mutation m = new Mutation(k.getRow());
m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
@@ -182,57 +171,28 @@ public class CheckForMetadataProblems {
Set<Entry<String,TreeSet<KeyExtent>>> es = tables.entrySet();
for (Entry<String,TreeSet<KeyExtent>> entry : es) {
- checkTable(entry.getKey(), entry.getValue(), patch);
+ checkTable(entry.getKey(), entry.getValue(), opts);
}
// end METADATA table sanity check
}
- private static String[] processOptions(String[] args) {
- ArrayList<String> al = new ArrayList<String>();
-
- for (String s : args) {
- if (s.equals("--debug")) {
- enableDebug();
- } else if (s.equals("--fix")) {
- fix = true;
- } else if (s.equals("--offline")) {
- offline = true;
- } else {
- al.add(s);
- }
- }
-
- if (offline && fix) {
- throw new IllegalArgumentException("Cannot fix in offline mode");
- }
+ static class Opts extends ClientOpts {
+ @Parameter(names="--fix", description="best-effort attempt to fix problems found")
+ boolean fix = false;
- return al.toArray(new String[al.size()]);
+ @Parameter(names="--offline", description="perform the check on the files directly")
+ boolean offline = false;
}
- private static void enableDebug() {
- Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
- logger.setLevel(Level.TRACE);
- }
public static void main(String[] args) throws Exception {
- args = processOptions(args);
+ Opts opts = new Opts();
+ opts.parseArgs(CheckForMetadataProblems.class.getName(), args);
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- Instance instance = HdfsZooInstance.getInstance();
- ServerConfiguration conf = new ServerConfiguration(instance);
-
- if (args.length == 2) {
- user = args[0];
- pass = args[1].getBytes();
- checkMetadataTableEntries(conf, fs, offline, fix);
- } else if (args.length == 0 && offline) {
- checkMetadataTableEntries(conf, fs, offline, fix);
- } else {
- System.out.println("Usage: " + CheckForMetadataProblems.class.getName() + " (--offline)|([--debug] [--fix] <username> <password>)");
- System.exit(-1);
- }
-
+ checkMetadataTableEntries(opts, fs);
+ opts.stopTracing();
if (sawProblems)
System.exit(-1);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java Wed Dec 19 16:25:03 2012
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.util;
import java.io.IOException;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.client.HdfsZooInstance;
@@ -26,10 +27,19 @@ import org.apache.accumulo.server.zookee
import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
+import com.beust.jcommander.Parameter;
+
public class CleanZookeeper {
private static final Logger log = Logger.getLogger(CleanZookeeper.class);
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
+ String keepers = "localhost:2181";
+ @Parameter(names={"--password"}, description="the system secret", password=true)
+ String auth;
+ }
+
/**
* @param args
* must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
@@ -37,14 +47,13 @@ public class CleanZookeeper {
* error connecting to accumulo or zookeeper
*/
public static void main(String[] args) throws IOException {
- if (args.length < 1) {
- System.err.println("Usage: " + CleanZookeeper.class.getName() + " hostname[:port] [auth]");
- System.exit(1);
- }
+ Opts opts = new Opts();
+ opts.parseArgs(CleanZookeeper.class.getName(), args);
+
String root = Constants.ZROOT;
IZooReaderWriter zk = ZooReaderWriter.getInstance();
- if (args.length == 2) {
- zk.getZooKeeper().addAuthInfo("digest", args[1].getBytes());
+ if (opts.auth != null) {
+ zk.getZooKeeper().addAuthInfo("digest", opts.auth.getBytes());
}
try {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java Wed Dec 19 16:25:03 2012
@@ -20,42 +20,61 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
public class DeleteZooInstance {
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
+ String instance;
+ }
+
+ static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
+ for (int i = 0; i < 10; i++){
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ return;
+ } catch (KeeperException.NotEmptyException ex) {
+ // ignored
+ } catch (Exception ex) {
+ throw ex;
+ }
+ }
+ }
+
/**
* @param args
* : the name or UUID of the instance to be deleted
*/
public static void main(String[] args) throws Exception {
- if (args.length < 1) {
- System.err.println("Usage: " + DeleteZooInstance.class.getName() + " [instanceName|UUID ... ]");
- System.exit(1);
- }
+ Opts opts = new Opts();
+ opts.parseArgs(DeleteZooInstance.class.getName(), args);
+
IZooReaderWriter zk = ZooReaderWriter.getInstance();
// try instance name:
Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
uuids.remove("instances");
- for (String name : args) {
- if (instances.contains(args[0])) {
- String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ if (instances.contains(opts.instance)) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
+ byte[] data = zk.getData(path, null);
+ deleteRetry(zk, path);
+ deleteRetry(zk, Constants.ZROOT + "/" + new String(data));
+ } else if (uuids.contains(opts.instance)) {
+ // look for the real instance name
+ for (String instance : instances) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
byte[] data = zk.getData(path, null);
- zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
- zk.recursiveDelete(Constants.ZROOT + "/" + new String(data), NodeMissingPolicy.SKIP);
- } else if (uuids.contains(name)) {
- // look for the real instance name
- for (String instance : instances) {
- String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
- byte[] data = zk.getData(path, null);
- if (name.equals(new String(data)))
- zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
- }
- zk.recursiveDelete(Constants.ZROOT + "/" + name, NodeMissingPolicy.SKIP);
+ if (opts.instance.equals(new String(data)))
+ deleteRetry(zk, path);
}
+ deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java Wed Dec 19 16:25:03 2012
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.util;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.commons.codec.binary.Base64;
@@ -27,6 +28,8 @@ import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
+import com.beust.jcommander.Parameter;
+
public class DumpZookeeper {
static IZooReaderWriter zk = null;
@@ -43,24 +46,31 @@ public class DumpZookeeper {
}
}
+ static class Opts extends Help {
+ @Parameter(names="--root", description="the root of the znode tree to dump")
+ String root = "/";
+ }
+
/**
* @param args
*/
public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
Logger.getRootLogger().setLevel(Level.WARN);
PrintStream out = System.out;
// int timeout = 30 * 1000;
// String server = args[0];
- String root = "/";
if (args.length > 0)
- root = args[0];
+ opts.root = args[0];
try {
zk = ZooReaderWriter.getInstance();
- write(out, 0, "<dump root='%s'>", root);
- for (String child : zk.getChildren(root, null))
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
if (!child.equals("zookeeper"))
- dump(out, root, child, 1);
+ dump(out, opts.root, child, 1);
write(out, 0, "</dump>");
} catch (Exception ex) {
log.error(ex, ex);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java Wed Dec 19 16:25:03 2012
@@ -18,8 +18,8 @@ package org.apache.accumulo.server.util;
import java.util.Set;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.master.state.tables.TableState;
@@ -30,7 +30,6 @@ import org.apache.accumulo.server.master
import org.apache.accumulo.server.master.state.TabletLocationState;
import org.apache.accumulo.server.master.state.TabletState;
import org.apache.accumulo.server.master.state.tables.TableManager;
-import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.log4j.Logger;
public class FindOfflineTablets {
@@ -40,15 +39,12 @@ public class FindOfflineTablets {
* @param args
*/
public static void main(String[] args) throws Exception {
- if (args.length != 4) {
- System.err.println("Usage: accumulo.server.util.FindOfflineTablets instance zookeepers");
- System.exit(1);
- }
- String instance = args[0];
- String keepers = args[1];
- Instance zooInst = new ZooKeeperInstance(instance, keepers);
- MetaDataTableScanner scanner = new MetaDataTableScanner(zooInst, SecurityConstants.getSystemCredentials(), new Range());
- LiveTServerSet tservers = new LiveTServerSet(zooInst, DefaultConfiguration.getDefaultConfiguration(), new Listener() {
+ ClientOpts opts = new ClientOpts();
+ opts.parseArgs(FindOfflineTablets.class.getName(), args);
+
+ Instance instance = opts.getInstance();
+ MetaDataTableScanner scanner = new MetaDataTableScanner(instance, opts.getAuthInfo(), new Range());
+ LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(), new Listener() {
@Override
public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
if (!deleted.isEmpty())
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java Wed Dec 19 16:25:03 2012
@@ -26,6 +26,7 @@ import java.util.UUID;
import jline.ConsoleReader;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -63,6 +64,8 @@ import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
+import com.beust.jcommander.Parameter;
+
/**
* This class is used to setup the directory structure and the root tablet to get an instance started
*
@@ -70,9 +73,6 @@ import org.apache.zookeeper.ZooDefs.Ids;
public class Initialize {
private static final Logger log = Logger.getLogger(Initialize.class);
private static final String ROOT_USER = "root";
- private static boolean clearInstanceName = false;
- private static String cliInstanceName = null;
- private static String cliPassword = null;
private static ConsoleReader reader = null;
@@ -109,7 +109,7 @@ public class Initialize {
initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
}
- public static boolean doInit(Configuration conf, FileSystem fs) throws IOException {
+ public static boolean doInit(Opts opts, Configuration conf, FileSystem fs) throws IOException {
if (!ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI).equals(""))
log.info("Hadoop Filesystem is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI));
else
@@ -152,34 +152,34 @@ public class Initialize {
// abort, we don't leave an inconsistent HDFS/ZooKeeper structure
String instanceNamePath;
try {
- instanceNamePath = getInstanceNamePath();
+ instanceNamePath = getInstanceNamePath(opts);
} catch (Exception e) {
log.fatal("Failed to talk to zookeeper", e);
return false;
}
- byte[] rootpass = getRootPassword();
- return initialize(instanceNamePath, fs, rootpass);
+ opts.rootpass = getRootPassword(opts);
+ return initialize(opts, instanceNamePath, fs);
}
- public static boolean initialize(String instanceNamePath, FileSystem fs, byte[] rootpass) {
+ public static boolean initialize(Opts opts, String instanceNamePath, FileSystem fs) {
UUID uuid = UUID.randomUUID();
try {
- initZooKeeper(uuid.toString(), instanceNamePath);
+ initZooKeeper(opts, uuid.toString(), instanceNamePath);
} catch (Exception e) {
log.fatal("Failed to initialize zookeeper", e);
return false;
}
try {
- initFileSystem(fs, fs.getConf(), uuid);
+ initFileSystem(opts, fs, fs.getConf(), uuid);
} catch (Exception e) {
log.fatal("Failed to initialize filesystem", e);
return false;
}
try {
- initSecurity(uuid.toString(), rootpass);
+ initSecurity(opts, uuid.toString());
} catch (Exception e) {
log.fatal("Failed to initialize security", e);
return false;
@@ -201,7 +201,7 @@ public class Initialize {
}
}
- private static void initFileSystem(FileSystem fs, Configuration conf, UUID uuid) throws IOException {
+ private static void initFileSystem(Opts opts, FileSystem fs, Configuration conf, UUID uuid) throws IOException {
FileStatus fstat;
// the actual disk location of the root tablet
@@ -335,14 +335,14 @@ public class Initialize {
}
}
- private static void initZooKeeper(String uuid, String instanceNamePath) throws KeeperException, InterruptedException {
+ private static void initZooKeeper(Opts opts, String uuid, String instanceNamePath) throws KeeperException, InterruptedException {
// setup basic data in zookeeper
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
// setup instance name
- if (clearInstanceName)
+ if (opts.clearInstanceName)
zoo.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
zoo.putPersistentData(instanceNamePath, uuid.getBytes(), NodeExistsPolicy.FAIL);
@@ -368,15 +368,15 @@ public class Initialize {
zoo.putPersistentData(zkInstanceRoot + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.FAIL);
}
- private static String getInstanceNamePath() throws IOException, KeeperException, InterruptedException {
+ private static String getInstanceNamePath(Opts opts) throws IOException, KeeperException, InterruptedException {
// setup the instance name
String instanceName, instanceNamePath = null;
boolean exists = true;
do {
- if (cliInstanceName == null) {
+ if (opts.cliInstanceName == null) {
instanceName = getConsoleReader().readLine("Instance name : ");
} else {
- instanceName = cliInstanceName;
+ instanceName = opts.cliInstanceName;
}
if (instanceName == null)
System.exit(0);
@@ -384,7 +384,7 @@ public class Initialize {
if (instanceName.length() == 0)
continue;
instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
- if (clearInstanceName) {
+ if (opts.clearInstanceName) {
exists = false;
break;
} else if ((boolean) (exists = ZooReaderWriter.getInstance().exists(instanceNamePath))) {
@@ -392,7 +392,7 @@ public class Initialize {
if (decision == null)
System.exit(0);
if (decision.length() == 1 && decision.toLowerCase(Locale.ENGLISH).charAt(0) == 'y') {
- clearInstanceName = true;
+ opts.clearInstanceName = true;
exists = false;
}
}
@@ -400,9 +400,9 @@ public class Initialize {
return instanceNamePath;
}
- private static byte[] getRootPassword() throws IOException {
- if (cliPassword != null) {
- return cliPassword.getBytes();
+ private static byte[] getRootPassword(Opts opts) throws IOException {
+ if (opts.cliPassword != null) {
+ return opts.cliPassword.getBytes();
}
String rootpass;
String confirmpass;
@@ -419,8 +419,8 @@ public class Initialize {
return rootpass.getBytes();
}
- private static void initSecurity(String iid, byte[] rootpass) throws AccumuloSecurityException {
- new ZKAuthenticator(iid).initializeSecurity(SecurityConstants.getSystemCredentials(), ROOT_USER, rootpass);
+ private static void initSecurity(Opts opts, String iid) throws AccumuloSecurityException {
+ new ZKAuthenticator(iid).initializeSecurity(SecurityConstants.getSystemCredentials(), ROOT_USER, opts.rootpass);
}
protected static void initMetadataConfig() throws IOException {
@@ -458,27 +458,27 @@ public class Initialize {
return (fs.exists(ServerConstants.getInstanceIdLocation()) || fs.exists(ServerConstants.getDataVersionLocation()));
}
+ static class Opts extends Help {
+ @Parameter(names="--reset-security", description="just update the security information")
+ boolean resetSecurity = false;
+ @Parameter(names="--clear-instance-name", description="delete any existing instance name without prompting")
+ boolean clearInstanceName = false;
+ @Parameter(names="--instance-name", description="the instance name, if not provided, will prompt")
+ String cliInstanceName;
+ @Parameter(names="--password", description="set the password on the command line")
+ String cliPassword;
+ @Parameter(names="--held", help=true)
+
+ byte[] rootpass = null;
+ }
+
+
+
public static void main(String[] args) {
- boolean justSecurity = false;
+ Opts opts = new Opts();
+ opts.parseArgs(Initialize.class.getName(), args);
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("--reset-security")) {
- justSecurity = true;
- } else if (args[i].equals("--clear-instance-name")) {
- clearInstanceName = true;
- } else if (args[i].equals("--instance-name")) {
- cliInstanceName = args[i+1];
- i++;
- } else if (args[i].equals("--password")) {
- cliPassword = args[i+1];
- i++;
- } else {
- RuntimeException e = new RuntimeException();
- log.fatal("Usage: [--reset-security] [--clear-instance-name] [--instance-name {name}] [--password {password}]");
- log.fatal("Bad argument " + args[i], e);
- throw e;
- }
- }
+ boolean justSecurity = false;
try {
SecurityUtil.serverLogin();
@@ -487,11 +487,13 @@ public class Initialize {
FileSystem fs = FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration());
if (justSecurity) {
- if (isInitialized(fs))
- initSecurity(HdfsZooInstance.getInstance().getInstanceID(), getRootPassword());
- else
+ if (isInitialized(fs)) {
+ opts.rootpass = getRootPassword(opts);
+ initSecurity(opts, HdfsZooInstance.getInstance().getInstanceID());
+ } else {
log.fatal("Attempted to reset security on accumulo before it was initialized");
- } else if (!doInit(conf, fs))
+ }
+ } else if (!doInit(opts, conf, fs))
System.exit(-1);
} catch (Exception e) {
log.fatal(e, e);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ListInstances.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ListInstances.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ListInstances.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ListInstances.java Wed Dec 19 16:25:03 2012
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.server.util;
-import java.util.ArrayList;
import java.util.Formattable;
import java.util.Formatter;
import java.util.List;
@@ -26,6 +25,7 @@ import java.util.TreeSet;
import java.util.UUID;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -33,6 +33,8 @@ import org.apache.accumulo.server.zookee
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.log4j.Logger;
+import com.beust.jcommander.Parameter;
+
public class ListInstances {
private static final Logger log = Logger.getLogger(ListInstances.class);
@@ -40,26 +42,26 @@ public class ListInstances {
private static final int NAME_WIDTH = 20;
private static final int UUID_WIDTH = 37;
private static final int MASTER_WIDTH = 30;
- private static String zooKeepers;
- private static boolean printErrors;
- private static boolean printAll;
- private static int errors = 0;
+
+ static class Opts extends Help {
+ @Parameter(names="--print-errors", description="display errors while listing instances")
+ boolean printErrors = false;
+ @Parameter(names="--print-all", description="print information for all instances, not just those with names")
+ boolean printAll = false;
+ @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact")
+ String keepers = null;
+ }
+ static Opts opts = new Opts();
+ static int errors = 0;
public static void main(String[] args) {
+ opts.parseArgs(ListInstances.class.getName(), args);
- args = processOptions(args);
-
- if (args.length > 1) {
- System.err.println("Usage : " + ListInstances.class.getName() + " [<zoo keepers>]");
- }
-
- if (args.length == 1) {
- zooKeepers = args[0];
- } else {
- zooKeepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
+ if (opts.keepers == null) {
+ opts.keepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
}
- System.out.println("INFO : Using ZooKeepers " + zooKeepers);
+ System.out.println("INFO : Using ZooKeepers " + opts.keepers);
TreeMap<String,UUID> instanceNames = getInstanceNames();
@@ -73,7 +75,7 @@ public class ListInstances {
TreeSet<UUID> instancedIds = getInstanceIDs();
instancedIds.removeAll(instanceNames.values());
- if (printAll) {
+ if (opts.printAll) {
for (UUID uuid : instancedIds) {
printInstanceInfo(null, uuid);
}
@@ -84,28 +86,12 @@ public class ListInstances {
System.out.println();
}
- if (!printErrors && errors > 0) {
+ if (!opts.printErrors && errors > 0) {
System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info");
}
}
- private static String[] processOptions(String[] args) {
- ArrayList<String> al = new ArrayList<String>();
-
- for (String s : args) {
- if (s.equals("--print-errors")) {
- printErrors = true;
- } else if (s.equals("--print-all")) {
- printAll = true;
- } else {
- al.add(s);
- }
- }
-
- return al.toArray(new String[al.size()]);
- }
-
private static class CharFiller implements Formattable {
char c;
@@ -220,7 +206,7 @@ public class ListInstances {
}
private static void handleException(Exception e) {
- if (printErrors) {
+ if (opts.printErrors) {
e.printStackTrace();
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java Wed Dec 19 16:25:03 2012
@@ -23,9 +23,9 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Value;
@@ -39,13 +39,11 @@ import org.apache.hadoop.fs.Path;
public class LocalityCheck {
public int run(String[] args) throws Exception {
- if (args.length < 4) {
- System.err.println("Usage: " + LocalityCheck.class.getName() + " instance zookeepers username password");
- System.exit(1);
- }
- ZooKeeperInstance instance = new ZooKeeperInstance(args[0], args[1]);
+ ClientOpts opts = new ClientOpts();
+ opts.parseArgs(LocalityCheck.class.getName(), args);
+
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- Connector connector = instance.getConnector(args[2], args[3].getBytes());
+ Connector connector = opts.getConnector();
Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RandomWriter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RandomWriter.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RandomWriter.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RandomWriter.java Wed Dec 19 16:25:03 2012
@@ -19,19 +19,16 @@ package org.apache.accumulo.server.util;
import java.util.Iterator;
import java.util.Random;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.server.cli.ClientOnDefaultTable;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
+import com.beust.jcommander.Parameter;
+
public class RandomWriter {
private static String table_name = "test_write_table";
@@ -78,31 +75,25 @@ public class RandomWriter {
return this;
}
}
-
- public static void main(String[] args) throws MutationsRejectedException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names="--count", description="number of mutations to write", required=true)
+ long count;
+
+ Opts(String table) { super(table); }
+ }
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts(table_name);
+ opts.parseArgs(RandomWriter.class.getName(), args);
+
long start = System.currentTimeMillis();
- if (args.length != 3) {
- log.error("Usage: bin/accumulo " + RandomWriter.class.getName() + " <username> <password> <num_mutations_to_write>");
- return;
- }
- log.info("starting at " + start + " for user " + args[0]);
+ log.info("starting at " + start + " for user " + opts.user);
try {
- Connector connector = HdfsZooInstance.getInstance().getConnector(args[0], args[1].getBytes());
- BatchWriter bw = connector.createBatchWriter(table_name, new BatchWriterConfig());
- long num_mutations = Long.parseLong(args[2]);
- log.info("Writing " + num_mutations + " mutations...");
- bw.addMutations(new RandomMutationGenerator(num_mutations));
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.getTableName(), opts.getBatchWriterConfig());
+ log.info("Writing " + opts.count + " mutations...");
+ bw.addMutations(new RandomMutationGenerator(opts.count));
bw.close();
- } catch (MutationsRejectedException e) {
- log.error(e);
- throw e;
- } catch (AccumuloException e) {
- log.error(e);
- throw e;
- } catch (AccumuloSecurityException e) {
- log.error(e);
- throw e;
- } catch (TableNotFoundException e) {
+ } catch (Exception e) {
log.error(e);
throw e;
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java Wed Dec 19 16:25:03 2012
@@ -19,22 +19,22 @@ package org.apache.accumulo.server.util;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.impl.Writer;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
+import com.beust.jcommander.Parameter;
+
/**
* Remove file entries for map files that don't exist.
*
@@ -42,21 +42,25 @@ import org.apache.log4j.Logger;
public class RemoveEntriesForMissingFiles {
private static Logger log = Logger.getLogger(RemoveEntriesForMissingFiles.class);
+ static class Opts extends ClientOpts {
+ @Parameter(names="--fix")
+ boolean fix = false;
+ }
+
public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(RemoveEntriesForMissingFiles.class.getName(), args);
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- if (args.length < 4) {
- System.err.println("Usage: accumulo.server.util.RemoveEntriesForMissingFiles instance zookeepers username password [delete]");
- System.exit(1);
- }
- Instance instance = new ZooKeeperInstance(args[0], args[1]);
- Connector connector = instance.getConnector(args[2], args[3].getBytes());
- Scanner metadata = connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
- metadata.setBatchSize(1000 * 1000);
+ Connector connector = opts.getConnector();
+ Scanner metadata = connector.createScanner(Constants.METADATA_TABLE_NAME, opts.auths);
+ metadata.setBatchSize(opts.scanBatchSize);
metadata.setRange(Constants.METADATA_KEYSPACE);
metadata.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
int count = 0;
int missing = 0;
- Writer writer = MetadataTable.getMetadataTable(SecurityConstants.getSystemCredentials());
+ BatchWriter writer = null;
+ if (opts.fix)
+ writer = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, opts.getBatchWriterConfig());
for (Entry<Key,Value> entry : metadata) {
count++;
Key key = entry.getKey();
@@ -67,12 +71,14 @@ public class RemoveEntriesForMissingFile
log.info("File " + map + " is missing");
Mutation m = new Mutation(key.getRow());
m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
- if (args.length > 4) {
- writer.update(m);
+ if (writer != null) {
+ writer.addMutation(m);
log.info("entry removed from metadata table: " + m);
}
}
}
+ if (writer != null && missing > 0)
+ writer.close();
log.info(String.format("%d files of %d missing", missing, count));
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java Wed Dec 19 16:25:03 2012
@@ -23,6 +23,7 @@ import java.util.Stack;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.commons.codec.binary.Base64;
@@ -36,6 +37,8 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import com.beust.jcommander.Parameter;
+
public class RestoreZookeeper {
private static class Restore extends DefaultHandler {
@@ -91,32 +94,37 @@ public class RestoreZookeeper {
}
}
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names="--overwrite")
+ boolean overwrite = false;
+ @Parameter(names="--file")
+ String file;
+ }
+
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
- String server = args[0];
int timeout = 30 * 1000;
InputStream in = System.in;
- boolean overwrite = false;
- if (args.length > 1) {
- in = new FileInputStream(args[1]);
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
}
- for (String arg : args)
- if (arg.equals("--overwrite"))
- overwrite = true;
-
- ZooKeeper zk = new ZooKeeper(server, timeout, new Watcher() {
+ ZooKeeper zk = new ZooKeeper(opts.keepers, timeout, new Watcher() {
@Override
public void process(WatchedEvent event) {}
});
SAXParserFactory factory = SAXParserFactory.newInstance();
SAXParser parser = factory.newSAXParser();
- parser.parse(in, new Restore(zk, overwrite));
+ parser.parse(in, new Restore(zk, opts.overwrite));
in.close();
}
}