You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/01/07 19:41:56 UTC
svn commit: r1429942 [1/2] - in /accumulo/trunk:
core/src/main/java/org/apache/accumulo/core/cli/
core/src/main/java/org/apache/accumulo/core/trace/
core/src/test/java/org/apache/accumulo/core/cli/
examples/simple/src/main/java/org/apache/accumulo/exam...
Author: ecn
Date: Mon Jan 7 18:41:55 2013
New Revision: 1429942
URL: http://svn.apache.org/viewvc?rev=1429942&view=rev
Log:
ACCUMULO-745 use scanner and batchwriter options only when they are needed
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/Help.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/trace/TraceDump.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/ContinuousQuery.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java
accumulo/trunk/examples/simple/src/test/java/org/apache/accumulo/examples/simple/dirlist/CountTest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/fate/Admin.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/CreateTestTable.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/GCLotsOfCandidatesTest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/QueryMetadataTable.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestBinaryRows.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestMultiTableIngest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestRandomDeletes.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/VerifyIngest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousQuery.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousStatsCollector.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RandomWriter.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java Mon Jan 7 18:41:55 2013
@@ -21,13 +21,11 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import org.apache.accumulo.cloudtrace.instrument.Trace;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
@@ -94,8 +92,6 @@ public class ClientOpts extends Help {
}
}
- private static final BatchWriterConfig BWDEFAULTS = new BatchWriterConfig();
-
@Parameter(names={"-u", "--user"}, description = "Connection user")
public String user = System.getProperty("user.name");
@@ -121,24 +117,6 @@ public class ClientOpts extends Help {
@Parameter(names={"-auths", "--auths"}, converter=AuthConverter.class, description="the authorizations to use when reading or writing")
public Authorizations auths = Constants.NO_AUTHS;
- @Parameter(names="--batchThreads", description="Number of threads to use when writing large batches")
- public Integer batchThreads = BWDEFAULTS.getMaxWriteThreads();
-
- @Parameter(names="--batchLatency", converter=TimeConverter.class, description="The maximum time to wait before flushing data to servers when writing")
- public Long batchLatency = BWDEFAULTS.getMaxLatency(TimeUnit.MILLISECONDS);
-
- @Parameter(names="--batchMemory", converter=MemoryConverter.class, description="memory used to batch data when writing")
- public Long batchMemory = BWDEFAULTS.getMaxMemory();
-
- @Parameter(names="--batchTimeout", converter=TimeConverter.class, description="timeout used to fail a batch write")
- public Long batchTimeout = BWDEFAULTS.getTimeout(TimeUnit.MILLISECONDS);
-
- @Parameter(names="--scanBatchSize", description="the number of key-values to pull during a scan or batch scan")
- public int scanBatchSize = 1000;
-
- @Parameter(names="--scanThreads", description="number of threads to use when batch scanning")
- public Integer scanThreads = 10;
-
@Parameter(names="--debug", description="turn on TRACE-level log messages")
public boolean debug = false;
@@ -166,20 +144,12 @@ public class ClientOpts extends Help {
Trace.off();
}
- public void parseArgs(String programName, String[] args) {
- super.parseArgs(programName, args);
+ public void parseArgs(String programName, String[] args, Object ... others) {
+ super.parseArgs(programName, args, others);
startDebugLogging();
startTracing(programName);
}
- public BatchWriterConfig getBatchWriterConfig() {
- BatchWriterConfig config = new BatchWriterConfig();
- config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS);
- config.setMaxMemory(this.batchMemory);
- config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS);
- return config;
- }
-
protected Instance cachedInstance = null;
@SuppressWarnings("deprecation")
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/Help.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/Help.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/Help.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/Help.java Mon Jan 7 18:41:55 2013
@@ -23,8 +23,12 @@ import com.beust.jcommander.ParameterExc
public class Help {
@Parameter(names={"-h", "-?", "--help", "-help"}, help=true)
public boolean help = false;
- public void parseArgs(String programName, String[] args) {
- JCommander commander = new JCommander(this);
+
+ public void parseArgs(String programName, String[] args, Object ... others) {
+ JCommander commander = new JCommander();
+ commander.addObject(this);
+ for (Object other : others)
+ commander.addObject(other);
commander.setProgramName(programName);
try {
commander.parse(args);
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/trace/TraceDump.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/trace/TraceDump.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/trace/TraceDump.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/trace/TraceDump.java Mon Jan 7 18:41:55 2013
@@ -30,6 +30,7 @@ import java.util.Set;
import org.apache.accumulo.cloudtrace.thrift.RemoteSpan;
import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
@@ -61,13 +62,14 @@ public class TraceDump {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(TraceDump.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(TraceDump.class.getName(), args, scanOpts);
int code = 0;
if (opts.list) {
- code = listSpans(opts);
+ code = listSpans(opts, scanOpts);
}
if (code == 0 && opts.dump) {
- code = dumpTrace(opts);
+ code = dumpTrace(opts, scanOpts);
}
System.exit(code);
}
@@ -83,13 +85,13 @@ public class TraceDump {
return spanList;
}
- private static int listSpans(Opts opts) throws Exception {
+ private static int listSpans(Opts opts, ScannerOpts scanOpts) throws Exception {
PrintStream out = System.out;
long endTime = System.currentTimeMillis();
long startTime = endTime - DEFAULT_TIME_IN_MILLIS;
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
Range range = new Range(new Text("start:" + Long.toHexString(startTime)), new Text("start:" + Long.toHexString(endTime)));
scanner.setRange(range);
out.println("Trace Day/Time (ms) Start");
@@ -104,14 +106,14 @@ public class TraceDump {
void print(String line);
}
- private static int dumpTrace(Opts opts) throws Exception {
+ private static int dumpTrace(Opts opts, ScannerOpts scanOpts) throws Exception {
final PrintStream out = System.out;
Connector conn = opts.getConnector();
int count = 0;
for (String traceId : opts.traceIds) {
Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
Range range = new Range(new Text(traceId.toString()));
scanner.setRange(range);
count = printTrace(scanner, new Printer() {
Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java (original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java Mon Jan 7 18:41:55 2013
@@ -16,7 +16,11 @@
*/
package org.apache.accumulo.core.cli;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.util.concurrent.TimeUnit;
@@ -35,21 +39,26 @@ public class TestClientOpts {
// document the defaults
ClientOpts args = new ClientOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
assertEquals(System.getProperty("user.name"), args.user);
assertNull(args.securePassword);
assertArrayEquals("secret".getBytes(), args.getPassword());
- assertEquals(new Long(cfg.getMaxLatency(TimeUnit.MILLISECONDS)), args.batchLatency);
- assertEquals(new Long(cfg.getTimeout(TimeUnit.MILLISECONDS)), args.batchTimeout);
- assertEquals(new Long(cfg.getMaxMemory()), args.batchMemory);
+ assertEquals(new Long(cfg.getMaxLatency(TimeUnit.MILLISECONDS)), bwOpts.batchLatency);
+ assertEquals(new Long(cfg.getTimeout(TimeUnit.MILLISECONDS)), bwOpts.batchTimeout);
+ assertEquals(new Long(cfg.getMaxMemory()), bwOpts.batchMemory);
assertFalse(args.debug);
assertFalse(args.trace);
- assertEquals(10, args.scanThreads.intValue());
+ assertEquals(10, bsOpts.scanThreads.intValue());
assertEquals(null, args.instance);
assertEquals(Constants.NO_AUTHS, args.auths);
assertEquals("localhost:2181", args.zookeepers);
assertFalse(args.help);
- args = evaluate(
+ JCommander jc = new JCommander();
+ jc.addObject(args);
+ jc.addObject(bwOpts);
+ jc.parse(
"-u", "bar",
"-p", "foo",
"--batchLatency", "3s",
@@ -61,17 +70,16 @@ public class TestClientOpts {
"-i", "instance",
"--auths", "G1,G2,G3",
"-z", "zoohost1,zoohost2",
- "--help"
- );
+ "--help");
assertEquals("bar", args.user);
assertNull(args.securePassword);
assertArrayEquals("foo".getBytes(), args.getPassword());
- assertEquals(new Long(3000), args.batchLatency);
- assertEquals(new Long(2000), args.batchTimeout);
- assertEquals(new Long(1024*1024), args.batchMemory);
+ assertEquals(new Long(3000), bwOpts.batchLatency);
+ assertEquals(new Long(2000), bwOpts.batchTimeout);
+ assertEquals(new Long(1024*1024), bwOpts.batchMemory);
assertTrue(args.debug);
assertTrue(args.trace);
- assertEquals(7, args.scanThreads.intValue());
+ assertEquals(7, bsOpts.scanThreads.intValue());
assertEquals("instance", args.instance);
assertEquals(new Authorizations("G1", "G2", "G3"), args.auths);
assertEquals("zoohost1,zoohost2", args.zookeepers);
@@ -79,10 +87,4 @@ public class TestClientOpts {
}
- private ClientOpts evaluate(String ... args) {
- ClientOpts ca = new ClientOpts();
- new JCommander(ca, args);
- return ca;
- }
-
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java Mon Jan 7 18:41:55 2013
@@ -20,8 +20,10 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
import java.util.Random;
+import org.apache.accumulo.core.cli.BatchScannerOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -190,10 +192,12 @@ public class RandomBatchScanner {
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Opts opts = new Opts();
- opts.parseArgs(RandomBatchScanner.class.getName(), args);
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
Connector connector = opts.getConnector();
- BatchScanner tsbr = connector.createBatchScanner(opts.tableName, opts.auths, opts.scanThreads);
+ BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
Random r;
if (opts.seed == null)
@@ -202,7 +206,7 @@ public class RandomBatchScanner {
r = new Random(opts.seed);
// do one cold
- doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, tsbr);
+ doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
System.gc();
System.gc();
@@ -214,8 +218,8 @@ public class RandomBatchScanner {
r = new Random(opts.seed);
// do one hot (connections already established, metadata table cached)
- doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, tsbr);
+ doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
- tsbr.close();
+ batchReader.close();
}
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java Mon Jan 7 18:41:55 2013
@@ -19,6 +19,7 @@ package org.apache.accumulo.examples.sim
import java.util.HashSet;
import java.util.Random;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -114,7 +115,8 @@ public class RandomBatchWriter {
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Opts opts = new Opts();
- opts.parseArgs(RandomBatchWriter.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
Random r;
if (opts.seed == null)
@@ -124,7 +126,7 @@ public class RandomBatchWriter {
}
Connector connector = opts.getConnector();
- BatchWriter bw = connector.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
// reuse the ColumnVisibility object to improve performance
ColumnVisibility cv = opts.visiblity;
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java Mon Jan 7 18:41:55 2013
@@ -21,8 +21,7 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.accumulo.core.cli.ClientOnDefaultTable;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
@@ -32,7 +31,6 @@ import org.apache.accumulo.core.data.Mut
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.commons.cli.ParseException;
import org.apache.hadoop.io.Text;
import com.beust.jcommander.Parameter;
@@ -44,7 +42,7 @@ public class ReadWriteExample {
private Connector conn;
- class Opts extends ClientOnDefaultTable {
+ static class Opts extends ClientOnDefaultTable {
@Parameter(names={"-C", "--createtable"}, description="create table before doing anything")
boolean createtable = false;
@Parameter(names={"-D", "--deletetable"}, description="delete table when finished")
@@ -65,16 +63,8 @@ public class ReadWriteExample {
// hidden constructor
private ReadWriteExample() {}
- // setup
- private Opts configure(String[] args) throws ParseException, AccumuloException, AccumuloSecurityException {
- Opts opts = new Opts();
- opts.parseArgs(ReadWriteExample.class.getName(), args);
-
+ private void execute(Opts opts, ScannerOpts scanOpts) throws Exception {
conn = opts.getConnector();
- return opts;
- }
-
- private void execute(Opts opts) throws Exception {
// create table
if (opts.createtable) {
SortedSet<Text> partitionKeys = new TreeSet<Text>();
@@ -92,7 +82,7 @@ public class ReadWriteExample {
// Note that the user needs to have the authorizations for the specified scan authorizations
// by an administrator first
Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
for (Entry<Key,Value> entry : scanner)
System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
}
@@ -130,8 +120,9 @@ public class ReadWriteExample {
public static void main(String[] args) throws Exception {
ReadWriteExample rwe = new ReadWriteExample();
-
- Opts opts = rwe.configure(args);
- rwe.execute(opts);
+ Opts opts = new Opts();
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(ReadWriteExample.class.getName(), args, scanOpts);
+ rwe.execute(opts, scanOpts);
}
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java Mon Jan 7 18:41:55 2013
@@ -20,7 +20,9 @@ import java.io.IOException;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
@@ -51,7 +53,9 @@ public class RowOperations {
MutationsRejectedException {
ClientOpts opts = new ClientOpts();
- opts.parseArgs(RowOperations.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(RowOperations.class.getName(), args, scanOpts, bwOpts);
// First the setup work
connector = opts.getConnector();
@@ -92,7 +96,7 @@ public class RowOperations {
mut3.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
// Now we'll make a Batch Writer
- bw = connector.createBatchWriter(table, opts.getBatchWriterConfig());
+ bw = connector.createBatchWriter(table, bwOpts.getBatchWriterConfig());
// And add the mutations
bw.addMutation(mut1);
@@ -103,9 +107,9 @@ public class RowOperations {
bw.flush();
// Now lets look at the rows
- Scanner rowThree = getRow(opts, new Text("row3"));
- Scanner rowTwo = getRow(opts, new Text("row2"));
- Scanner rowOne = getRow(opts, new Text("row1"));
+ Scanner rowThree = getRow(opts, scanOpts, new Text("row3"));
+ Scanner rowTwo = getRow(opts, scanOpts, new Text("row2"));
+ Scanner rowOne = getRow(opts, scanOpts, new Text("row1"));
// And print them
log.info("This is everything");
@@ -115,13 +119,13 @@ public class RowOperations {
System.out.flush();
// Now lets delete rowTwo with the iterator
- rowTwo = getRow(opts, new Text("row2"));
+ rowTwo = getRow(opts, scanOpts, new Text("row2"));
deleteRow(rowTwo);
// Now lets look at the rows again
- rowThree = getRow(opts, new Text("row3"));
- rowTwo = getRow(opts, new Text("row2"));
- rowOne = getRow(opts, new Text("row1"));
+ rowThree = getRow(opts, scanOpts, new Text("row3"));
+ rowTwo = getRow(opts, scanOpts, new Text("row2"));
+ rowOne = getRow(opts, scanOpts, new Text("row1"));
// And print them
log.info("This is row1 and row3");
@@ -133,12 +137,12 @@ public class RowOperations {
// Should only see the two rows
// Now lets delete rowOne without passing in the iterator
- deleteRow(opts, row1);
+ deleteRow(opts, scanOpts, row1);
// Now lets look at the rows one last time
- rowThree = getRow(opts, new Text("row3"));
- rowTwo = getRow(opts, new Text("row2"));
- rowOne = getRow(opts, new Text("row1"));
+ rowThree = getRow(opts, scanOpts, new Text("row3"));
+ rowTwo = getRow(opts, scanOpts, new Text("row2"));
+ rowOne = getRow(opts, scanOpts, new Text("row1"));
// And print them
log.info("This is just row3");
@@ -169,8 +173,8 @@ public class RowOperations {
* @throws AccumuloSecurityException
* @throws AccumuloException
*/
- private static void deleteRow(ClientOpts opts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- deleteRow(getRow(opts, row));
+ private static void deleteRow(ClientOpts opts, ScannerOpts scanOpts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ deleteRow(getRow(opts, scanOpts, row));
}
/**
@@ -214,10 +218,10 @@ public class RowOperations {
* @throws AccumuloException
* @throws IOException
*/
- private static Scanner getRow(ClientOpts opts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ private static Scanner getRow(ClientOpts opts, ScannerOpts scanOpts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
// Create a scanner
Scanner scanner = connector.createScanner(table, Constants.NO_AUTHS);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
// Say start key is the one with key of row
// and end key is the one that immediately follows the row
scanner.setRange(new Range(row));
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java Mon Jan 7 18:41:55 2013
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.examples.simple.client;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -55,9 +56,10 @@ public class SequentialBatchWriter {
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
Opts opts = new Opts();
- opts.parseArgs(SequentialBatchWriter.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts);
Connector connector = opts.getConnector();
- BatchWriter bw = connector.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
long end = opts.start + opts.num;
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java Mon Jan 7 18:41:55 2013
@@ -19,7 +19,9 @@ package org.apache.accumulo.examples.sim
import java.util.Iterator;
import java.util.Map.Entry;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@ -41,6 +43,9 @@ public class FileCount {
private int inserts;
private Opts opts;
+ private ScannerOpts scanOpts;
+ private BatchWriterOpts bwOpts;
+
private static class CountValue {
int dirCount = 0;
@@ -229,8 +234,10 @@ public class FileCount {
}
}
- FileCount(Opts opts) throws Exception {
+ FileCount(Opts opts, ScannerOpts scanOpts, BatchWriterOpts bwOpts) throws Exception {
this.opts = opts;
+ this.scanOpts = scanOpts;
+ this.bwOpts = bwOpts;
}
public void run() throws Exception {
@@ -240,8 +247,8 @@ public class FileCount {
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(opts.tableName, opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
- BatchWriter bw = conn.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
+ scanner.setBatchSize(scanOpts.scanBatchSize);
+ BatchWriter bw = conn.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
long t1 = System.currentTimeMillis();
@@ -273,10 +280,12 @@ public class FileCount {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
String programName = FileCount.class.getName();
- opts.parseArgs(programName, args);
+ opts.parseArgs(programName, args, scanOpts, bwOpts);
- FileCount fileCount = new FileCount(opts);
+ FileCount fileCount = new FileCount(opts, scanOpts, bwOpts);
fileCount.run();
}
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java Mon Jan 7 18:41:55 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
@@ -137,7 +138,8 @@ public class Ingest {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(Ingest.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(Ingest.class.getName(), args, bwOpts);
Connector conn = opts.getConnector();
if (!conn.tableOperations().exists(opts.nameTable))
@@ -149,9 +151,9 @@ public class Ingest {
conn.tableOperations().attachIterator(opts.dataTable, new IteratorSetting(1, ChunkCombiner.class));
}
- BatchWriter dirBW = conn.createBatchWriter(opts.nameTable, opts.getBatchWriterConfig());
- BatchWriter indexBW = conn.createBatchWriter(opts.indexTable, opts.getBatchWriterConfig());
- BatchWriter dataBW = conn.createBatchWriter(opts.dataTable, opts.getBatchWriterConfig());
+ BatchWriter dirBW = conn.createBatchWriter(opts.nameTable, bwOpts.getBatchWriterConfig());
+ BatchWriter indexBW = conn.createBatchWriter(opts.indexTable, bwOpts.getBatchWriterConfig());
+ BatchWriter dataBW = conn.createBatchWriter(opts.dataTable, bwOpts.getBatchWriterConfig());
FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
for (String dir : opts.directories) {
recurse(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java Mon Jan 7 18:41:55 2013
@@ -24,6 +24,7 @@ import java.security.NoSuchAlgorithmExce
import java.util.ArrayList;
import java.util.List;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
@@ -183,14 +184,15 @@ public class FileDataIngest {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(FileDataIngest.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);
Connector conn = opts.getConnector();
if (!conn.tableOperations().exists(opts.tableName)) {
conn.tableOperations().create(opts.tableName);
conn.tableOperations().attachIterator(opts.tableName, new IteratorSetting(1, ChunkCombiner.class));
}
- BatchWriter bw = conn.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
+ BatchWriter bw = conn.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
for (String filename : opts.files) {
fdi.insertFileData(filename, bw);
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java Mon Jan 7 18:41:55 2013
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.examples.simple.helloworld;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -37,10 +38,11 @@ public class InsertWithBatchWriter {
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, MutationsRejectedException, TableExistsException,
TableNotFoundException {
ClientOnRequiredTable opts = new ClientOnRequiredTable();
- opts.parseArgs(InsertWithBatchWriter.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(InsertWithBatchWriter.class.getName(), args, bwOpts);
Connector connector = opts.getConnector();
- MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(opts.getBatchWriterConfig());
+ MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
if (!connector.tableOperations().exists(opts.tableName))
connector.tableOperations().create(opts.tableName);
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java Mon Jan 7 18:41:55 2013
@@ -20,6 +20,7 @@ import java.util.Iterator;
import java.util.Map.Entry;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -46,12 +47,13 @@ public class ReadData {
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Opts opts = new Opts();
- opts.parseArgs(ReadData.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(ReadData.class.getName(), args, scanOpts);
Connector connector = opts.getConnector();
Scanner scan = connector.createScanner(opts.tableName, opts.auths);
- scan.setBatchSize(opts.scanBatchSize);
+ scan.setBatchSize(scanOpts.scanBatchSize);
Key start = null;
if (opts.startKey != null)
start = new Key(new Text(opts.startKey));
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java Mon Jan 7 18:41:55 2013
@@ -19,6 +19,7 @@ package org.apache.accumulo.examples.sim
import java.util.HashSet;
import java.util.Map.Entry;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
@@ -152,7 +153,8 @@ public class InterferenceTest {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(InterferenceTest.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(InterferenceTest.class.getName(), args, bwOpts);
if (opts.iterations < 1)
opts.iterations = Long.MAX_VALUE;
@@ -161,7 +163,7 @@ public class InterferenceTest {
if (!conn.tableOperations().exists(opts.tableName))
conn.tableOperations().create(opts.tableName);
- Thread writer = new Thread(new Writer(conn.createBatchWriter(opts.tableName, opts.getBatchWriterConfig()), opts.iterations));
+ Thread writer = new Thread(new Writer(conn.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig()), opts.iterations));
writer.start();
Reader r;
if (opts.isolated)
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/ContinuousQuery.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/ContinuousQuery.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/ContinuousQuery.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/ContinuousQuery.java Mon Jan 7 18:41:55 2013
@@ -20,8 +20,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
import java.util.Random;
+import org.apache.accumulo.core.cli.BatchScannerOpts;
import org.apache.accumulo.core.cli.ClientOpts;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
@@ -57,7 +59,8 @@ public class ContinuousQuery {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(ContinuousQuery.class.getName(), args);
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(ContinuousQuery.class.getName(), args, bsOpts);
Connector conn = opts.getConnector();
@@ -65,7 +68,8 @@ public class ContinuousQuery {
Random rand = new Random();
- BatchScanner bs = conn.createBatchScanner(opts.table, opts.auths, opts.scanThreads);
+ BatchScanner bs = conn.createBatchScanner(opts.table, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
for (long i = 0; i < opts.iterations; i += 1) {
Text[] columns = randTerms.get(rand.nextInt(randTerms.size()));
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java Mon Jan 7 18:41:55 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.data.Mutation;
@@ -100,11 +101,12 @@ public class Index {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(Index.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(Index.class.getName(), args, bwOpts);
String splitRegex = "\\W+";
- BatchWriter bw = opts.getConnector().createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
+ BatchWriter bw = opts.getConnector().createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
for (String filename : opts.files) {
index(opts.partitions, new File(filename), splitRegex, bw);
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java Mon Jan 7 18:41:55 2013
@@ -20,7 +20,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.cli.BatchScannerOpts;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
@@ -51,10 +53,12 @@ public class Query {
*/
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(Query.class.getName(), args);
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(Query.class.getName(), args, bsOpts);
Connector conn = opts.getConnector();
- BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, opts.batchThreads);
+ BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
Text columns[] = new Text[opts.terms.size()];
int i = 0;
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java Mon Jan 7 18:41:55 2013
@@ -18,7 +18,9 @@ package org.apache.accumulo.examples.sim
import java.util.Map.Entry;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@ -47,13 +49,15 @@ public class Reverse {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(Reverse.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(Reverse.class.getName(), args, scanOpts, bwOpts);
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(opts.shardTable, opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
- BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, opts.getBatchWriterConfig());
+ scanner.setBatchSize(scanOpts.scanBatchSize);
+ BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, bwOpts.getBatchWriterConfig());
for (Entry<Key,Value> entry : scanner) {
Key key = entry.getKey();
Modified: accumulo/trunk/examples/simple/src/test/java/org/apache/accumulo/examples/simple/dirlist/CountTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/test/java/org/apache/accumulo/examples/simple/dirlist/CountTest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/test/java/org/apache/accumulo/examples/simple/dirlist/CountTest.java (original)
+++ accumulo/trunk/examples/simple/src/test/java/org/apache/accumulo/examples/simple/dirlist/CountTest.java Mon Jan 7 18:41:55 2013
@@ -21,6 +21,8 @@ import java.util.Map.Entry;
import junit.framework.TestCase;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
@@ -63,10 +65,12 @@ public class CountTest extends TestCase
assertFalse(scanner.iterator().hasNext());
Opts opts = new Opts();
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
opts.instance = "counttest";
opts.tableName = "dirlisttable";
opts.mock = true;
- FileCount fc = new FileCount(opts);
+ FileCount fc = new FileCount(opts, scanOpts, bwOpts);
fc.run();
ArrayList<Pair<String,String>> expected = new ArrayList<Pair<String,String>>();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/fate/Admin.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/fate/Admin.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/fate/Admin.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/fate/Admin.java Mon Jan 7 18:41:55 2013
@@ -20,12 +20,14 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.AdminUtil;
import org.apache.accumulo.fate.ZooStore;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -57,7 +59,7 @@ public class Admin {
}
public static void main(String[] args) throws Exception {
- ClientOpts opts = new ClientOpts();
+ Help opts = new Help();
JCommander jc = new JCommander(opts);
jc.setProgramName(Admin.class.getName());
FailOpts fail = new FailOpts();
@@ -73,7 +75,7 @@ public class Admin {
AdminUtil<Master> admin = new AdminUtil<Master>();
- Instance instance = opts.getInstance();
+ Instance instance = HdfsZooInstance.getInstance();
String path = ZooUtil.getRoot(instance) + Constants.ZFATE;
String masterPath = ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK;
IZooReaderWriter zk = ZooReaderWriter.getRetryingInstance();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/CreateTestTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/CreateTestTable.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/CreateTestTable.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/CreateTestTable.java Mon Jan 7 18:41:55 2013
@@ -20,6 +20,8 @@ import java.util.Map.Entry;
import java.util.TreeSet;
import org.apache.accumulo.server.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@ -40,9 +42,9 @@ public class CreateTestTable {
Opts() { super("mrtest1"); }
}
- private static void readBack(Connector conn, Opts opts) throws Exception {
+ private static void readBack(Connector conn, Opts opts, ScannerOpts scanOpts) throws Exception {
Scanner scanner = conn.createScanner("mrtest1", opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
int count = 0;
for (Entry<Key,Value> elt : scanner) {
String expected = String.format("%05d", count);
@@ -55,7 +57,9 @@ public class CreateTestTable {
public static void main(String[] args) throws Exception {
String program = CreateTestTable.class.getName();
Opts opts = new Opts();
- opts.parseArgs(program, args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(program, args, bwOpts, scanOpts);
// create the test table within accumulo
Connector connector = opts.getConnector();
@@ -69,7 +73,7 @@ public class CreateTestTable {
// presplit
connector.tableOperations().create(opts.getTableName());
connector.tableOperations().addSplits(opts.getTableName(), keys);
- BatchWriter b = connector.createBatchWriter(opts.getTableName(), opts.getBatchWriterConfig());
+ BatchWriter b = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
// populate
for (int i = 0; i < opts.count; i++) {
@@ -80,7 +84,7 @@ public class CreateTestTable {
b.close();
}
- readBack(connector, opts);
+ readBack(connector, opts, scanOpts);
opts.stopTracing();
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/GCLotsOfCandidatesTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/GCLotsOfCandidatesTest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/GCLotsOfCandidatesTest.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/GCLotsOfCandidatesTest.java Mon Jan 7 18:41:55 2013
@@ -18,6 +18,7 @@ package org.apache.accumulo.server.test;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
@@ -32,11 +33,12 @@ import org.apache.hadoop.io.Text;
public class GCLotsOfCandidatesTest {
public static void main(String args[]) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
ClientOpts opts = new ClientOpts();
- opts.parseArgs(GCLotsOfCandidatesTest.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(GCLotsOfCandidatesTest.class.getName(), args, bwOpts);
Connector conn = opts.getConnector();
conn.securityOperations().grantTablePermission(conn.whoami(), Constants.METADATA_TABLE_NAME, TablePermission.WRITE);
- BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, opts.getBatchWriterConfig());
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, bwOpts.getBatchWriterConfig());
for (int i = 0; i < 10000; ++i) {
final Text emptyText = new Text("");
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/QueryMetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/QueryMetadataTable.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/QueryMetadataTable.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/QueryMetadataTable.java Mon Jan 7 18:41:55 2013
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -90,11 +91,12 @@ public class QueryMetadataTable {
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Opts opts = new Opts();
- opts.parseArgs(QueryMetadataTable.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(QueryMetadataTable.class.getName(), args, scanOpts);
Connector connector = opts.getConnector();
Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
Text mdrow = new Text(KeyExtent.getMetadataEntry(new Text(Constants.METADATA_TABLE_ID), null));
HashSet<Text> rowSet = new HashSet<Text>();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestBinaryRows.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestBinaryRows.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestBinaryRows.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestBinaryRows.java Mon Jan 7 18:41:55 2013
@@ -22,6 +22,8 @@ import java.util.Random;
import java.util.TreeSet;
import org.apache.accumulo.server.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@ -79,13 +81,15 @@ public class TestBinaryRows {
public static void main(String[] args) {
Opts opts = new Opts();
- opts.parseArgs(TestBinaryRows.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
try {
Connector connector = opts.getConnector();
if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
- BatchWriter bw = connector.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
boolean delete = opts.mode.equals("delete");
for (long i = 0; i < opts.num; i++) {
@@ -104,7 +108,7 @@ public class TestBinaryRows {
bw.close();
} else if (opts.mode.equals("verifyDeleted")) {
Scanner s = connector.createScanner(opts.tableName, opts.auths);
- s.setBatchSize(opts.scanBatchSize);
+ s.setBatchSize(scanOpts.scanBatchSize);
Key startKey = new Key(encodeLong(opts.start), "cf".getBytes(), "cq".getBytes(), new byte[0], Long.MAX_VALUE);
Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), "cf".getBytes(), "cq".getBytes(), new byte[0], 0);
s.setBatchSize(50000);
@@ -122,7 +126,7 @@ public class TestBinaryRows {
Scanner s = connector.createScanner(opts.tableName, opts.auths);
Key startKey = new Key(encodeLong(opts.start), "cf".getBytes(), "cq".getBytes(), new byte[0], Long.MAX_VALUE);
Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), "cf".getBytes(), "cq".getBytes(), new byte[0], 0);
- s.setBatchSize(opts.scanBatchSize);
+ s.setBatchSize(scanOpts.scanBatchSize);
s.setRange(new Range(startKey, stopKey));
long i = opts.start;
@@ -161,7 +165,7 @@ public class TestBinaryRows {
long row = (Math.abs(r.nextLong()) % opts.num) + opts.start;
Scanner s = connector.createScanner(opts.tableName, opts.auths);
- s.setBatchSize(opts.scanBatchSize);
+ s.setBatchSize(scanOpts.scanBatchSize);
Key startKey = new Key(encodeLong(row), "cf".getBytes(), "cq".getBytes(), new byte[0], Long.MAX_VALUE);
Key stopKey = new Key(encodeLong(row), "cf".getBytes(), "cq".getBytes(), new byte[0], 0);
s.setRange(new Range(startKey, stopKey));
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestMultiTableIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestMultiTableIngest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestMultiTableIngest.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestMultiTableIngest.java Mon Jan 7 18:41:55 2013
@@ -19,7 +19,8 @@ package org.apache.accumulo.server.test;
import java.util.ArrayList;
import java.util.Map.Entry;
-import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -29,6 +30,7 @@ import org.apache.accumulo.core.client.S
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.hadoop.io.Text;
import com.beust.jcommander.Parameter;
@@ -46,11 +48,11 @@ public class TestMultiTableIngest {
int count = 10000;
}
- private static void readBack(Opts opts, Connector conn) throws Exception {
+ private static void readBack(Opts opts, ScannerOpts scanOpts, Connector conn) throws Exception {
int i = 0;
for (String table : tableNames) {
Scanner scanner = conn.createScanner(table, opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
int count = i;
for (Entry<Key,Value> elt : scanner) {
String expected = String.format("%05d", count);
@@ -64,7 +66,9 @@ public class TestMultiTableIngest {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(TestMultiTableIngest.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(TestMultiTableIngest.class.getName(), args, scanOpts, bwOpts);
// create the test table within accumulo
Connector connector;
try {
@@ -84,7 +88,7 @@ public class TestMultiTableIngest {
MultiTableBatchWriter b;
try {
- b = connector.createMultiTableBatchWriter(opts.getBatchWriterConfig());
+ b = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -102,7 +106,7 @@ public class TestMultiTableIngest {
}
}
try {
- readBack(opts, connector);
+ readBack(opts, scanOpts, connector);
} catch (Exception e) {
throw new RuntimeException(e);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestRandomDeletes.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestRandomDeletes.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestRandomDeletes.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/TestRandomDeletes.java Mon Jan 7 18:41:55 2013
@@ -22,6 +22,8 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@ -62,11 +64,11 @@ public class TestRandomDeletes {
}
}
- private static TreeSet<RowColumn> scanAll(ClientOpts opts, Text t) throws Exception {
+ private static TreeSet<RowColumn> scanAll(ClientOpts opts, ScannerOpts scanOpts, Text t) throws Exception {
TreeSet<RowColumn> result = new TreeSet<RowColumn>();
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(t.toString(), auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
for (Entry<Key,Value> entry : scanner) {
Key key = entry.getKey();
Column column = new Column(TextUtil.getBytes(key.getColumnFamily()), TextUtil.getBytes(key.getColumnQualifier()), TextUtil.getBytes(key
@@ -76,13 +78,13 @@ public class TestRandomDeletes {
return result;
}
- private static long scrambleDeleteHalfAndCheck(ClientOpts opts, Text t, Set<RowColumn> rows) throws Exception {
+ private static long scrambleDeleteHalfAndCheck(ClientOpts opts, ScannerOpts scanOpts, BatchWriterOpts bwOpts, Text t, Set<RowColumn> rows) throws Exception {
int result = 0;
ArrayList<RowColumn> entries = new ArrayList<RowColumn>(rows);
java.util.Collections.shuffle(entries);
Connector connector = opts.getConnector();
- BatchWriter mutations = connector.createBatchWriter(t.toString(), opts.getBatchWriterConfig());
+ BatchWriter mutations = connector.createBatchWriter(t.toString(), bwOpts.getBatchWriterConfig());
for (int i = 0; i < (entries.size() + 1) / 2; i++) {
RowColumn rc = entries.get(i);
@@ -95,7 +97,7 @@ public class TestRandomDeletes {
mutations.close();
- Set<RowColumn> current = scanAll(opts, t);
+ Set<RowColumn> current = scanAll(opts, scanOpts, t);
current.removeAll(rows);
if (current.size() > 0) {
throw new RuntimeException(current.size() + " records not deleted");
@@ -106,19 +108,21 @@ public class TestRandomDeletes {
static public void main(String[] args) {
ClientOpts opts = new ClientOpts();
- opts.parseArgs(TestRandomDeletes.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(TestRandomDeletes.class.getName(), args, scanOpts, bwOpts);
try {
long deleted = 0;
Text t = new Text("test_ingest");
- TreeSet<RowColumn> doomed = scanAll(opts, t);
+ TreeSet<RowColumn> doomed = scanAll(opts, scanOpts, t);
log.info("Got " + doomed.size() + " rows");
long startTime = System.currentTimeMillis();
while (true) {
- long half = scrambleDeleteHalfAndCheck(opts, t, doomed);
+ long half = scrambleDeleteHalfAndCheck(opts, scanOpts, bwOpts, t, doomed);
deleted += half;
if (half == 0)
break;
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/VerifyIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/VerifyIngest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/VerifyIngest.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/VerifyIngest.java Mon Jan 7 18:41:55 2013
@@ -22,6 +22,7 @@ import java.util.Map.Entry;
import java.util.Random;
import org.apache.accumulo.cloudtrace.instrument.Trace;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
@@ -59,7 +60,8 @@ public class VerifyIngest {
public static void main(String[] args) {
Opts opts = new Opts();
- opts.parseArgs(VerifyIngest.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(VerifyIngest.class.getName(), args, scanOpts);
Instance instance = opts.getInstance();
try {
if (opts.trace) {
@@ -149,7 +151,7 @@ public class VerifyIngest {
Key startKey = new Key(new Text("row_" + String.format("%010d", expectedRow)));
Scanner scanner = connector.createScanner("test_ingest", labelAuths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
scanner.setRange(new Range(startKey, endKey));
for (int j = 0; j < opts.cols; j++) {
scanner.fetchColumn(new Text(opts.columnFamily), new Text("col_" + String.format("%05d", j)));
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java Mon Jan 7 18:41:55 2013
@@ -23,7 +23,10 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@ -47,26 +50,29 @@ public class ContinuousBatchWalker {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(ContinuousBatchWalker.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(ContinuousBatchWalker.class.getName(), args, scanOpts, bsOpts);
Random r = new Random();
Authorizations auths = opts.randomAuths.getAuths(r);
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(opts.getTableName(), auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
- BatchScanner bs = conn.createBatchScanner(opts.getTableName(), auths, opts.scanThreads);
+ BatchScanner bs = conn.createBatchScanner(opts.getTableName(), auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
while (true) {
- Set<Text> batch = getBatch(scanner, opts.min, opts.max, opts.scanBatchSize, r);
+ Set<Text> batch = getBatch(scanner, opts.min, opts.max, scanOpts.scanBatchSize, r);
List<Range> ranges = new ArrayList<Range>(batch.size());
for (Text row : batch) {
ranges.add(new Range(row));
}
- runBatchScan(opts.scanBatchSize, bs, batch, ranges);
+ runBatchScan(scanOpts.scanBatchSize, bs, batch, ranges);
UtilWaitThread.sleep(opts.sleepTime);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java Mon Jan 7 18:41:55 2013
@@ -30,6 +30,7 @@ import java.util.zip.Checksum;
import org.apache.accumulo.cloudtrace.instrument.CountSampler;
import org.apache.accumulo.cloudtrace.instrument.Trace;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ClientOnDefaultTable;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
@@ -137,7 +138,8 @@ public class ContinuousIngest {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(ContinuousIngest.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(ContinuousIngest.class.getName(), args, bwOpts);
initVisibilities(opts);
@@ -151,7 +153,7 @@ public class ContinuousIngest {
conn.tableOperations().create(opts.getTableName());
} catch (TableExistsException tee) {}
- BatchWriter bw = conn.createBatchWriter(opts.getTableName(), opts.getBatchWriterConfig());
+ BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
bw = Trace.wrapAll(bw, new CountSampler(1024));
Random r = new Random();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java Mon Jan 7 18:41:55 2013
@@ -21,6 +21,7 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
import org.apache.accumulo.core.data.Key;
@@ -124,7 +125,8 @@ public class ContinuousMoru extends Conf
@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
Opts opts = new Opts();
- opts.parseArgs(ContinuousMoru.class.getName(), args);
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
@@ -146,9 +148,9 @@ public class ContinuousMoru extends Conf
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setMaxLatency(job.getConfiguration(), (int) (opts.batchLatency / 1000.0));
- AccumuloOutputFormat.setMaxMutationBufferSize(job.getConfiguration(), opts.batchMemory);
- AccumuloOutputFormat.setMaxWriteThreads(job.getConfiguration(), opts.batchThreads);
+ AccumuloOutputFormat.setMaxLatency(job.getConfiguration(), (int) (bwOpts.batchLatency / 1000.0));
+ AccumuloOutputFormat.setMaxMutationBufferSize(job.getConfiguration(), bwOpts.batchMemory);
+ AccumuloOutputFormat.setMaxWriteThreads(job.getConfiguration(), bwOpts.batchThreads);
Configuration conf = job.getConfiguration();
conf.setLong(MIN, opts.min);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousQuery.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousQuery.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousQuery.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousQuery.java Mon Jan 7 18:41:55 2013
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.test.
import java.util.Map.Entry;
import java.util.Random;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
@@ -38,11 +39,12 @@ public class ContinuousQuery {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(ContinuousQuery.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(ContinuousQuery.class.getName(), args, scanOpts);
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
Random r = new Random();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java Mon Jan 7 18:41:55 2013
@@ -20,6 +20,7 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
@@ -41,7 +42,8 @@ public class ContinuousScanner {
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(ContinuousScanner.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(ContinuousScanner.class.getName(), args, scanOpts);
Random r = new Random();
@@ -50,7 +52,7 @@ public class ContinuousScanner {
Connector conn = opts.getConnector();
Authorizations auths = opts.randomAuths.getAuths(r);
Scanner scanner = conn.createScanner(opts.getTableName(), auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanOpts.scanBatchSize);
double delta = Math.min(.05, .05 / (opts.numToScan / 1000.0));
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousStatsCollector.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousStatsCollector.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousStatsCollector.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousStatsCollector.java Mon Jan 7 18:41:55 2013
@@ -25,6 +25,7 @@ import java.util.TimerTask;
import org.apache.accumulo.cloudtrace.instrument.Tracer;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.server.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
@@ -56,10 +57,12 @@ public class ContinuousStatsCollector {
static class StatsCollectionTask extends TimerTask {
private final String tableId;
- private Opts opts;
+ private final Opts opts;
+ private final int scanBatchSize;
- public StatsCollectionTask(Opts opts) {
+ public StatsCollectionTask(Opts opts, int scanBatchSize) {
this.opts = opts;
+ this.scanBatchSize = scanBatchSize;
this.tableId = Tables.getNameToIdMap(opts.getInstance()).get(opts.tableName);
System.out
.println("TIME TABLET_SERVERS TOTAL_ENTRIES TOTAL_INGEST TOTAL_QUERY TABLE_RECS TABLE_RECS_IN_MEM TABLE_INGEST TABLE_QUERY TABLE_TABLETS TABLE_TABLETS_ONLINE"
@@ -86,7 +89,7 @@ public class ContinuousStatsCollector {
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, opts.auths);
- scanner.setBatchSize(opts.scanBatchSize);
+ scanner.setBatchSize(scanBatchSize);
scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
scanner.addScanIterator(new IteratorSetting(1000, "cfc", ColumnFamilyCounter.class.getName()));
scanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
@@ -171,10 +174,11 @@ public class ContinuousStatsCollector {
public static void main(String[] args) {
Opts opts = new Opts();
- opts.parseArgs(ContinuousStatsCollector.class.getName(), args);
+ ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(ContinuousStatsCollector.class.getName(), args, scanOpts);
Timer jtimer = new Timer();
- jtimer.schedule(new StatsCollectionTask(opts), 0, 30000);
+ jtimer.schedule(new StatsCollectionTask(opts, scanOpts.scanBatchSize), 0, 30000);
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java Mon Jan 7 18:41:55 2013
@@ -35,6 +35,7 @@ import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.server.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.cli.BatchScannerOpts;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.impl.Tables;
@@ -248,7 +249,8 @@ public class UndefinedAnalyzer {
*/
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
- opts.parseArgs(UndefinedAnalyzer.class.getName(), args);
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(UndefinedAnalyzer.class.getName(), args, opts);
List<UndefinedNode> undefs = new ArrayList<UndefinedNode>();
@@ -263,8 +265,8 @@ public class UndefinedAnalyzer {
}
Connector conn = opts.getConnector();
- BatchScanner bscanner = conn.createBatchScanner(opts.getTableName(), opts.auths, opts.batchThreads);
- bscanner.setTimeout(opts.batchTimeout, TimeUnit.MILLISECONDS);
+ BatchScanner bscanner = conn.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads);
+ bscanner.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
List<Range> refs = new ArrayList<Range>();
for (UndefinedNode undefinedNode : undefs)
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java?rev=1429942&r1=1429941&r2=1429942&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java Mon Jan 7 18:41:55 2013
@@ -38,7 +38,7 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.server.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
@@ -70,6 +70,7 @@ import org.apache.accumulo.core.util.Cac
import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.Stat;
import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOnRequiredTable;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.hadoop.conf.Configuration;
@@ -99,7 +100,8 @@ public class CollectTabletStats {
public static void main(String[] args) throws Exception {
final CollectOptions opts = new CollectOptions();
- opts.parseArgs(CollectTabletStats.class.getName(), args);
+ final ScannerOpts scanOpts = new ScannerOpts();
+ opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts);
String columnsTmp[] = new String[] {};
if (opts.columns != null)
@@ -214,7 +216,7 @@ public class CollectTabletStats {
for (final KeyExtent ke : tabletsToTest) {
Test test = new Test(ke) {
public int runTest() throws Exception {
- return scanTablet(conn, opts.tableName, opts.auths, opts.scanBatchSize, ke.getPrevEndRow(), ke.getEndRow(), columns);
+ return scanTablet(conn, opts.tableName, opts.auths, scanOpts.scanBatchSize, ke.getPrevEndRow(), ke.getEndRow(), columns);
}
};
@@ -230,7 +232,7 @@ public class CollectTabletStats {
threadPool.submit(new Runnable() {
public void run() {
try {
- calcTabletStats(conn, opts.tableName, opts.auths, opts.scanBatchSize, ke, columns);
+ calcTabletStats(conn, opts.tableName, opts.auths, scanOpts.scanBatchSize, ke, columns);
} catch (Exception e) {
e.printStackTrace();
}