You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/01/20 01:28:46 UTC
svn commit: r1233703 - in
/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk:
BulkPlusOne.java Setup.java Verify.java
Author: kturner
Date: Fri Jan 20 00:28:46 2012
New Revision: 1233703
URL: http://svn.apache.org/viewvc?rev=1233703&view=rev
Log:
ACCUMULO-334 Made Bulk RW test use random start points for files and log those start points. Also made it use better tables names and delete the table name when done.
Modified:
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java?rev=1233703&r1=1233702&r2=1233703&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java Fri Jan 20 00:28:46 2012
@@ -16,8 +16,11 @@
*/
package org.apache.accumulo.server.test.randomwalk.bulk;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.Random;
+import java.util.TreeSet;
import java.util.UUID;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -34,7 +37,7 @@ import org.apache.log4j.Logger;
public class BulkPlusOne extends BulkTest {
- public static final long LOTS = 100000;
+ public static final int LOTS = 100000;
public static final int COLS = 10;
public static final int HEX_SIZE = (int) Math.ceil(Math.log(LOTS) / Math.log(16));
public static final String FMT = "r%0" + HEX_SIZE + "x";
@@ -47,25 +50,36 @@ public class BulkPlusOne extends BulkTes
FileSystem fs = (FileSystem) state.get("fs");
fs.mkdirs(fail);
int parts = rand.nextInt(10) + 1;
- long ctr = 0;
- log.debug("preparing bulk file with " + parts + " parts");
+
+ TreeSet<Integer> startRows = new TreeSet<Integer>();
+ startRows.add(0);
+ while (startRows.size() < parts)
+ startRows.add(rand.nextInt(LOTS));
+
+ List<String> printRows = new ArrayList<String>(startRows.size());
+ for (Integer row : startRows)
+ printRows.add(String.format(FMT, row));
+
+ log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1));
String cols[] = new String[COLS];
for (int i = 0; i < cols.length; i++) {
cols[i] = String.format("%03d", i);
}
+ List<Integer> rows = new ArrayList<Integer>(startRows);
+ rows.add(LOTS);
+
for (int i = 0; i < parts; i++) {
FileSKVWriter f = FileOperations.getInstance().openWriter(dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION, fs, fs.getConf(),
AccumuloConfiguration.getDefaultConfiguration());
f.startDefaultLocalityGroup();
- int end = (int) LOTS / parts;
- if (i == parts - 1)
- end = (int) (LOTS - ctr);
- for (int j = 0; j < end; j++) {
+ int start = rows.get(i);
+ int end = rows.get(i + 1);
+ for (int j = start; j < end; j++) {
+ String row = String.format(FMT, j);
for (String col : cols) {
- f.append(new Key(String.format(FMT, ctr), "cf", col), value);
+ f.append(new Key(row, "cf", col), value);
}
- ctr++;
}
f.close();
}
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java?rev=1233703&r1=1233702&r2=1233703&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java Fri Jan 20 00:28:46 2012
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.test.randomwalk.bulk;
+import java.net.InetAddress;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -46,7 +47,9 @@ public class Setup extends Test {
@Override
public void visit(State state, Properties props) throws Exception {
Random rand = new Random();
- tableName = Integer.toHexString(Math.abs(rand.nextInt()));
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+ String pid = state.getPid();
+ tableName = String.format("bulk_%s_%s_%d", hostname, pid, System.currentTimeMillis());
log.info("Starting bulk test on " + tableName);
List<PerColumnIteratorConfig> aggregators = Collections.singletonList(new PerColumnIteratorConfig(new Text("cf".getBytes()), null,
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java?rev=1233703&r1=1233702&r2=1233703&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java Fri Jan 20 00:28:46 2012
@@ -51,6 +51,7 @@ public class Verify extends Test {
throw new Exception("Bad key at " + entry);
}
log.info("Test successful on table " + Setup.getTableName());
+ state.getConnector().tableOperations().delete(Setup.getTableName());
}
}