You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2013/06/19 00:51:35 UTC
svn commit: r1494359 -
/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
Author: sershe
Date: Tue Jun 18 22:51:35 2013
New Revision: 1494359
URL: http://svn.apache.org/r1494359
Log:
HBASE-8700 IntegrationTestBigLinkedList can fail due to random number collision
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java?rev=1494359&r1=1494358&r2=1494359&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java Tue Jun 18 22:51:35 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.test;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.io.StringWriter;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
@@ -55,10 +56,9 @@ import org.apache.hadoop.hbase.io.Immuta
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Counters;
@@ -148,6 +148,7 @@ import org.junit.experimental.categories
*/
@Category(IntegrationTests.class)
public class IntegrationTestBigLinkedList extends Configured implements Tool {
+ private static final byte[] NO_KEY = new byte[1];
private static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
@@ -171,9 +172,23 @@ public class IntegrationTestBigLinkedLis
private static final String GENERATOR_NUM_MAPPERS_KEY
= "IntegrationTestBigLinkedList.generator.map.tasks";
+ private static final String GENERATOR_WIDTH_KEY
+ = "IntegrationTestBigLinkedList.generator.width";
+
+ private static final String GENERATOR_WRAP_KEY
+ = "IntegrationTestBigLinkedList.generator.wrap";
+
+ protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster
+
+ private static final int WIDTH_DEFAULT = 1000000;
+ private static final int WRAP_DEFAULT = 25;
+
+ private static final int ROWKEY_LENGTH = 16;
+
static class CINode {
- long key;
- long prev;
+ byte[] key;
+ byte[] prev;
+
String client;
long count;
}
@@ -185,14 +200,11 @@ public class IntegrationTestBigLinkedLis
private static final Log LOG = LogFactory.getLog(Generator.class);
- private static final int WIDTH = 1000000;
- private static final int WRAP = WIDTH * 25;
-
public static enum Counts {
UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
}
- static class GeneratorInputFormat extends InputFormat<LongWritable,NullWritable> {
+ static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> {
static class GeneratorInputSplit extends InputSplit implements Writable {
@Override
public long getLength() throws IOException, InterruptedException {
@@ -210,7 +222,7 @@ public class IntegrationTestBigLinkedLis
}
}
- static class GeneratorRecordReader extends RecordReader<LongWritable,NullWritable> {
+ static class GeneratorRecordReader extends RecordReader<BytesWritable,NullWritable> {
private long count;
private long numNodes;
private Random rand;
@@ -220,8 +232,10 @@ public class IntegrationTestBigLinkedLis
}
@Override
- public LongWritable getCurrentKey() throws IOException, InterruptedException {
- return new LongWritable(Math.abs(rand.nextLong()));
+ public BytesWritable getCurrentKey() throws IOException, InterruptedException {
+ byte[] bytes = new byte[ROWKEY_LENGTH];
+ rand.nextBytes(bytes);
+ return new BytesWritable(bytes);
}
@Override
@@ -249,7 +263,7 @@ public class IntegrationTestBigLinkedLis
}
@Override
- public RecordReader<LongWritable,NullWritable> createRecordReader(
+ public RecordReader<BytesWritable,NullWritable> createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
GeneratorRecordReader rr = new GeneratorRecordReader();
rr.initialize(split, context);
@@ -303,18 +317,19 @@ public class IntegrationTestBigLinkedLis
* |___________________________|
*/
static class GeneratorMapper
- extends Mapper<LongWritable, NullWritable, NullWritable, NullWritable> {
+ extends Mapper<BytesWritable, NullWritable, NullWritable, NullWritable> {
Random rand = new Random();
- long[] first = null;
- long[] prev = null;
- long[] current = new long[WIDTH];
+ byte[][] first = null;
+ byte[][] prev = null;
+ byte[][] current = null;
byte[] id;
long count = 0;
int i;
HTable table;
long numNodes;
- long wrap = WRAP;
+ long wrap;
+ int width;
protected void setup(Context context) throws IOException, InterruptedException {
id = Bytes.toBytes(UUID.randomUUID().toString());
@@ -322,9 +337,14 @@ public class IntegrationTestBigLinkedLis
table = new HTable(conf, getTableName(conf));
table.setAutoFlush(false);
table.setWriteBufferSize(4 * 1024 * 1024);
- numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
- if (numNodes < 25000000) {
- wrap = numNodes;
+ this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT);
+ current = new byte[this.width][];
+ int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT);
+ this.wrap = (long)wrapMultiplier * width;
+ this.numNodes = context.getConfiguration().getLong(
+ GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT);
+ if (this.numNodes < this.wrap) {
+ this.wrap = this.numNodes;
}
};
@@ -333,17 +353,17 @@ public class IntegrationTestBigLinkedLis
};
@Override
- protected void map(LongWritable key, NullWritable value, Context output) throws IOException {
- current[i++] = Math.abs(key.get());
-
- if (i == current.length) {
+ protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
+ current[i] = new byte[key.getLength()];
+ System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
+ if (++i == current.length) {
persist(output, count, prev, current, id);
i = 0;
if (first == null)
first = current;
prev = current;
- current = new long[WIDTH];
+ current = new byte[this.width][];
count += current.length;
output.setStatus("Count " + count);
@@ -361,18 +381,18 @@ public class IntegrationTestBigLinkedLis
}
}
- private static void circularLeftShift(long[] first) {
- long ez = first[0];
+ private static <T> void circularLeftShift(T[] first) {
+ T ez = first[0];
for (int i = 0; i < first.length - 1; i++)
first[i] = first[i + 1];
first[first.length - 1] = ez;
}
- private void persist(Context output, long count, long[] prev, long[] current, byte[] id)
+ private void persist(Context output, long count, byte[][] prev, byte[][] current, byte[] id)
throws IOException {
for (int i = 0; i < current.length; i++) {
- Put put = new Put(Bytes.toBytes(current[i]));
- put.add(FAMILY_NAME, COLUMN_PREV, Bytes.toBytes(prev == null ? -1 : prev[i]));
+ Put put = new Put(current[i]);
+ put.add(FAMILY_NAME, COLUMN_PREV, prev == null ? NO_KEY : prev[i]);
if (count >= 0) {
put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i));
@@ -396,15 +416,18 @@ public class IntegrationTestBigLinkedLis
public int run(String[] args) throws Exception {
if (args.length < 3) {
System.out.println("Usage : " + Generator.class.getSimpleName() +
- " <num mappers> <num nodes per map> <tmp output dir>");
- System.out.println(" where <num nodes per map> should be a multiple of 25M");
+ " <num mappers> <num nodes per map> <tmp output dir> [<width> <wrap multiplier>]");
+ System.out.println(" where <num nodes per map> should be a multiple of " +
+ " width*wrap multiplier, 25M by default");
return 0;
}
int numMappers = Integer.parseInt(args[0]);
long numNodes = Long.parseLong(args[1]);
Path tmpOutput = new Path(args[2]);
- return run(numMappers, numNodes, tmpOutput);
+ Integer width = (args.length < 4) ? null : Integer.parseInt(args[3]);
+ Integer wrapMuplitplier = (args.length < 5) ? null : Integer.parseInt(args[4]);
+ return run(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
}
protected void createSchema() throws IOException {
@@ -418,8 +441,8 @@ public class IntegrationTestBigLinkedLis
admin.close();
}
- public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput)
- throws Exception {
+ public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMuplitplier) throws Exception {
LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
+ ", numNodes=" + numNodes);
Job job = new Job(getConf());
@@ -429,11 +452,10 @@ public class IntegrationTestBigLinkedLis
job.setJarByClass(getClass());
job.setInputFormatClass(GeneratorInputFormat.class);
- job.setOutputKeyClass(LongWritable.class);
+ job.setOutputKeyClass(BytesWritable.class);
job.setOutputValueClass(NullWritable.class);
- job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
- job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+ setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
job.setMapperClass(Mapper.class); //identity mapper
@@ -445,7 +467,8 @@ public class IntegrationTestBigLinkedLis
return success ? 0 : 1;
}
- public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Exception {
+ public int runGenerator(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMuplitplier) throws Exception {
LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
createSchema();
@@ -460,8 +483,7 @@ public class IntegrationTestBigLinkedLis
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(NullWritable.class);
- job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
- job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+ setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
job.setMapperClass(GeneratorMapper.class);
@@ -476,13 +498,13 @@ public class IntegrationTestBigLinkedLis
return success ? 0 : 1;
}
- public int run(int numMappers, long numNodes, Path tmpOutput) throws Exception {
- int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput);
+ public int run(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMuplitplier) throws Exception {
+ int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
if (ret > 0) {
return ret;
}
-
- return runGenerator(numMappers, numNodes, tmpOutput);
+ return runGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
}
}
@@ -493,72 +515,83 @@ public class IntegrationTestBigLinkedLis
static class Verify extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(Verify.class);
- private static final VLongWritable DEF = new VLongWritable(-1);
+ private static final BytesWritable DEF = new BytesWritable(NO_KEY);
private Job job;
- public static class VerifyMapper extends TableMapper<LongWritable, VLongWritable> {
- private LongWritable row = new LongWritable();
- private LongWritable ref = new LongWritable();
- private VLongWritable vrow = new VLongWritable();
+ public static class VerifyMapper extends TableMapper<BytesWritable, BytesWritable> {
+ private BytesWritable row = new BytesWritable();
+ private BytesWritable ref = new BytesWritable();
@Override
protected void map(ImmutableBytesWritable key, Result value, Context context)
throws IOException ,InterruptedException {
- row.set(Bytes.toLong(key.get()));
+ byte[] rowKey = key.get();
+ row.set(rowKey, 0, rowKey.length);
context.write(row, DEF);
-
- long prev = Bytes.toLong(value.getValue(FAMILY_NAME, COLUMN_PREV));
- if (prev >= 0) {
- ref.set(prev);
- vrow.set(Bytes.toLong(key.get()));
- context.write(ref, vrow);
+ byte[] prev = value.getValue(FAMILY_NAME, COLUMN_PREV);
+ if (prev != null && prev.length > 0) {
+ ref.set(prev, 0, prev.length);
+ context.write(ref, row);
+ } else {
+ LOG.warn(String.format("Prev is not set for: %s", Bytes.toStringBinary(rowKey)));
}
}
}
public static enum Counts {
- UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT, EXTRAREFERENCES
}
- public static class VerifyReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
- private ArrayList<Long> refs = new ArrayList<Long>();
+ public static class VerifyReducer extends Reducer<BytesWritable,BytesWritable,Text,Text> {
+ private ArrayList<byte[]> refs = new ArrayList<byte[]>();
- public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context)
+ public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
throws IOException, InterruptedException {
int defCount = 0;
refs.clear();
- for (VLongWritable type : values) {
- if (type.get() == -1) {
+ for (BytesWritable type : values) {
+ if (type.getLength() == DEF.getLength()) {
defCount++;
} else {
- refs.add(type.get());
+ byte[] bytes = new byte[type.getLength()];
+ System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
+ refs.add(bytes);
}
}
// TODO check for more than one def, should not happen
- if (defCount == 0 && refs.size() > 0) {
- // this is bad, found a node that is referenced but not defined. It must have been
- //lost, emit some info about this node for debugging purposes.
-
- StringBuilder sb = new StringBuilder();
+ StringBuilder refsSb = null;
+ String keyString = null;
+ if (defCount == 0 || refs.size() != 1) {
+ refsSb = new StringBuilder();
String comma = "";
- for (Long ref : refs) {
- sb.append(comma);
+ for (byte[] ref : refs) {
+ refsSb.append(comma);
comma = ",";
- sb.append(String.format("%016x", ref));
+ refsSb.append(Bytes.toStringBinary(ref));
}
+ byte[] bytes = new byte[key.getLength()];
+ keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength());
+ }
- context.write(new Text(String.format("%016x", key.get())), new Text(sb.toString()));
+ if (defCount == 0 && refs.size() > 0) {
+ // this is bad, found a node that is referenced but not defined. It must have been
+ // lost, emit some info about this node for debugging purposes.
+ context.write(new Text(keyString), new Text(refsSb.toString()));
context.getCounter(Counts.UNDEFINED).increment(1);
-
} else if (defCount > 0 && refs.size() == 0) {
// node is defined but not referenced
+ context.write(new Text(keyString), new Text("none"));
context.getCounter(Counts.UNREFERENCED).increment(1);
} else {
+ if (refs.size() > 1) {
+ context.write(new Text(keyString), new Text(refsSb.toString()));
+ context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
+ }
// node is defined and referenced
context.getCounter(Counts.REFERENCED).increment(1);
}
@@ -599,7 +632,7 @@ public class IntegrationTestBigLinkedLis
scan.setCacheBlocks(false);
TableMapReduceUtil.initTableMapperJob(getTableName(getConf()), scan,
- VerifyMapper.class, LongWritable.class, VLongWritable.class, job);
+ VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
@@ -622,6 +655,7 @@ public class IntegrationTestBigLinkedLis
Counter referenced = counters.findCounter(Counts.REFERENCED);
Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
Counter undefined = counters.findCounter(Counts.UNDEFINED);
+ Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);
boolean success = true;
//assert
@@ -632,7 +666,9 @@ public class IntegrationTestBigLinkedLis
}
if (unreferenced.getValue() > 0) {
- LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
+ boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
+ LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
+ + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
success = false;
}
@@ -653,15 +689,15 @@ public class IntegrationTestBigLinkedLis
private static final Log LOG = LogFactory.getLog(Loop.class);
- protected void runGenerator(int numMappers, long numNodes, String outputDir) throws Exception {
+ protected void runGenerator(int numMappers, long numNodes,
+ String outputDir, Integer width, Integer wrapMuplitplier) throws Exception {
Path outputPath = new Path(outputDir);
UUID uuid = UUID.randomUUID(); //create a random UUID.
Path generatorOutput = new Path(outputPath, uuid.toString());
Generator generator = new Generator();
generator.setConf(getConf());
- int retCode = generator.run(numMappers, numNodes, generatorOutput);
-
+ int retCode = generator.run(numMappers, numNodes, generatorOutput, width, wrapMuplitplier);
if (retCode > 0) {
throw new RuntimeException("Generator failed with return code: " + retCode);
}
@@ -690,10 +726,9 @@ public class IntegrationTestBigLinkedLis
@Override
public int run(String[] args) throws Exception {
if (args.length < 5) {
- System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers>");
+ System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers> [<width> <wrap multiplier>]");
return 1;
}
-
LOG.info("Running Loop with args:" + Arrays.deepToString(args));
int numIterations = Integer.parseInt(args[0]);
@@ -701,6 +736,8 @@ public class IntegrationTestBigLinkedLis
long numNodes = Long.parseLong(args[2]);
String outputDir = args[3];
int numReducers = Integer.parseInt(args[4]);
+ Integer width = (args.length < 6) ? null : Integer.parseInt(args[5]);
+ Integer wrapMuplitplier = (args.length < 7) ? null : Integer.parseInt(args[6]);
long expectedNumNodes = 0;
@@ -708,9 +745,9 @@ public class IntegrationTestBigLinkedLis
numIterations = Integer.MAX_VALUE; //run indefinitely (kind of)
}
- for (int i=0; i < numIterations; i++) {
+ for (int i = 0; i < numIterations; i++) {
LOG.info("Starting iteration = " + i);
- runGenerator(numMappers, numNodes, outputDir);
+ runGenerator(numMappers, numNodes, outputDir, width, wrapMuplitplier);
expectedNumNodes += numMappers * numNodes;
runVerify(outputDir, numReducers, expectedNumNodes);
@@ -751,10 +788,10 @@ public class IntegrationTestBigLinkedLis
scan.setBatch(10000);
if (cmd.hasOption("s"))
- scan.setStartRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("s"), 16).longValue()));
+ scan.setStartRow(Bytes.toBytesBinary(cmd.getOptionValue("s")));
if (cmd.hasOption("e"))
- scan.setStopRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("e"), 16).longValue()));
+ scan.setStopRow(Bytes.toBytesBinary(cmd.getOptionValue("e")));
int limit = 0;
if (cmd.hasOption("l"))
@@ -769,7 +806,8 @@ public class IntegrationTestBigLinkedLis
int count = 0;
while (result != null && count++ < limit) {
node = getCINode(result, node);
- System.out.printf("%016x:%016x:%012d:%s\n", node.key, node.prev, node.count, node.client);
+ System.out.printf("%s:%s:%012d:%s\n", Bytes.toStringBinary(node.key),
+ Bytes.toStringBinary(node.prev), node.count, node.client);
result = scanner.next();
}
scanner.close();
@@ -788,10 +826,10 @@ public class IntegrationTestBigLinkedLis
System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
return 0;
}
- long val = new BigInteger(args[0], 16).longValue();
+ byte[] val = Bytes.toBytesBinary(args[0]);
org.apache.hadoop.hbase.client.Delete delete
- = new org.apache.hadoop.hbase.client.Delete(Bytes.toBytes(val));
+ = new org.apache.hadoop.hbase.client.Delete(val);
HTable table = new HTable(getConf(), getTableName(getConf()));
@@ -811,6 +849,8 @@ public class IntegrationTestBigLinkedLis
public int run(String[] args) throws IOException {
Options options = new Options();
options.addOption("n", "num", true, "number of queries");
+ options.addOption("s", "start", true, "key to start at, binary string");
+ options.addOption("l", "logevery", true, "log every N queries");
GnuParser parser = new GnuParser();
CommandLine cmd = null;
@@ -831,30 +871,40 @@ public class IntegrationTestBigLinkedLis
if (cmd.hasOption('n')) {
maxQueries = Long.parseLong(cmd.getOptionValue("n"));
}
-
- HTable table = new HTable(getConf(), getTableName(getConf()));
-
Random rand = new Random();
+ boolean isSpecificStart = cmd.hasOption('s');
+ byte[] startKey = isSpecificStart ? Bytes.toBytesBinary(cmd.getOptionValue('s')) : null;
+ int logEvery = cmd.hasOption('l') ? Integer.parseInt(cmd.getOptionValue('l')) : 1;
+ HTable table = new HTable(getConf(), getTableName(getConf()));
long numQueries = 0;
-
- while (numQueries < maxQueries) {
- CINode node = findStartNode(rand, table);
+ // If isSpecificStart is set, only walk one list from that particular node.
+ // Note that in case of circular (or P-shaped) list it will walk forever, as is
+ // the case in normal run without startKey.
+ while (numQueries < maxQueries && (numQueries == 0 || !isSpecificStart)) {
+ if (!isSpecificStart) {
+ startKey = new byte[ROWKEY_LENGTH];
+ rand.nextBytes(startKey);
+ }
+ CINode node = findStartNode(table, startKey);
+ if (node == null && isSpecificStart) {
+ System.err.printf("Start node not found: %s \n", Bytes.toStringBinary(startKey));
+ }
numQueries++;
- while (node != null && node.prev >= 0 && numQueries < maxQueries) {
- long prev = node.prev;
-
+ while (node != null && node.prev.length != NO_KEY.length && numQueries < maxQueries) {
+ byte[] prev = node.prev;
long t1 = System.currentTimeMillis();
node = getNode(prev, table, node);
long t2 = System.currentTimeMillis();
- System.out.printf("CQ %d %016x \n", t2 - t1, prev); //cold cache
- numQueries++;
-
- t1 = System.currentTimeMillis();
- node = getNode(prev, table, node);
- t2 = System.currentTimeMillis();
- System.out.printf("HQ %d %016x \n", t2 - t1, prev); //hot cache
+ if (numQueries % logEvery == 0) {
+ System.out.printf("CQ %d: %d %s \n", numQueries, t2 - t1, Bytes.toStringBinary(prev));
+ }
numQueries++;
+ if (node == null) {
+ System.err.printf("UNDEFINED NODE %s \n", Bytes.toStringBinary(prev));
+ } else if (node.prev.length == NO_KEY.length) {
+ System.err.printf("TERMINATING NODE %s \n", Bytes.toStringBinary(node.key));
+ }
}
}
@@ -862,9 +912,9 @@ public class IntegrationTestBigLinkedLis
return 0;
}
- private static CINode findStartNode(Random rand, HTable table) throws IOException {
+ private static CINode findStartNode(HTable table, byte[] startKey) throws IOException {
Scan scan = new Scan();
- scan.setStartRow(Bytes.toBytes(Math.abs(rand.nextLong())));
+ scan.setStartRow(startKey);
scan.setBatch(1);
scan.addColumn(FAMILY_NAME, COLUMN_PREV);
@@ -876,7 +926,7 @@ public class IntegrationTestBigLinkedLis
if ( result != null) {
CINode node = getCINode(result, new CINode());
- System.out.printf("FSR %d %016x\n", t2 - t1, node.key);
+ System.out.printf("FSR %d %s\n", t2 - t1, Bytes.toStringBinary(node.key));
return node;
}
@@ -885,8 +935,8 @@ public class IntegrationTestBigLinkedLis
return null;
}
- private CINode getNode(long row, HTable table, CINode node) throws IOException {
- Get get = new Get(Bytes.toBytes(row));
+ private CINode getNode(byte[] row, HTable table, CINode node) throws IOException {
+ Get get = new Get(row);
get.addColumn(FAMILY_NAME, COLUMN_PREV);
Result result = table.get(get);
return getCINode(result, node);
@@ -898,11 +948,14 @@ public class IntegrationTestBigLinkedLis
}
private static CINode getCINode(Result result, CINode node) {
- node.key = Bytes.toLong(result.getRow());
+ node.key = new byte[result.getRow().length];
+ System.arraycopy(result.getRow(), 0, node.key, 0, node.key.length);
if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) {
- node.prev = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_PREV));
+ byte[] value = result.getValue(FAMILY_NAME, COLUMN_PREV);
+ node.prev = new byte[value.length];
+ System.arraycopy(value, 0, node.prev, 0, node.prev.length);
} else {
- node.prev = -1;
+ node.prev = NO_KEY;
}
if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) {
node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT));
@@ -1016,4 +1069,16 @@ public class IntegrationTestBigLinkedLis
int ret = ToolRunner.run(HBaseConfiguration.create(), new IntegrationTestBigLinkedList(), args);
System.exit(ret);
}
+
+ private static void setJobConf(Job job, int numMappers, long numNodes,
+ Integer width, Integer wrapMuplitplier) {
+ job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
+ job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+ if (width != null) {
+ job.getConfiguration().setInt(GENERATOR_WIDTH_KEY, width.intValue());
+ }
+ if (wrapMuplitplier != null) {
+ job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMuplitplier.intValue());
+ }
+ }
}