You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/07/22 06:59:29 UTC
svn commit: r1505597 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
Author: liyin
Date: Mon Jul 22 04:59:29 2013
New Revision: 1505597
URL: http://svn.apache.org/r1505597
Log:
[master] Verifying that deletes work fine with HFileOutputFormat
Author: gauravm
Summary:
@groys wanted to know a good way to bulk delete. We found that HFileOutputFormat supports Deletes as well, so making a slight modification in a unit test to
verify that.
Test Plan: Unit Test
Reviewers: liyintang, groys, manukranthk
Reviewed By: manukranthk
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D890008
Task ID: 2614378
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java?rev=1505597&r1=1505596&r2=1505597&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapred/HexStringTableRecordReaderImpl.java Mon Jul 22 04:59:29 2013
@@ -25,8 +25,8 @@ import java.math.BigInteger;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.util.Bytes;
-import com.sun.org.apache.commons.logging.Log;
-import com.sun.org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class HexStringTableRecordReaderImpl extends TableRecordReaderImpl {
final Log LOG = LogFactory.getLog(HexStringTableRecordReaderImpl.class);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java?rev=1505597&r1=1505596&r2=1505597&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatWithHFileOutputFormat.java Mon Jul 22 04:59:29 2013
@@ -27,9 +27,10 @@ public class TestTableInputFormatWithHFi
static final String TABLENAME = "testMultipleHLogs";
static final byte[] CF1 = Bytes.toBytes("cf1");
static final byte[] CF2 = Bytes.toBytes("cf2");
- static final byte[][] FAMILIES = new byte[][]{CF1, CF2};
+ static final byte[] CF3 = Bytes.toBytes("cf3");
+ static final byte[][] FAMILIES = new byte[][]{CF1, CF2, CF3};
static final int REGION_NUM = 20;
- static final byte[] QAULIFIER = Bytes.toBytes("q");
+ static final byte[] QUALIFIER = Bytes.toBytes("q");
static final byte[] VALUE = Bytes.toBytes("v");
static final Path OUTPUTPATH = new Path("TEST-OUTPUT");
static HTable htable;
@@ -82,7 +83,10 @@ public class TestTableInputFormatWithHFi
// Put some data for each Region
for (byte[] row : htable.getStartKeys()) {
Put p = new Put(row);
- p.add(CF1, QAULIFIER, VALUE);
+ p.add(CF1, QUALIFIER, VALUE);
+ // We would delete the kv for CF3, to verify that Deletes work with
+ // HFileOutputFormat
+ p.add(CF3, QUALIFIER, VALUE);
htable.put(p);
htable.flushCommits();
}
@@ -92,7 +96,8 @@ public class TestTableInputFormatWithHFi
// Create the scan object
Scan scan = new Scan();
scan.addFamily(CF1);
-
+ scan.addFamily(CF3);
+
// Create and initialize the MR job
Job job = new Job(conf, "process column contents");
FileOutputFormat.setOutputPath(job, OUTPUTPATH);
@@ -120,20 +125,23 @@ public class TestTableInputFormatWithHFi
Scan scan = new Scan();
scan.addFamily(CF1);
scan.addFamily(CF2);
+ scan.addFamily(CF3);
ResultScanner s = htable.getScanner(scan);
Result result = null;
int count = 0;
while((result = s.next()) != null) {
count++;
-
+
+ // We should only see CF1 and CF2. And not see CF3 since, we also added
+ // Delete kvs in the MR job.
Assert.assertEquals(2, result.list().size());
KeyValue kvFromCF1 = result.list().get(0);
KeyValue kvFromCF2 = result.list().get(1);
-
+
Assert.assertTrue(Bytes.compareTo(kvFromCF1.getFamily(), CF1) == 0);
Assert.assertTrue(Bytes.compareTo(kvFromCF2.getFamily(), CF2) == 0);
-
+
Assert.assertTrue(Bytes.compareTo(kvFromCF1.getRow(), kvFromCF2.getRow()) == 0);
Assert.assertTrue(Bytes.compareTo(kvFromCF1.getQualifier(), kvFromCF2.getQualifier()) == 0);
Assert.assertTrue(kvFromCF1.getTimestamp() == kvFromCF2.getTimestamp());
@@ -147,10 +155,13 @@ public class TestTableInputFormatWithHFi
private KeyValue previousKV = null;
public void map(ImmutableBytesWritable key, Result result, Context context)
throws IOException, InterruptedException {
- Assert.assertEquals(1, result.size());
+ Assert.assertEquals(2, result.size());
KeyValue tmp = result.list().get(0);
KeyValue currentKV = new KeyValue(tmp.getRow(), CF2, tmp.getQualifier(), tmp.getTimestamp(), tmp.getValue());
-
+
+ KeyValue tmp2 = result.list().get(1);
+ KeyValue deleteKV = new KeyValue(tmp2.getRow(), CF3, tmp2.getQualifier(), tmp.getTimestamp(), KeyValue.Type.Delete);
+
// Sanity check that the output key value is sorted
if (previousKV != null) {
Assert.assertTrue(KeyValue.COMPARATOR.compare(currentKV, previousKV) >= 0);
@@ -158,6 +169,7 @@ public class TestTableInputFormatWithHFi
previousKV = currentKV;
System.out.println("current KV: " + Bytes.toStringBinary(currentKV.getBuffer()));
context.write(key, currentKV);
+ context.write(key, deleteKV);
}
}
}