You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/03/25 01:45:42 UTC
git commit: HBase tests are no longer compile with HBase 0.94
(Konstantin Boudnik via rvs)
Updated Branches:
refs/heads/master 59f069bc1 -> ed58bb5e4
HBase tests are no longer compile with HBase 0.94 (Konstantin Boudnik via rvs)
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ed58bb5e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ed58bb5e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ed58bb5e
Branch: refs/heads/master
Commit: ed58bb5e4da8a49a9fc836713330141a7d783917
Parents: 59f069b
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Sun Mar 24 17:45:09 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Sun Mar 24 17:45:09 2013 -0700
----------------------------------------------------------------------
.../itest/hbase/smoke/IncrementalPELoad.java | 5 ++-
.../itest/hbase/smoke/TestHBaseImportExport.groovy | 4 +-
.../itest/hbase/smoke/TestHFileOutputFormat.java | 2 +-
.../apache/bigtop/itest/hbase/system/Putter.java | 7 ++++-
.../apache/bigtop/itest/hbase/system/Scanner.java | 7 ++++-
.../hbase/system/TestConcurrentScanAndPut.java | 17 ++++++++-------
.../itest/hbase/system/TestLoadAndVerify.java | 2 +-
.../bigtop/itest/hbase/util/HBaseTestUtil.java | 13 ++++++++---
bigtop-tests/test-artifacts/pom.xml | 2 +-
9 files changed, 36 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java
index aaf7d82..d1cb391 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java
@@ -99,13 +99,14 @@ public class IncrementalPELoad extends Configured implements Tool {
}
public int run(String[] args) throws Exception {
- Job job = new Job(getConf(), "testMRIncrementalLoad");
+ Configuration conf = getConf();
+ Job job = new Job(conf, "testMRIncrementalLoad");
job.setJarByClass(IncrementalPELoad.class);
job.setInputFormatClass(NMapInputFormat.class);
job.setMapperClass(IncrementalPELoad.RandomKVGeneratingMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(KeyValue.class);
- HTable table = new HTable(Bytes.toBytes(args[0]));
+ HTable table = new HTable(conf, Bytes.toBytes(args[0]));
HFileOutputFormat.configureIncrementalLoad(job, table);
FileOutputFormat.setOutputPath(job, new Path(args[1]));
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy
index 2ea01db..66696d3 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy
@@ -68,13 +68,13 @@ class TestHBaseImportExport {
HTableDescriptor htd_orig =
HBaseTestUtil.createTestTableDescriptor("orig", TEST_FAMILY)
admin.createTable(htd_orig)
- orig = new String(htd_orig.getName())
+ orig = htd_orig.getName()
origTable = new HTable(conf, orig)
HTableDescriptor htd_export =
HBaseTestUtil.createTestTableDescriptor("export", TEST_FAMILY)
admin.createTable(htd_export)
- export = new String(htd_export.getName())
+ export = htd_export.getName()
exportTable = new HTable(conf, export)
// Write some rows to the table.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java
index f444226..37f7009 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java
@@ -126,7 +126,7 @@ public class TestHFileOutputFormat {
tbldesc.addFamily(coldesc1);
tbldesc.addFamily(coldesc2);
admin.createTable(tbldesc, splitKeys);
- HTable table = new HTable(TABLE_NAME);
+ HTable table = new HTable(conf, TABLE_NAME);
assertEquals("Should start with empty table",
0, HBaseTestUtil.countRows(table));
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java
index 953fc8d..0aaffb3 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.Map;
import java.util.NavigableMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -108,10 +110,11 @@ public class Putter {
loops = Integer.parseInt(argv[i]);
}
}
+ Configuration conf = HBaseConfiguration.create();
- String tableName = argv[0];
+ byte [] tableName = Bytes.toBytes(argv[0]);
int val = Integer.parseInt(argv[1]);
- HTable table = new HTable(tableName);
+ HTable table = new HTable(conf, tableName);
for (int i = 0; i < loops; i++) {
try {
doScanAndPut(table, val, autoflush);
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java
index 8f4433b..b6a7e2d 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java
@@ -25,6 +25,8 @@ import java.util.NavigableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -106,8 +108,9 @@ public class Scanner {
System.err.println(" <value>: a numeric value [0,500)");
System.exit(1);
}
+ Configuration conf = HBaseConfiguration.create();
- String tableName = argv[0];
+ byte [] tableName = Bytes.toBytes(argv[0]);
int val = Integer.parseInt(argv[1]);
int loops = 1;
for (int i = 1; i < argv.length; i++) {
@@ -117,7 +120,7 @@ public class Scanner {
}
}
- HTable table = new HTable(tableName);
+ HTable table = new HTable(conf, tableName);
int exitVal = 0;
for (int i = 0; i < loops; i++) {
try {
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java
index 08e8781..0b1414d 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java
@@ -53,7 +53,7 @@ public class TestConcurrentScanAndPut {
public static Shell putSh = new Shell( "/bin/bash -s" );
public static HBaseAdmin admin;
- public static String tableName;
+ public static byte [] tableName;
public static String putter_pid;
public static int scannerLoops;
@@ -75,7 +75,7 @@ public class TestConcurrentScanAndPut {
}
tableName =
- new String(HBaseTestUtil.getTestTableName("concurrentScanAndPut"));
+ Bytes.toBytes(new String(HBaseTestUtil.getTestTableName("concurrentScanAndPut")));
HTableDescriptor htd = new HTableDescriptor(tableName);
for (int i = 0; i < 10; i++) {
htd.addFamily(new HColumnDescriptor("f" + i));
@@ -83,7 +83,7 @@ public class TestConcurrentScanAndPut {
admin = new HBaseAdmin(conf);
admin.createTable(htd);
- HTable table = new HTable(tableName);
+ HTable table = new HTable(conf, tableName);
ArrayList<Put> puts = new ArrayList<Put>(1000);
Random rnd = new Random();
@@ -137,25 +137,26 @@ public class TestConcurrentScanAndPut {
System.out.println("Killing putter process");
putSh.exec("kill -9 " + putter_pid);
- System.out.println("Removing test table " + tableName);
+ System.out.println("Removing test table " + Bytes.toString(tableName));
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
@Test
public void testConcurrentScanAndPut() {
- System.out.println("Starting puts to test table " + tableName);
+ String tableNameStr = Bytes.toString(tableName);
+ System.out.println("Starting puts to test table " + tableNameStr);
putSh.exec("(HBASE_CLASSPATH=. " +
"hbase com.cloudera.itest.hbase.system.Putter " +
- tableName + " 13 -l " + putterLoops +
+ tableNameStr + " 13 -l " + putterLoops +
" > /dev/null 2>&1 & echo $! ) 2> /dev/null");
putter_pid = putSh.getOut().get(0);
System.out.println("Starting concurrent scans of test table " +
- tableName);
+ tableNameStr);
scanSh.exec("HBASE_CLASSPATH=. hbase " +
"com.cloudera.itest.hbase.system.Scanner " +
- tableName + " 13 -l " + scannerLoops + " 2>/dev/null");
+ tableNameStr + " 13 -l " + scannerLoops + " 2>/dev/null");
int splitRows = scanSh.getRet();
System.out.println("Split rows: " + splitRows);
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
index 3841bd5..3bf403e 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
@@ -121,7 +121,7 @@ public class TestLoadAndVerify extends Configured implements Tool {
public void setup(Context context) throws IOException {
conf = context.getConfiguration();
recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT);
- String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT);
+ byte [] tableName = Bytes.toBytes(conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT));
numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT);
table = new HTable(conf, tableName);
table.setWriteBufferSize(4*1024*1024);
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java
index d78b2e6..8526754 100644
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java
+++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java
@@ -34,9 +34,11 @@ import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
public class HBaseTestUtil {
@@ -84,10 +86,13 @@ public class HBaseTestUtil {
byte[] family, byte[] qualifier,
byte[] startKey, byte[] endKey, int numRows) throws IOException
{
- HFile.Writer writer =
- HFile.getWriterFactory(conf).createWriter(fs, path,
- BLOCKSIZE, COMPRESSION,
- KeyValue.KEY_COMPARATOR);
+ HFile.WriterFactory wf = HFile.getWriterFactory(conf, new CacheConfig(conf));
+ wf.withChecksumType(ChecksumType.CRC32);
+ wf.withBlockSize(BLOCKSIZE);
+ wf.withCompression(COMPRESSION);
+ wf.withComparator(KeyValue.KEY_COMPARATOR);
+ wf.withPath(fs, path);
+ HFile.Writer writer = wf.create();
long now = System.currentTimeMillis();
try {
// subtract 2 since iterateOnSplits doesn't include boundary keys
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml
index 856e6e6..db9b524 100644
--- a/bigtop-tests/test-artifacts/pom.xml
+++ b/bigtop-tests/test-artifacts/pom.xml
@@ -54,7 +54,7 @@
<hadoop.version>2.0.0-SNAPSHOT</hadoop.version>
<pig.version>0.9.2</pig.version>
<sqoop.version>1.4.1-incubating</sqoop.version>
- <hbase.version>0.92.1</hbase.version>
+ <hbase.version>0.94.5</hbase.version>
<zookeeper.version>3.4.3</zookeeper.version>
<giraph.version>0.2-SNAPSHOT</giraph.version>
<solr.version>4.0.0-BETA</solr.version>