You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/03/22 03:08:59 UTC
git commit: GIRAPH-510: Remove HBase Cruft
Updated Branches:
refs/heads/trunk f4bd1996e -> 156682791
GIRAPH-510: Remove HBase Cruft
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/15668279
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/15668279
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/15668279
Branch: refs/heads/trunk
Commit: 156682791504c2e433382551e1be301e1d9146cb
Parents: f4bd199
Author: Nitay Joffe <ni...@fb.com>
Authored: Thu Mar 21 22:03:03 2013 -0400
Committer: Nitay Joffe <ni...@fb.com>
Committed: Thu Mar 21 22:08:45 2013 -0400
----------------------------------------------------------------------
CHANGELOG | 2 +
.../io/hbase/TestHBaseRootMarkerVertextFormat.java | 70 +++++++++++----
2 files changed, 55 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/15668279/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index a1446a7..db8f4ce 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-510: Remove HBase Cruft (kelarini via nitay)
+
GIRAPH-566: Make option for aggregators to be configurable (majakabiljo)
GIRAPH-575: update hive-io (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/15668279/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
----------------------------------------------------------------------
diff --git a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
index 4d9588a..7415ddc 100644
--- a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
+++ b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
@@ -29,8 +29,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.Get;
@@ -51,6 +53,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.util.UUID;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -65,7 +68,7 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
*
* @param testName name of the test case
*/
- private HBaseTestingUtility testUtil = new HBaseTestingUtility();
+
private final Logger log = Logger.getLogger(TestHBaseRootMarkerVertextFormat.class);
private final String TABLE_NAME = "simple_graph";
@@ -73,12 +76,33 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
private final String QUALIFER = "children";
private final String OUTPUT_FIELD = "parent";
+ private HBaseTestingUtility testUtil;
+ private Path hbaseRootdir;
+
+
public TestHBaseRootMarkerVertextFormat() {
super(TestHBaseRootMarkerVertextFormat.class.getName());
+
+ // Let's set up the hbase root directory.
+ Configuration conf = HBaseConfiguration.create();
+ try {
+ FileSystem fs = FileSystem.get(conf);
+ String randomStr = UUID.randomUUID().toString();
+ String tmpdir = System.getProperty("java.io.tmpdir") +
+ randomStr + "/";
+ hbaseRootdir = fs.makeQualified(new Path(tmpdir));
+ conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
+ fs.mkdirs(hbaseRootdir);
+ } catch(IOException ioe) {
+ fail("Could not create hbase root directory.");
+ }
+
+ // Start the test utility.
+ testUtil = new HBaseTestingUtility(conf);
}
@Test
- public void testHBaseInputOutput() throws Exception{
+ public void testHBaseInputOutput() throws Exception {
if (System.getProperty("prop.mapred.job.tracker") != null) {
if(log.isInfoEnabled())
@@ -93,7 +117,7 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
"Make sure you built the main Giraph artifact?.");
}
- String INPUT_FILE = "graph.csv";
+ String INPUT_FILE = hbaseRootdir.toString() + "/graph.csv";
//First let's load some data using ImportTsv into our mock table.
String[] args = new String[] {
"-Dimporttsv.columns=HBASE_ROW_KEY,cf:"+QUALIFER,
@@ -102,17 +126,23 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
INPUT_FILE
};
- MiniZooKeeperCluster zkCluster = testUtil.startMiniZKCluster();
- MiniHBaseCluster cluster = testUtil.startMiniHBaseCluster(2, 2);
-
- GenericOptionsParser opts =
- new GenericOptionsParser(cluster.getConfiguration(), args);
- Configuration conf = opts.getConfiguration();
- args = opts.getRemainingArgs();
+ MiniHBaseCluster cluster = null;
+ MiniZooKeeperCluster zkCluster = null;
+ FileSystem fs = null;
try {
-
- FileSystem fs = FileSystem.get(conf);
+ // using the restart method allows us to avoid having the hbase
+ // root directory overwritten by /home/$username
+ zkCluster = testUtil.startMiniZKCluster();
+ testUtil.restartHBaseCluster(2);
+ cluster = testUtil.getMiniHBaseCluster();
+
+ GenericOptionsParser opts =
+ new GenericOptionsParser(cluster.getConfiguration(), args);
+ Configuration conf = opts.getConfiguration();
+ args = opts.getRemainingArgs();
+
+ fs = FileSystem.get(conf);
FSDataOutputStream op = fs.create(new Path(INPUT_FILE), true);
String line1 = "0001,0002\n";
String line2 = "0002,0004\n";
@@ -132,8 +162,7 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
HTableDescriptor desc = new HTableDescriptor(TAB);
desc.addFamily(new HColumnDescriptor(FAM));
HBaseAdmin hbaseAdmin=new HBaseAdmin(conf);
- if (hbaseAdmin.isTableAvailable(TABLE_NAME))
- {
+ if (hbaseAdmin.isTableAvailable(TABLE_NAME)) {
hbaseAdmin.disableTable(TABLE_NAME);
hbaseAdmin.deleteTable(TABLE_NAME);
}
@@ -171,10 +200,17 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
assertNotNull(parentBytes);
assertTrue(parentBytes.length > 0);
Assert.assertEquals("0001", Bytes.toString(parentBytes));
-
} finally {
- cluster.shutdown();
- zkCluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ if (zkCluster != null) {
+ zkCluster.shutdown();
+ }
+ // clean test files
+ if (fs != null) {
+ fs.delete(hbaseRootdir);
+ }
}
}