You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directory.apache.org by se...@apache.org on 2010/02/05 00:03:49 UTC
svn commit: r906716 - in /directory/sandbox/seelmann/hbase-partition: ./
src/main/java/org/apache/directory/server/core/partition/hbase/
src/main/java/org/apache/directory/server/core/partition/hbase/table/
src/test/java/org/apache/directory/server/cor...
Author: seelmann
Date: Thu Feb 4 23:03:48 2010
New Revision: 906716
URL: http://svn.apache.org/viewvc?rev=906716&view=rev
Log:
Fixed embedded HBase partition
Modified:
directory/sandbox/seelmann/hbase-partition/pom.xml
directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java
directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java
directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java
directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java
directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java
directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java
Modified: directory/sandbox/seelmann/hbase-partition/pom.xml
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/pom.xml?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/pom.xml (original)
+++ directory/sandbox/seelmann/hbase-partition/pom.xml Thu Feb 4 23:03:48 2010
@@ -97,38 +97,32 @@
<groupId>org.apache.directory.hbase</groupId>
<artifactId>hadoop-test</artifactId>
<version>0.20.1</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.directory.hbase</groupId>
<artifactId>hbase-test</artifactId>
<version>0.20.3-RC3</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.directory.hbase</groupId>
<artifactId>zookeeper</artifactId>
<version>3.2.2</version>
- <scope>test</scope>
</dependency>
<!-- HBase transitive test depencencies -->
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.4</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.0.1</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java (original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java Thu Feb 4 23:03:48 2010
@@ -67,7 +67,10 @@
public Cache( int size, int ttl )
{
this.ttl = ttl;
- this.map = new LRUMap( size );
+ if ( size > 0 )
+ {
+ this.map = new LRUMap( size );
+ }
}
@@ -79,6 +82,11 @@
*/
public synchronized void put( K key, V value )
{
+ if ( map == null )
+ {
+ return;
+ }
+
if ( value == null )
{
map.remove( key );
@@ -99,6 +107,11 @@
*/
public synchronized boolean contains( K key )
{
+ if ( map == null )
+ {
+ return false;
+ }
+
return get( key ) != null;
}
@@ -112,6 +125,11 @@
*/
public synchronized V get( K key )
{
+ if ( map == null )
+ {
+ return null;
+ }
+
Entry entry = ( Entry ) map.get( key );
if ( entry != null )
{
@@ -137,6 +155,11 @@
*/
public synchronized void clear()
{
+ if ( map == null )
+ {
+ return;
+ }
+
map.clear();
}
Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java (original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java Thu Feb 4 23:03:48 2010
@@ -26,8 +26,11 @@
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,67 +48,131 @@
private static final Logger LOG = LoggerFactory.getLogger( EmbeddedHBase.class );
- protected File workingDirectory;
- protected HBaseConfiguration conf;
- protected MiniHBaseCluster cluster;
- protected MiniZooKeeperCluster zooKeeperCluster;
+ private static final EmbeddedHBase INSTANCE = new EmbeddedHBase();
+
+ private File workingDirectory;
+ private HBaseConfiguration conf;
+ private MiniHBaseCluster cluster;
+ private MiniZooKeeperCluster zooKeeperCluster;
+ private int count = 0;
+
+
+ private EmbeddedHBase()
+ {
+ }
/**
- * Creates a new instance of EmbeddedHBase.
- * Note that the configuration is modified to use the local file system
- * rather than an DFS.
+ * Gets a EmbeddedHBase instance.
*
* @param workingDirectory the working directory
- * @param conf the HBase configuration
* @throws Exception
*/
- public EmbeddedHBase( File workingDirectory, HBaseConfiguration conf ) throws Exception
+ public static EmbeddedHBase getInstance( File workingDirectory )
{
- this.workingDirectory = workingDirectory;
- this.conf = conf;
-
- // use local filesystem
- conf.set( "fs.default.name", "file:///" );
- // zookeeper working directory
- conf.set( "test.build.data", new File( workingDirectory, "zookeeper" ).getAbsolutePath() );
- // hbase working directory
- conf.set( HConstants.HBASE_DIR, new File( workingDirectory, "hbase" ).toURI().toURL().toString() );
+ synchronized ( INSTANCE )
+ {
+ if ( INSTANCE.workingDirectory == null )
+ {
+ INSTANCE.workingDirectory = workingDirectory;
+ INSTANCE.conf = new HBaseConfiguration();
+ }
+ return INSTANCE;
+ }
}
/**
* Starts the embedded HBase cluster.
*
+ * @param tableName the table name of the table to wait for to become ready.
+ *
* @throws Exception the exception
*/
- public void start() throws Exception
+ public synchronized void start( String tableName ) throws Exception
{
- try
+ if ( count == 0 )
{
- // Note that this is done before we create the MiniHBaseCluster because we
- // need to edit the config to add the ZooKeeper servers.
- zooKeeperCluster = new MiniZooKeeperCluster();
- int clientPort = this.zooKeeperCluster.startup( workingDirectory );
- conf.set( "hbase.zookeeper.property.clientPort", Integer.toString( clientPort ) );
+ LOG.info( "Start embedded HBase" );
+ try
+ {
+ // hdfs working directory
+ System.setProperty( "test.build.data", workingDirectory.getAbsolutePath() );
+ // zookeeper working directory
+ conf.set( "test.build.data", new File( workingDirectory, "zookeeper" ).getAbsolutePath() );
+ // use local filesystem
+ conf.set( "fs.default.name", "file:///" );
+ // hbase working directory
+ conf.set( HConstants.HBASE_DIR, new File( workingDirectory, "hbase" ).toURI().toURL().toString() );
+
+ // Note that this is done before we create the MiniHBaseCluster because we
+ // need to edit the config to add the ZooKeeper servers.
+ zooKeeperCluster = new MiniZooKeeperCluster();
+ int clientPort = this.zooKeeperCluster.startup( workingDirectory );
+ conf.set( "hbase.zookeeper.property.clientPort", Integer.toString( clientPort ) );
- // start the mini cluster
- cluster = new MiniHBaseCluster( conf, 1 );
+ // start the mini cluster
+ cluster = new MiniHBaseCluster( conf, 1 );
+ }
+ catch ( Exception e )
+ {
+ LOG.error( "Error while starting embedded HBase.", e );
+ if ( cluster != null )
+ {
+ cluster.shutdown();
+ }
+ if ( zooKeeperCluster != null )
+ {
+ zooKeeperCluster.shutdown();
+ }
+ throw e;
+ }
}
- catch ( Exception e )
+
+ // increment counter
+ count++;
+
+ /*
+ * Wait till the tree table is ready.
+ */
+ LOG.debug( "Waiting for table {} to become ready...", tableName );
+
+ // remember the current retry settings
+ String retriesNumber = conf.get( "hbase.client.retries.number" );
+ String pause = conf.get( "hbase.client.pause" );
+ // set some extended retry settings, 2 minutes should be enough...
+ conf.set( "hbase.client.retries.number", "60" ); // number of retries
+ conf.set( "hbase.client.pause", "2000" ); // pause in milli seconds
+ try
{
- LOG.error( "Exception in start.", e );
- if ( cluster != null )
+ HBaseAdmin admin = new HBaseAdmin( conf );
+
+ if ( tableName != null && admin.tableExists( tableName ) )
{
- cluster.shutdown();
+ HTable table = new HTable( conf, tableName );
+ Get get = new Get( Bytes.toBytes( 0L ) );
+ table.get( get );
+ table.close();
+ LOG.debug( "Table {} is ready.", tableName );
}
- if ( zooKeeperCluster != null )
+ else
{
- zooKeeperCluster.shutdown();
+ LOG.debug( "Table {} doesn't exist, seems to be a clean start.", tableName );
}
+ }
+ catch ( Exception e )
+ {
+ e.printStackTrace();
+ LOG.error( "Error while waiting for table to become ready.", e );
throw e;
}
+ finally
+ {
+ // reset the retry settings
+ conf.set( "hbase.client.retries.number", retriesNumber );
+ conf.set( "hbase.client.pause", pause );
+ }
}
@@ -114,18 +181,33 @@
*
* @throws Exception the exception
*/
- public void shutdown() throws Exception
+ public synchronized void shutdown() throws Exception
{
+ if ( count > 1 )
+ {
+ // dont't stop the cluster till the last shutdown.
+ count--;
+ return;
+ }
+
try
{
+ LOG.info( "Shutdown embedded HBase" );
+ count = 0;
+
// open the META table now to ensure cluster is running before shutdown.
new HTable( conf, HConstants.META_TABLE_NAME );
HConnectionManager.deleteConnectionInfo( conf, true );
if ( this.cluster != null )
{
+ LOG.debug( "Shutdown cluster..." );
this.cluster.shutdown();
+ LOG.debug( "...done" );
+
+ LOG.debug( "Shutdown zookeeper..." );
this.zooKeeperCluster.shutdown();
+ LOG.debug( "...done" );
}
}
catch ( Exception e )
@@ -134,4 +216,15 @@
}
}
+
+ /**
+ * Gets the HBase configuration.
+ *
+ * @return the HBase configuration
+ */
+ public HBaseConfiguration getConfiguration()
+ {
+ return conf;
+ }
+
}
Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java (original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java Thu Feb 4 23:03:48 2010
@@ -92,12 +92,15 @@
public void init( SchemaManager schemaManager ) throws Exception
{
- this.configuration = new HBaseConfiguration();
if ( workingDirectory != null )
{
- // note that the configuration is modified to use the local file system
- this.embeddedHBase = new EmbeddedHBase( workingDirectory, configuration );
- this.embeddedHBase.start();
+ embeddedHBase = EmbeddedHBase.getInstance( workingDirectory );
+ embeddedHBase.start( tablePrefix + HBaseMasterTable.TREE_TABLE );
+ configuration = embeddedHBase.getConfiguration();
+ }
+ else
+ {
+ configuration = new HBaseConfiguration();
}
this.schemaManager = schemaManager;
@@ -139,11 +142,6 @@
public void destroy() throws Exception
{
- if ( embeddedHBase != null )
- {
- embeddedHBase.shutdown();
- embeddedHBase = null;
- }
if ( masterTable != null )
{
masterTable.close();
@@ -164,6 +162,12 @@
oneAliasIndex.close();
subAliasIndex.close();
+ if ( embeddedHBase != null )
+ {
+ embeddedHBase.shutdown();
+ embeddedHBase = null;
+ }
+
schemaManager = null;
}
Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java (original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java Thu Feb 4 23:03:48 2010
@@ -84,8 +84,8 @@
public static final byte[] NORM_ATTRIBUTE_DELIMITER_BYTES = new byte[]
{ NORM_ATTRIBUTE_DELIMITER };
- private static final String MASTER_TABLE = "master";
- private static final String TREE_TABLE = "tree";
+ public static final String MASTER_TABLE = "master";
+ public static final String TREE_TABLE = "tree";
private HBaseConfiguration configuration;
private SchemaManager schemaManager;
Modified: directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java (original)
+++ directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java Thu Feb 4 23:03:48 2010
@@ -51,9 +51,12 @@
super( 1, startDfs );
setName( clazz.getName() );
- // use target as test directory base
+ // use target as test directory base...
File testDir = new File( "target/data" );
+ // ...for zookeeper
conf.set( TEST_DIRECTORY_KEY, testDir.getAbsolutePath() );
+ // ...for hdfs
+ System.setProperty( "test.build.data", testDir.getAbsolutePath() );
// setup local file system if no DFS is used
if ( !startDfs )
Modified: directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java (original)
+++ directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java Thu Feb 4 23:03:48 2010
@@ -19,8 +19,6 @@
*/
package org.apache.directory.server.core.partition.hbase;
-import java.util.Set;
-
import org.apache.directory.server.annotations.CreateLdapServer;
import org.apache.directory.server.annotations.CreateTransport;
import org.apache.directory.server.core.CoreSession;
@@ -29,7 +27,6 @@
import org.apache.directory.server.core.annotations.CreatePartition;
import org.apache.directory.server.core.integ.AbstractLdapTestUnit;
import org.apache.directory.server.core.integ.FrameworkRunner;
-import org.apache.directory.server.core.partition.Partition;
import org.apache.directory.server.core.partition.hbase.index.HBaseUserColumnIndex;
import org.apache.directory.server.core.partition.hbase.index.HBaseUserRowIndex;
import org.apache.directory.server.core.partition.hbase.it.AbstractHBasePartitionIT;
@@ -41,8 +38,8 @@
/**
- * Starts up an ApacheDS LDAP server with an HBase partition.
- * The partition starts up an embedded HBase partition.
+ * Starts up an ApacheDS LDAP server with two HBase partitions.
+ * The partitions starts up one embedded HBase partition.
*
* @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
* @version $Rev$, $Date$
@@ -59,22 +56,22 @@
type = HBaseEmbeddedPartition.class,
cacheSize=1000,
indexes = {
- @CreateIndex( attribute="cn", cacheSize=100, type = HBaseUserColumnIndex.class ),
- @CreateIndex( attribute="uid", cacheSize=100, type = HBaseUserColumnIndex.class ),
+ @CreateIndex( attribute="cn", cacheSize=1000, type = HBaseUserColumnIndex.class ),
+ @CreateIndex( attribute="uid", cacheSize=1000, type = HBaseUserColumnIndex.class ),
@CreateIndex( attribute="dc", cacheSize=10, type = HBaseUserRowIndex.class ),
@CreateIndex( attribute="o", cacheSize=10, type = HBaseUserRowIndex.class ),
- @CreateIndex( attribute="ou", cacheSize=100, type = HBaseUserRowIndex.class ),
+ @CreateIndex( attribute="ou", cacheSize=10, type = HBaseUserRowIndex.class ),
- @CreateIndex( attribute="objectClass", cacheSize=100, type = HBaseUserRowIndex.class ),
+ @CreateIndex( attribute="objectClass", cacheSize=100, type = HBaseUserRowIndex.class )
}
)
-// ,
-// @CreatePartition(
-// name = "sevenSeas",
-// suffix = "o=sevenSeas",
-// type = HBaseEmbeddedPartition.class
-// )
+ ,
+ @CreatePartition(
+ name = "sevenSeas",
+ suffix = "o=sevenSeas",
+ type = HBaseEmbeddedPartition.class
+ )
})
@CreateLdapServer(transports =
{ @CreateTransport(protocol = "LDAP", port = 10389, nbThreads=16) })
@@ -88,8 +85,6 @@
public void initTestData() throws Exception
{
session = ldapServer.getDirectoryService().getAdminSession();
- Set<? extends Partition> partitions = ldapServer.getDirectoryService().getPartitions();
- System.out.println(partitions);
if ( !session.exists( new LdapDN( "o=hbase" ) ) || !session.exists( new LdapDN( "ou=test-ou,o=hbase" ) )
|| !session.exists( new LdapDN( "cn=test-person,ou=test-ou,o=hbase" ) ) )