You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directory.apache.org by ka...@apache.org on 2013/12/03 07:17:26 UTC
svn commit: r1547297 - in /directory/apacheds/trunk/mavibot-partition/src:
main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/
test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/
Author: kayyagari
Date: Tue Dec 3 06:17:25 2013
New Revision: 1547297
URL: http://svn.apache.org/r1547297
Log:
intiial commit related to bulk loading data into a mavibot partition
Added:
directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java
directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java
directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java
Added: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java?rev=1547297&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java (added)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java Tue Dec 3 06:17:25 2013
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.directory.server.core.partition.impl.btree.mavibot;
+
+import java.util.Comparator;
+
+import org.apache.directory.api.ldap.model.name.Dn;
+import org.apache.directory.mavibot.btree.Tuple;
+
+/**
+ * TODO LdifTupleComparator.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public class LdifTupleComparator implements Comparator<Tuple<Dn, String>>
+{
+
+ @Override
+ public int compare( Tuple<Dn, String> t1, Tuple<Dn, String> t2 )
+ {
+ Dn dn1 = t1.getKey();
+ Dn dn2 = t2.getKey();
+
+ if( dn1.isAncestorOf( dn2 ) )
+ {
+ return -1;
+ }
+ else if( dn2.isAncestorOf( dn1 ) )
+ {
+ return 1;
+ }
+ else if ( dn1.equals( dn2 ) )
+ {
+ return 0;
+ }
+
+ return dn1.getNormName().compareTo( dn2.getNormName() );
+ }
+
+}
Added: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java?rev=1547297&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java (added)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java Tue Dec 3 06:17:25 2013
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.directory.server.core.partition.impl.btree.mavibot;
+
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.directory.api.ldap.model.ldif.LdifEntry;
+import org.apache.directory.api.ldap.model.ldif.LdifReader;
+import org.apache.directory.api.ldap.model.name.Dn;
+import org.apache.directory.mavibot.btree.Tuple;
+import org.apache.directory.mavibot.btree.util.TupleReaderWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO LdifTupleReaderWriter.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public class LdifTupleReaderWriter implements TupleReaderWriter<Dn, String>
+{
+
+ private LdifReader reader = null;
+
+ private static final Logger LOG = LoggerFactory.getLogger( LdifTupleReaderWriter.class );
+
+
+ @Override
+ public Tuple<Dn, String> readSortedTuple( DataInputStream in )
+ {
+ try
+ {
+ if( in.available() > 0 )
+ {
+ Tuple<Dn, String> t = new Tuple<Dn, String>();
+ t.setKey( new Dn( in.readUTF() ) );
+ t.setValue( in.readUTF() );
+
+ return t;
+ }
+ }
+ catch ( Exception e )
+ {
+ LOG.error( "Failed to read a sorted tuple", e );
+ }
+
+ return null;
+ }
+
+
+ @Override
+ public Tuple<Dn, String> readUnsortedTuple( DataInputStream in )
+ {
+ try
+ {
+ if ( reader == null )
+ {
+ reader = new LdifReader( in );
+ }
+ }
+ catch ( Exception e )
+ {
+ String msg = "Failed to open the LDIF input stream";
+ LOG.error( msg, e );
+
+ throw new RuntimeException( msg, e );
+ }
+
+ Tuple<Dn, String> t = null;
+
+ if ( reader.hasNext() )
+ {
+ LdifEntry ldifEntry = reader.next();
+
+ if ( ldifEntry == null )
+ {
+ throw new IllegalStateException(
+ "Received null entry while parsing, check the LDIF file for possible incorrect/corrupted entries" );
+ }
+
+ t = new Tuple<Dn, String>();
+
+ t.setKey( ldifEntry.getDn() );
+ t.setValue( ldifEntry.getOffset() + ":" + ldifEntry.getLengthBeforeParsing() );
+ }
+
+ return t;
+ }
+
+
+ @Override
+ public void storeSortedTuple( Tuple<Dn, String> t, DataOutputStream out ) throws IOException
+ {
+ out.writeUTF( t.getKey().getName() );
+ out.writeUTF( t.getValue() );
+ }
+
+}
Added: directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java?rev=1547297&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java (added)
+++ directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java Tue Dec 3 06:17:25 2013
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.directory.server.core.partition.impl.btree.mavibot;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.util.Iterator;
+
+import org.apache.directory.api.ldap.model.name.Dn;
+import org.apache.directory.api.ldap.model.name.Rdn;
+import org.apache.directory.mavibot.btree.Tuple;
+import org.apache.directory.mavibot.btree.managed.BulkDataSorter;
+import org.junit.Ignore;
+
+/**
+ * TODO LdifBulkLoaderTest.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+@Ignore
+public class LdifBulkLoaderTest
+{
+ String personTemplate = "# just a comment\n"+
+ "dn: uid={uid},{parent}\n" +
+ "objectClass: top\n" +
+ "objectClass: person\n" +
+ "objectClass: organizationalPerson\n" +
+ "objectClass: inetOrgPerson\n" +
+ "givenName: {uid}_{uid}\n" +
+ "sn: {uid}_sn\n" +
+ "cn: {uid}_cn\n" +
+ "uid: {uid}\n\n";
+
+ String ouTemplate = "# just another comment\n"+
+ "dn: {ouDn}\n" +
+ "objectclass: top\n" +
+ "objectclass: organizationalUnit\n" +
+ "ou: {ou}\n\n";
+
+ private void sort() throws Exception
+ {
+ LdifTupleReaderWriter readerWriter = new LdifTupleReaderWriter();
+ LdifTupleComparator tupleComparator = new LdifTupleComparator();
+
+ BulkDataSorter bs = new BulkDataSorter<Dn, String>( readerWriter, tupleComparator, 2 );
+
+ File file = createLdif();
+
+ bs.sort( file );
+
+ Iterator<Tuple<Dn, String>> itr = bs.getMergeSortedTuples();
+ while( itr.hasNext() )
+ {
+ Tuple t = itr.next();
+ System.out.println( t.getKey() + " - " + t.getValue() );
+ }
+
+ //BTreeBuilder<, V>
+ }
+
+ private File createLdif() throws Exception
+ {
+ File file = File.createTempFile( "bulkload-test", ".ldif" );
+ file.deleteOnExit();
+
+ FileWriter fw = new FileWriter( file );
+
+ Dn parentDn = new Dn( "ou=grandchildren,ou=children,ou=parent,ou=system" );
+
+ Dn currentDn = parentDn;
+
+ for( Rdn rdn : parentDn.getRdns() )
+ {
+ for( int i =0; i< 2; i++ )
+ {
+ String user = personTemplate.replace( "{uid}", "user"+i );
+ user = user.replace( "{parent}", currentDn.getName() );
+
+ fw.write( user );
+ }
+
+
+ String userBranch = ouTemplate.replace( "{ou}", rdn.getValue().getString() );
+ userBranch = userBranch.replace( "{ouDn}", currentDn.getName() );
+
+ fw.write( userBranch );
+
+ currentDn = currentDn.getParent();
+ }
+
+ fw.close();
+
+ return file;
+ }
+
+ public static void main( String[] args ) throws Exception
+ {
+ LdifBulkLoaderTest bl = new LdifBulkLoaderTest();
+ bl.sort();
+ }
+}