You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directory.apache.org by ak...@apache.org on 2004/01/30 07:19:41 UTC
svn commit: rev 6362 - in incubator/directory/snickers/trunk/ber: src/java/org/apache/snickers/asn src/java/org/apache/snickers/ber src/java/org/apache/snickers/test xdocs xdocs/empty
Author: akarasulu
Date: Thu Jan 29 22:19:41 2004
New Revision: 6362
Added:
incubator/directory/snickers/trunk/ber/xdocs/
incubator/directory/snickers/trunk/ber/xdocs/design.xml
incubator/directory/snickers/trunk/ber/xdocs/empty/
incubator/directory/snickers/trunk/ber/xdocs/empty/index.xml
incubator/directory/snickers/trunk/ber/xdocs/empty/navigation.xml
incubator/directory/snickers/trunk/ber/xdocs/eureka.xml
incubator/directory/snickers/trunk/ber/xdocs/index.xml
incubator/directory/snickers/trunk/ber/xdocs/navigation.xml
incubator/directory/snickers/trunk/ber/xdocs/news.xml
incubator/directory/snickers/trunk/ber/xdocs/rt.xml
Modified:
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/asn/BitString.java (props changed)
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/BerUtils.java (props changed)
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java (contents, props changed)
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java (contents, props changed)
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java (contents, props changed)
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TypeClass.java (props changed)
incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/test/TestBitString.java (props changed)
Log:
Added keyword expansion for author and revision and some baseline documentation
oh I also trimmed some lines down to 80 characters.
Modified: incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java
==============================================================================
--- incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java (original)
+++ incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java Thu Jan 29 22:19:41 2004
@@ -51,9 +51,11 @@
package org.apache.snickers.ber;
/**
- * @author Wes McKean
- *
* Tag values for primitives
+ *
+ * @author Wes McKean
+ * @author $Author$
+ * @version $Rev$
*/
public interface Primative {
public final int BOOLEAN = 1;
Modified: incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java
==============================================================================
--- incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java (original)
+++ incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java Thu Jan 29 22:19:41 2004
@@ -217,7 +217,8 @@
* @return the id of this encoded value, or 0 if it doesn't have one
*/
public int getId() {
- return ( tag & (Binary.BIT_4 | Binary.BIT_3 | Binary.BIT_2 | Binary.BIT_1 | Binary.BIT_0 ) );
+ return ( tag & (Binary.BIT_4 | Binary.BIT_3 | Binary.BIT_2 |
+ Binary.BIT_1 | Binary.BIT_0 ) );
}
/**
Modified: incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java
==============================================================================
--- incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java (original)
+++ incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java Thu Jan 29 22:19:41 2004
@@ -49,11 +49,13 @@
*/
package org.apache.snickers.ber;
+
import org.apache.commons.codec.DecoderException;
+
/**
- * Basic interface for handling TLVs from the TLVParser. The order of event firing
- * will normally (always) consist of:
+ * Basic interface for handling TLVs from the TLVParser. The order of event
+ * firing will normally (always) consist of:
*
* beginTag, length, data, received, endTag
*
@@ -61,57 +63,76 @@
* @author $Author$
* @version $Rev$
*/
-public interface TLVHandler {
+public interface TLVHandler
+{
/**
* Fired by the TLVParser to indicate that a new TLV is being decoded
*
- * @param typeClass a type class object representing one of the four ASN.1 type classes
- * @param id The encoded structure id. Not all encoded values will have an associated id
+ * @param typeClass a type class object representing one of the four ASN.1
+ * type classes
+ * @param id The encoded structure id. Not all encoded values will have an
+ * associated id
* @param isPrimative a boolean indicating that this value is a primitive
* @throws DecoderException if an error occurs during handling
*/
- public void beginTag( TypeClass typeClass, int id, boolean isPrimative ) throws DecoderException;
+ public void beginTag( TypeClass typeClass, int id, boolean isPrimative )
+ throws DecoderException;
/**
- * Fired by the TLVParser to indicate that the processing of the current TLV has been completed
+ * Fired by the TLVParser to indicate that the processing of the current TLV
+ * has been completed
*
- * @param typeClass a type class object representing one of the four ASN.1 type classes
- * @param id The encoded structure id. Not all encoded values will have an associated id
+ * @param typeClass a type class object representing one of the four ASN.1
+ * type classes
+ * @param id The encoded structure id. Not all encoded values will have
+ * an associated id
* @throws DecoderException if an error occurs during handling
*/
public void endTag( TypeClass typeClass, int id ) throws DecoderException;
/**
- * Fired by the TLVParser to indicate the length of the V portion of the TLV structure. Per the
- * ASN.1 spec, there are three seperate encoding forms: short, long, indefinite. Please note that
- * the length will be zero if the form is indefinite.
+ * Fired by the TLVParser to indicate the length of the V portion of the
+ * TLV structure. Per the ASN.1 spec, there are three seperate encoding
+ * forms: short, long, indefinite. Please note that the length will be
+ * zero if the form is indefinite.
*
- * @param typeClass a type class object representing one of the four ASN.1 type classes
- * @param id The encoded structure id. Not all encoded values will have an associated id
+ * @param typeClass a type class object representing one of the four ASN.1
+ * type classes
+ * @param id The encoded structure id. Not all encoded values will have
+ * an associated id
* @param length the length of the V portion of the TLV structure
* @throws DecoderException if an error occurs during handling
*/
- public void length( TypeClass typeClass, int id, int length ) throws DecoderException;
+ public void length( TypeClass typeClass, int id, int length )
+ throws DecoderException;
/**
- * Fired by the TLVParser when data is received. The data may be all of the required length,
- * or it may be coming over in chunks. In other words, this event may be fired multiple times.
+ * Fired by the TLVParser when data is received. The data may be all of
+ * the required length, or it may be coming over in chunks. In other words,
+ * this event may be fired multiple times.
*
- * @param typeClass a type class object representing one of the four ASN.1 type classes
- * @param id The encoded structure id. Not all encoded values will have an associated id
+ * @param typeClass a type class object representing one of the four ASN.1
+ * type classes
+ * @param id The encoded structure id. Not all encoded values will have an
+ * associated id
* @param data the byte data associated with the value
* @throws DecoderException if an error occurs during handling
*/
- public void data( TypeClass typeClass, int id, byte[] data ) throws DecoderException;
+ public void data( TypeClass typeClass, int id, byte[] data )
+ throws DecoderException;
/**
- * Fired by the TLVParser once the entire TLV packet has been received. This method may
- * need to be removed if we think it is going to be too much of a memory hog.
+ * Fired by the TLVParser once the entire TLV packet has been received.
+ * This method may need to be removed if we think it is going to be too
+ * much of a memory hog.
*
- * @param typeClass a type class object representing one of the four ASN.1 type classes
- * @param id The encoded structure id. Not all encoded values will have an associated id
+ * @param typeClass a type class object representing one of the four
+ * ASN.1 type classes
+ * @param id The encoded structure id. Not all encoded values will have
+ * an associated id
* @param tlv the TLV object which has just finished being parsed.
* @throws DecoderException if an error occurs during handling
*/
- public void received( TypeClass typeClass, int id, TLV tlv ) throws DecoderException;
+ public void received( TypeClass typeClass, int id, TLV tlv )
+ throws DecoderException;
}
Added: incubator/directory/snickers/trunk/ber/xdocs/design.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/design.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <properties>
+ <author email="akarasulu@apache.org">Alex Karasulu</author>
+ <title>Design Documentation</title>
+ </properties>
+ <body>
+ <section name="Design Documentation">
+ <p>Hang tight we'll be working this as we solidify our thoughts.</p>
+ </section>
+ </body>
+</document>
\ No newline at end of file
Added: incubator/directory/snickers/trunk/ber/xdocs/empty/index.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/empty/index.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <properties>
+ <author email="akarasulu@apache.org">Alex Karasulu</author>
+ <title>Empty!</title>
+ </properties>
+ <body>
+ <section name="Empty!">
+ <p>This is an empty place holder for pages one directory above.</p>
+ </section>
+ </body>
+</document>
Added: incubator/directory/snickers/trunk/ber/xdocs/empty/navigation.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/empty/navigation.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project>
+ <title>Apache Directory</title>
+ <body>
+ <links>
+ <item name="Apache" href="http://apache.org/"/>
+ <item name="Snickers"
+ href="http://incubator.apache.org/directory/snickers"/>
+ <item name="Directory" href="http://incubator.apache.org/directory"/>
+ </links>
+
+ <menu name="About Snickers">
+ <item name="top item0" href="/empty/index.html"/>
+ <item name="top item1" href="/empty/index.html"/>
+ <item name="top item2" href="/empty/index.html"/>
+ <item name="BER Library" href="/index.html"/>
+ <item name="top item3" href="/empty/index.html"/>
+ <item name="top item4" href="/empty/index.html"/>
+ </menu>
+ </body>
+</project>
Added: incubator/directory/snickers/trunk/ber/xdocs/eureka.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/eureka.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,929 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <properties>
+ <author email="akarasulu@apache.org">Alex Karasulu</author>
+ <title>Eureka!</title>
+ </properties>
+
+ <body>
+
+ <section name="And we saw the light ... ">
+ <p>
+ One day Wes and Alex started talking about going to town on a new
+ ASN.1 BER Library and here's what happened ...
+ </p>
+ <subsection name="The Conversation">
+ <source>
+[SNIP]
+
+Wes at Work says:
+I've been thinking about the decoding process a bit over the weekend.
+
+Alex Karasulu says:
+k I'm listening
+
+Wes at Work says:
+and encoding.
+
+Wes at Work says:
+I'm not sure at the initial stage there will be *one* decoder.
+
+Wes at Work says:
+We will need some place to hold our TLV tree.
+
+Wes at Work says:
+and also, I was thinking about really long messages.
+
+Alex Karasulu says:
+you need multiple codecs (coder decoders)
+
+Alex Karasulu says:
+right
+
+[SNIP]
+
+Wes at Work says:
+We got one part that builds the tree
+
+Wes at Work says:
+part two should be the translation.
+
+[SNIP]
+
+Wes at Work says:
+I think the only issue we have is how to handle chunking, and blocking versus
+non-blocking code.
+
+Wes at Work says:
+And also, dealing with really huge messages.
+
+Wes at Work says:
+It obviously won't make sense to build a TLV tree in its entirety for a huge
+search result.
+
+Alex Karasulu says:
+right I agree
+
+Alex Karasulu says:
+for encoding there is a mechanism for breaking down large TLVs of simple types
+down
+
+Wes at Work says:
+encoding is a non issue as far as chunking goes.
+
+Alex Karasulu says:
+basically in the book they talk about 3 ways of specifying length
+
+Alex Karasulu says:
+the L part
+
+Alex Karasulu says:
+right but it effects decoding
+
+Alex Karasulu says:
+but if another provider is doing encoding I see what you mean
+
+Alex Karasulu says:
+basically we can break stuff down by injecting the 3rd indeterminate length form
+
+Alex Karasulu says:
+follow me out
+
+Wes at Work says:
+You give the encoder an output interface, and every time it fills up the byte
+bufffer, it spits it out.
+
+Alex Karasulu says:
+Strictly talking about decoding and chunk sizes for now.
+
+Wes at Work says:
+K. decoding then.
+
+Alex Karasulu says:
+just for background - you read the section on the 3 different modes for
+specifying length right: short, long and indeterminant?
+
+[SNIP]
+
+Alex Karasulu says:
+Your reading and encounter a really big simple type using the long encoding for
+L. So you know what you have to read is a hugh blob of data in one big hunk.
+Basically there is some threashold u use to judge whether or not the blob is too
+big and needs to be chopped up.
+
+Wes at Work says:
+I did read the section the length.
+
+Alex Karasulu says:
+cool
+
+Wes at Work says:
+I actually printed the whole appendix out and read it.
+
+Wes at Work says:
+on BER.
+
+Alex Karasulu says:
+cool that's what I was reffering to
+
+Wes at Work says:
+An encoder can choose any one he wants.
+
+Alex Karasulu says:
+Now your decoder can break down the long format into the indeterminate format
+nesting smaller TLVs inside the TLV. Hence converting the simple TLV into a
+constructed one.
+
+Alex Karasulu says:
+The key here is not to keep all the tlvs in memory or the entire encoded buffer
+in memory
+
+Wes at Work says:
+For decoding, there are messages where keeping the intermediate form in memory
+is not an issue, and with others, there are.
+
+Wes at Work says:
+issues.
+
+Alex Karasulu says:
+Right depends on the message size
+
+Wes at Work says:
+The client will want to process most of the messages as a complete object.
+
+Wes at Work says:
+By definition, it will be in memory.
+
+Alex Karasulu says:
+Yeah I know what you are saying. We need to make the library not do this
+though. Then there would be more than one copy in memory. Leave it upto the
+user to determine how the data is dealt with. Eventually we can take messures
+to stream data if we want instead of having it all in memory.
+
+Wes at Work says:
+Back up just a second.
+
+Alex Karasulu says:
+There are funky tactics we can employ way down the road - but for the time
+being lets make it so our codecs dont need massive footprints
+
+Alex Karasulu says:
+sure talk to me
+
+Wes at Work says:
+I used this technique in a Btrieve interface I wrote for U. S. South...
+
+Wes at Work says:
+which I stole from OpenTDS.
+
+Alex Karasulu says:
+Btrieve?
+
+Wes at Work says:
+Yea, an ISAM database.
+
+Alex Karasulu says:
+oh ok
+
+Wes at Work says:
+It used byte buffers to send and retrieve records.
+
+Wes at Work says:
+I wrote a java class that basically treated the byte array as primitives.
+
+Alex Karasulu says:
+cool so you're already of the mindset to keeping the decoding and encoding
+memory footprints small
+
+Wes at Work says:
+That might not work with us though.
+
+Wes at Work says:
+It might.
+
+Wes at Work says:
+All we need to know
+
+Wes at Work says:
+is that this field goes with this TLV.
+
+Wes at Work says:
+and convert it on the fly.
+
+Wes at Work says:
+Also, we an simply dump the TLVs when we are done.
+
+Alex Karasulu says:
+yeah that's part of some tables we may need to maintain with a mappiung
+
+Alex Karasulu says:
+right I think we're on the same page
+
+Alex Karasulu says:
+I have a small idea though
+
+Alex Karasulu says:
+Basically wrt the codec's interfaces
+
+Alex Karasulu says:
+To me you give an array of bytes in a byte[] or a ByteBuffer (this is the
+delivered partial chunk) and you get back a set of TLVs for that chunk.
+
+Alex Karasulu says:
+or take it in the opposite direction for a encoder
+
+Alex Karasulu says:
+this is your stage 1 (BER bytes ->TLVs)
+
+Alex Karasulu says:
+now we need to find a way to represent TLVs in a linear fashion and still
+maintain the tree structure. However we don't want direct back references
+to where the list of TLVs plug into the entire tree because this would mean
+we have to have the whole tree in memory.
+
+Alex Karasulu says:
+does that make sense I know its a lil nebulous
+
+Wes at Work says:
+Keep it simple
+
+Alex Karasulu says:
+ok in decoding bytes go in and TLVs come out
+
+Wes at Work says:
+Right.
+
+Alex Karasulu says:
+state is maintained between times u pump in bytes
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+Yup.
+
+Alex Karasulu says:
+now the TLVs comming out are a peice of the TLV tree
+
+Wes at Work says:
+You got to be able to handle partial Ts, Ls, and Vs.
+
+Alex Karasulu says:
+right that's part of the state stuff
+
+Alex Karasulu says:
+if you're stuck in the middle of a simple tlv then you don't pump it out until
+the chunks to complete it have arrived
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+right.
+
+Alex Karasulu says:
+So the key here is to have the right TLV represntation or data structure. We
+have some requirements on this.
+
+Alex Karasulu says:
+the TLVs that come out of the decoder cannot directly, with java references,
+refer to other TLVs that came out before. Because these references would
+require the entire TLV tree in memory.
+
+Alex Karasulu says:
+This is one of those requirements you agree?
+
+Wes at Work says:
+I don't see that being an issue.
+
+Wes at Work says:
+The parent needs to know about the children, but not vis a versa.
+
+Alex Karasulu says:
+right
+
+Wes at Work says:
+and I don't see how you are going to be able to assemble an ASN.1 message in a
+state driven fashion without making it very complicated.
+
+Alex Karasulu says:
+that's our primary issue here
+
+Wes at Work says:
+and have two decoders hooked together as well.
+
+Alex Karasulu says:
+its a big problem to overcome
+
+Alex Karasulu says:
+and do it elegantly
+
+Alex Karasulu says:
+If we do this then our BER ASN.1 codec will be hot working in a non-blocking
+fashion and being very efficient. It's like the way SAX is used for reading
+XML for our ASN.1 messages instead of using DOM.
+
+Alex Karasulu says:
+the ideas are similar
+
+Alex Karasulu says:
+you didn't think this was gonna be a cake walk did ya
+
+Wes at Work says:
+Hmmmm.
+
+Alex Karasulu says:
+you do understand where I was coming from wit the sax and dom stuff right?
+
+Wes at Work says:
+yea.
+
+Wes at Work says:
+That I understand.
+
+Alex Karasulu says:
+do you think its possible?
+
+Wes at Work says:
+So you have an event driven ASN.1 parser.
+
+Wes at Work says:
+I think that's still easy.
+
+Wes at Work says:
+However, assembling them into the messages is still complicated.
+
+Wes at Work says:
+every ASN.1 message type would have to be derived from our parser.
+
+Wes at Work says:
+Then a factory could create the message type based on the application type.
+
+Alex Karasulu says:
+hmmm
+
+Alex Karasulu says:
+what do you mean by: "every ASN.1 message type would have to be derived from
+our parser.
+
+Wes at Work says:
+You want the ASN.1 messages to be able to assemble themselves? or no.
+
+Alex Karasulu says:
+Now you're talking about using the ASN.1 specification like a DTD to drive the
+decoding
+
+Alex Karasulu says:
+?
+
+Alex Karasulu says:
+Yep I see yes
+
+Alex Karasulu says:
+u use the ASN.1 spec or a set of classes generated by an ASN.1 spec compiler
+
+Alex Karasulu says:
+question is do we need a compiler now?
+
+Wes at Work says:
+Right.
+
+Wes at Work says:
+Factory returns the ASN.1 message on the application tag.
+
+Alex Karasulu says:
+right I see where your going with the design
+
+Wes at Work says:
+the parser then passes everything to the ASN.reader interface,
+
+Wes at Work says:
+SAX like.
+
+Alex Karasulu says:
+Hmm sounds like it should be very possible
+
+Wes at Work says:
+of the application object.
+
+Wes at Work says:
+who knows how to assemble himself.
+
+Alex Karasulu says:
+right
+
+Alex Karasulu says:
+This is huge
+
+Alex Karasulu says:
+I wonder if other ASN.1 tools have this sax like mechanism already in place.
+
+Wes at Work says:
+But how do we handle ASN.1 messages which need to be streamed.
+
+Wes at Work says:
+like a huge search result.
+
+Alex Karasulu says:
+that's not so much the issue
+
+Alex Karasulu says:
+a large result set takes n+2 messages
+
+Alex Karasulu says:
+sorry n+1
+
+Wes at Work says:
+You have a search result tight.
+
+Wes at Work says:
+Tag = Applicationz
+Length = 00
+Value = Search Results
+
+Wes at Work says:
+Now V is made up of thousands of result messages.
+
+Alex Karasulu says:
+In the LDAP protocol a search result is returned as n+1 messages.
+
+Alex Karasulu says:
+each result is an SearchEntryResponse for the 'n' and one SearchDoneResponse
+PDU to end the resultset
+
+Alex Karasulu says:
+n+1 messages
+
+Wes at Work says:
+Ah.
+
+Wes at Work says:
+But are they wrapped in an application TLV?
+
+Alex Karasulu says:
+but think of a large blob of data
+
+Wes at Work says:
+or is it just one stream of TLVs.
+
+Alex Karasulu says:
+like say some binary chunk
+
+Alex Karasulu says:
+the application TLV for each response type is in the LDAP message envelope.
+There is a top level LDAP message type which is a TLV then the different
+response types have you know some enumeration values to determine which
+response type the top level envelope or application TLV represents
+
+Wes at Work says:
+Right.
+
+Alex Karasulu says:
+but your question is valid for say a single SearchEntryResponse where one of
+the attributes is a huge binary chunk
+
+Wes at Work says:
+So the event firing for the top level envelope will be different than the TLVs
+which are part of the envelope.
+
+Alex Karasulu says:
+the top level LDAPMessage envelope defined for the LDAP asn.1 will be a
+constructred TLV
+
+Alex Karasulu says:
+event might fire for it
+
+Alex Karasulu says:
+same one every time
+
+Wes at Work says:
+Right, but not after the entire TLV is read into memry.
+
+Wes at Work says:
+that would defeat our SAX based parser.
+
+Alex Karasulu says:
+but its constitution will change depending on the type of message it is
+
+Alex Karasulu says:
+right
+
+Alex Karasulu says:
+exactly
+
+Wes at Work says:
+I'm with you.
+
+Alex Karasulu says:
+you would get a start_ldap_message event
+
+Wes at Work says:
+Actually,
+
+Wes at Work says:
+for the envelope, you would need to hit the factory.
+
+Wes at Work says:
+to get the appropriate LDAP message.
+
+Alex Karasulu says:
+then perhaps the message_type_event will fire to note the contained TLV that
+specifies the LDAP application's message type.
+
+Alex Karasulu says:
+et. cetera. see where i'm going with it - you don't need the entire message to
+fire its arrival. Like sax where you say start tag for this element then the
+contained elemenets then close tags etc.
+
+Wes at Work says:
+Got ya.
+
+Wes at Work says:
+I think that's pretty cool.
+
+Alex Karasulu says:
+I think we're getting somewhere cool here I'm very excited. I need to take
+another look at a sax implementation again out there. It will give me some
+insight into some possible general architecture for us.
+
+Alex Karasulu says:
+Now going back to the massive chunk of binary. So we have a
+SearchEntryResponse with an entry of the result set containing an attribute
+that is a huge binary chunk. How do we stream it out right? Then we can
+talk about how we stream it in.
+
+Alex Karasulu says:
+Streaming it out is easy. Let's for a moment presume that we can actually
+stream out of the jdbm stuff. You basically convert the long known length BER
+encoding to the indeterminant encoding. Then send out individual chunks of
+this binary attribute in separate TLVs. So you're turning big assed primitive
+TLVs into constructed TLVs chunking out the content hence not needing the
+entire V in memor
+
+Alex Karasulu says:
+y.
+
+Wes at Work says:
+That's fine for us. We have control over the encoding.
+
+Wes at Work says:
+We won't be so lucky on the inbound side.
+
+Alex Karasulu says:
+Right
+
+Alex Karasulu says:
+Now let's think about that beast.
+
+Alex Karasulu says:
+We have a binary -> tlv encoder spitting out tlvs with each bit of input
+
+Alex Karasulu says:
+meant decoder above sorry
+
+Alex Karasulu says:
+now if the indeterminate length is used by the client when encoding and
+sending to the server the server is ok the data is already chopped up and
+its all good. If not and the long length encoding is used then the data
+comes into the server's decoder in chunks but the decoder sees a hugh
+long length.
+
+Alex Karasulu says:
+Based on some threshold the decoder translates the incoming long length and
+values for the simple type (primitive TLV) into a constructed TLV breaking
+up the large know length TLV into the indeterminant form which can be spit
+out with a few nested TLVs at a time (with each input chunk going into the
+decoder).
+
+Alex Karasulu says:
+You follow? Decoder automatically breaks up large primitive long length encoded
+TLVs into the indeterminate form and spits those out in peices rather than the
+one large primitive TLV.
+
+Wes at Work says:
+What does that buy us?
+
+Alex Karasulu says:
+streaming
+
+Wes at Work says:
+Is not the ASN message gonna re-assemble it anyways.
+
+Wes at Work says:
+Do you still end up with 200K picture in the ASN.1 message.
+
+Alex Karasulu says:
+yeah that's application specific - remember we're talking just the BER->TLV
+codec
+
+Alex Karasulu says:
+the other codec is Type to TLV
+
+Wes at Work says:
+If we are using a SAX based parser, then the Type will be assembling itself as
+the TLVs are decoded and fired.
+
+Alex Karasulu says:
+keeping it streaming means you don't have 2X the data or 400K in use just to
+get the 200K picture
+
+Alex Karasulu says:
+right
+
+Wes at Work says:
+At some point, you are going to have to put your faith in the garbage collector.
+
+Alex Karasulu says:
+right but that's not in the codec BER to TLV code
+
+Alex Karasulu says:
+keep that lean and mean - why you ask
+
+Wes at Work says:
+Also, if you want a truly small memory footprint, then you could put stuff like
+that in a small embedded database.
+
+Alex Karasulu says:
+well the TLV to Type code can be made lean and mean too
+
+Wes at Work says:
+I just don't think at this stage that we need to be all that worried about huge
+blocks of binary data.
+
+Alex Karasulu says:
+right we use referrals to data on disk to manage large peices of data that
+needsto be streamed but this we can do later.
+
+Wes at Work says:
+Exactly.
+
+Alex Karasulu says:
+yes but we want the options to be open - right now we can just design the
+interfaces so all this can be added later.
+
+Alex Karasulu says:
+Interfaces and contracts should be designed to allow these very low memory
+footprints. Thinking through the process and what it takes to get there
+makes us understand better what the design and interfaces should look like.
+
+Alex Karasulu says:
+I don't care if the first implementation is a hog
+
+Wes at Work says:
+The BER stuff today doesn't deal with this.
+
+Wes at Work says:
+It doesn't care.
+
+Alex Karasulu says:
+for large peices of data
+
+Wes at Work says:
+It's an application issue.
+
+Alex Karasulu says:
+right
+
+Alex Karasulu says:
+what the app does with it is upto the app but lets keep the ber codecs low in
+memory image regardless of the fact that some app will be a pig and stream the
+data into memory anyway. This is all that I'm trying to say.
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+K.
+
+Alex Karasulu says:
+cool we're tight on this but I think it will take more research on both our
+parts - anyway apache is back up again after a power failure. Here's the new
+stuff I created for ya:
+http://cvs.apache.org/viewcvs.cgi/incubator/directory/snickers/?root=Apache-SVN
+
+Alex Karasulu says:
+that's the top level of the snickers (snacc replacement) subproject
+
+Alex Karasulu says:
+that's all you and Jeff with the C based version of this thang
+
+Wes at Work says:
+Right.
+
+Wes at Work says:
+You won't find much other ASN.1 stuff out there.
+
+Wes at Work says:
+I'm comfortable that no one is doing it this way, either.
+
+Wes at Work says:
+It will make it unqiuely, Apache.
+
+Alex Karasulu says:
+Ok. Let's touch base in a day or two to regroup
+
+Wes at Work says:
+Do you think ASN.1 is going to die?
+
+Alex Karasulu says:
+this is all good stuff and I'll try to get it out there.
+
+Alex Karasulu says:
+no way
+
+Alex Karasulu says:
+ASN.1 is awesome stuff
+
+Wes at Work says:
+We'll see.
+
+Alex Karasulu says:
+SNMP is based on it and so is Kerberose
+
+Alex Karasulu says:
+what's the alternative?
+
+Wes at Work says:
+XML is what everyone is using now.
+
+Alex Karasulu says:
+well there is XER for ASN.1
+
+Alex Karasulu says:
+XML Encoding Rules
+
+Alex Karasulu says:
+ASN.1 can go to BER, PER, XER, and DER
+
+Wes at Work says:
+Yes.
+
+Alex Karasulu says:
+the encoding does not effect the ASN.1 specification and that is what makes
+ASN.1 a winner always.
+
+Wes at Work says:
+Slapping XML on ASN.1 ain't the same.
+
+Alex Karasulu says:
+the XML format is just for the encoding of the data types
+
+Wes at Work says:
+I agree that ASN.1 is a good protocol.
+
+Alex Karasulu says:
+protocol specification syntax
+
+Alex Karasulu says:
+it kicks ass I think and is here to stay.
+
+Wes at Work says:
+If we do this, we are going to go backwards right?
+
+Wes at Work says:
+Do the compiler last.
+
+Alex Karasulu says:
+go backwards?
+
+Alex Karasulu says:
+yeah that might be the case or we can work it together.
+
+Wes at Work says:
+You need to let me work this.
+
+Alex Karasulu says:
+I can do the compiler with you and you can handle the runtime
+
+Wes at Work says:
+You got other things to do.
+
+Alex Karasulu says:
+ok its all you then
+
+Alex Karasulu says:
+I'm just a follower
+
+Wes at Work says:
+I won't mind help with the compiler.
+
+Wes at Work says:
+Just don't get going on it any time soon
+
+Alex Karasulu says:
+sure I have extensive javacc and antlr experience
+
+Wes at Work says:
+Deal.
+
+Alex Karasulu says:
+hehe no worries with that my plate as you know is overflowing.
+
+Alex Karasulu says:
+my bladder too
+
+Alex Karasulu says:
+I'll catch ya later I need to hit the head
+
+Wes at Work says:
+Talk about the decoder's stream.
+
+Wes at Work says:
+K
+
+Alex Karasulu says:
+ttyl
+
+Wes at Work says:
+Talk later then.
+
+Alex Karasulu says:
+ok gimme 45 seconds
+
+Alex Karasulu says:
+I'm back
+
+Alex Karasulu says:
+what about the decoder's stream.
+
+Wes at Work says:
+So, how do we feed the decoder then.
+
+Alex Karasulu says:
+Its all about how we design our interfaces. You know I've been looking at
+commons-codec and see some potential but changes will be needed.
+
+Alex Karasulu says:
+Follow me for a sec.
+
+Alex Karasulu says:
+Now the codec interfaces are designed to convert stuff in one shot.
+bytes in bytes out sort of thang. Very blocking dependent stuff and not very
+cool for us with a SEDA and NIO based server.
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+right.
+
+Alex Karasulu says:
+As you might have guessed this is not good for servers that need to keep
+memory footprints low while servicing possible serveral hundred requests
+per second.
+
+Alex Karasulu says:
+So what do we do? We design new non-blocking and NIO based interfaces for the
+codec API and submit them.
+
+Alex Karasulu says:
+its down again damn
+
+Wes at Work says:
+I got my update
+
+Alex Karasulu says:
+cool
+
+Wes at Work says:
+Must of brought it down.
+
+Alex Karasulu says:
+yeah maybe it will be up soon
+
+Alex Karasulu says:
+anyway
+
+Alex Karasulu says:
+We redesign these codec interfaces to manage an encoding session and a decoding
+session so chunks can be process in a stateful manner to be conducive to
+non-blocking use.
+
+Alex Karasulu says:
+Or we use events like you said
+
+Alex Karasulu says:
+Basically we contribute this to the commons stuff and make sure the community
+understands why and what we're doing. That way they can double check us.
+
+Alex Karasulu says:
+Then we use those interfaces to implement the ASN.1 stuff.
+
+Wes at Work says:
+Right.
+
+Alex Karasulu says:
+We do this in the snickers area but put back as much into the commons codec as
+we can. You game with this strategy?
+
+Wes at Work says:
+Yea, that's fine.
+
+Wes at Work says:
+I'll check out commons code as soon as it comes up.
+
+</source>
+ </subsection>
+ </section>
+ </body>
+</document>
Added: incubator/directory/snickers/trunk/ber/xdocs/index.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/index.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <properties>
+ <author email="akarasulu@apache.org">Alex Karasulu</author>
+ <title>Snickers ASN.1 BER Library</title>
+ </properties>
+ <body>
+ <section name="News and Information">
+ <ul>
+ <li>
+ 26-JAN-2004: <a href="news.html">Created Snickers Sub-Project</a>
+ </li>
+ </ul>
+ </section>
+ <section name="What is it and why is it called Snickers?">
+ <p>
+ The Basic Encoding Rules (BER) Library is a simple encoding for Abstract
+ Syntax Notation One (ASN.1) defined data structures. It replaces the
+ SNACC4J runtime libraries and adds some enhancements. It's designed
+ for efficiency and speed specifically for use in SEDA based protocol
+ servers using a BER codec for their Protocol Data Units (PDU).
+ </p>
+ <p>
+ Originally we used the SNACC4J compiler to generate stubs that knew how
+ to encode and decode themselves. Due to licensing issues we had to
+ abandon the use of SNACC4J. Rather than reimplement it we decided to
+ innovate on top of the algorithms used by Sample and Neufeld (the
+ original creators of the first free SNAC compiler).
+ </p>
+ <p>
+ A member of the directory team, Robb Penoyer, thought it was funny to
+ call whatever replaced the SNACC4J runtime, Snickers. We think he was
+ hungry when he came up with this one. Since then the name Snickers has
+ stuck to the belly of the directory project and refers to everything
+ dealing with ASN.1.
+ </p>
+ </section>
+ <section name="How will it work?">
+ <p>
+ The BER encoder and decoder (codec) will be designed to operate very
+ much like the Simple API for XML (SAX). It will generate events which
+ are calls on a handler as it encounters low level encoded structures
+ called Tag Length Value (TLV) triplets.
+ </p>
+ <p>
+ Rather than return values, which could be extremely large, in one peice
+ the decoder for example returns peices of a value until it completes
+ processing the entire V of the TLV. This makes the decoder highly
+ attractive for servers using non-blocking IO and SocketChannels. This
+ design gives it a small decoding footprint regardless of the size of
+ the Protocol Data Unit (PDU) being processed. It also makes it much
+ faster since the decoder deals with a small simple task without much
+ conditional logic for processing a PDU. We hope the combined benefits
+ of non-blocking IO and this sleek codec, make any BER based protocol
+ server extremely responsive under heavy loads with massive concurrency.
+ </p>
+ </section>
+ <section name="How to use the API">
+ <p>
+ Fill this in once we have an API.
+ </p>
+ <subsection name="Some Sub-Topic">
+ <p>
+ Fill this in.
+ </p>
+ </subsection>
+ <subsection name="General Resources">
+ <ul>
+ <li><a href="rt.html">Random Thoughts</a></li>
+ <li><a href="eureka.html">Eureka!</a></li>
+ </ul>
+ </subsection>
+ </section>
+ </body>
+</document>
Added: incubator/directory/snickers/trunk/ber/xdocs/navigation.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/navigation.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project>
+ <title>Apache Directory</title>
+ <body>
+ <links>
+ <item name="Apache" href="http://apache.org/"/>
+ <item name="Snickers"
+ href="http://incubator.apache.org/directory/snickers"/>
+ <item name="Directory" href="http://incubator.apache.org/directory"/>
+ </links>
+ <menu name="About Snickers">
+ <item name="top item0" href="/empty/index.html"/>
+ <item name="top item1" href="/empty/index.html"/>
+ <item name="top item2" href="/empty/index.html"/>
+ <item name="BER Library" href="/index.html">
+ <item name="Overview" href="/index.html"/>
+ <item name="Design" href="/design.html"/>
+ <item name="Latest News" href="/news.html"/>
+ <item name="Random Thoughts" href="/rt.html"/>
+ <item name="Eureka" href="/eureka.html"/>
+ <item name="Sample's Original SNACC"
+ href="http://www.apache.org/~akarasulu/resources/ASN.1/snacc.pdf"/>
+ </item>
+ <item name="top item3" href="/empty/index.html"/>
+ <item name="top item4" href="/empty/index.html"/>
+ </menu>
+ </body>
+</project>
Added: incubator/directory/snickers/trunk/ber/xdocs/news.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/news.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <properties>
+ <author email="akarasulu@apache.org">Alex Karasulu</author>
+ <title>Snickers BER Library News</title>
+ </properties>
+
+ <body>
+
+ <section name="News and Information">
+
+ <!-- insert new news items as a subsection here -->
+
+ <subsection name="26-JAN-2004: Created Snickers Sub-Project">
+ <p> We realized that IBM's SNACC just wont do a long time ago due to
+ licensing restrictions. Anyway IBM had taken the original SNACC
+ algorithm which was open and closed the source so it grew cobwebs. It
+ was time for something better. We soon came to the realization that we
+ would have to due better than just a reimplementation of the original
+ SNACC generated codecs. </p>
+ <p>
+ BTW SNACC is an acronym for the Sample, Neufeld ASN.1 to C/C++ compiler.
+ Although this part of snickers is not an ASN.1 to C/C++ compiler it does
+ deal with the part of the generated code that decodes and encodes ASN.1
+ based data structures to and from the basic encoding.
+ </p>
+ <p>
+ The problem with the snacc approach to encoding and decoding ASN.1
+ structures rests in the fact that an entire copy of the structure must
+ be kept in memory while it is encoded and finishes decoding. Still the
+ processing may require even more memory while handling this process
+ depending on the TLV count to average value size ratio of the PDU.
+ </p>
+ <p>
+ We decided to devise our own mechanism for encoding and decoding BER
+ based PDUs. Our approach minimizes the memory footprints while decoding
+ and encoding messages without requiring the entire memory footprint of
+ the PDU in memory. This is what gave birth to the snickers project.
+ </p>
+ </subsection>
+
+ </section>
+ </body>
+</document>
Added: incubator/directory/snickers/trunk/ber/xdocs/rt.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/rt.xml Thu Jan 29 22:19:41 2004
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <properties>
+ <author email="akarasulu@apache.org">Alex Karasulu</author>
+ <title>Random Thoughts</title>
+ </properties>
+
+ <body>
+ <section name="Wes' Random Thoughts">
+ <subsection name="26-JAN-2004: Some Thoughts Here :-)">
+ <p>
+ Put em here!
+ </p>
+ </subsection>
+ </section>
+ <section name="Alex's Random Thoughts">
+ <subsection name="26-JAN-2004: Some Thoughts Here :-)">
+ <p>
+ Put em here!
+ </p>
+ </subsection>
+ </section>
+ <section name="Jeff's Random Thoughts">
+ <subsection name="26-JAN-2004: Some Thoughts Here :-)">
+ <p>
+ Put em here!
+ </p>
+ </subsection>
+ </section>
+ </body>
+</document>