You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mina.apache.org by ng...@apache.org on 2010/01/13 14:23:44 UTC

svn commit: r898739 - in /mina/sandbox/vysper/branches/nbxml-sax/src: main/java/org/apache/vysper/xml/fragment/ main/java/org/apache/vysper/xml/sax/impl/ test/java/org/apache/vysper/xml/sax/ test/java/org/apache/vysper/xml/sax/impl/

Author: ngn
Date: Wed Jan 13 13:23:44 2010
New Revision: 898739

URL: http://svn.apache.org/viewvc?rev=898739&view=rev
Log:
Move state from tokenizer to the parser
Clean up

Added:
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/AbstractAsyncXMLReaderTestCase.java
      - copied, changed from r896357, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/AbstractAsyncXMLReaderTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/DefaultAsyncXMLReaderTestCase.java
      - copied, changed from r895485, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/DefaultAsyncXMLReaderTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/NameTestCase.java
      - copied, changed from r895495, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/NameTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseAttributesTestCase.java
      - copied, changed from r895481, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseAttributesTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseCommentsTestCase.java
      - copied, changed from r896357, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseCommentsTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseElementsTestCase.java
      - copied, changed from r895076, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseElementsTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseNamespacesTestCase.java
      - copied, changed from r895495, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseNamespacesTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseTextTestCase.java
      - copied, changed from r895076, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseTextTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/TestHandler.java
      - copied, changed from r895076, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/TestHandler.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/XMPPContentHandlerTestCase.java
      - copied, changed from r895485, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/XMPPContentHandlerTestCase.java
Removed:
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/AbstractAsyncXMLReaderTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/DefaultAsyncXMLReaderTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/NameTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseAttributesTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseCommentsTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseElementsTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseNamespacesTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseTextTestCase.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/TestHandler.java
    mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/XMPPContentHandlerTestCase.java
Modified:
    mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/fragment/Renderer.java
    mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/DefaultNonBlockingXMLReader.java
    mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLParser.java
    mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLTokenizer.java

Modified: mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/fragment/Renderer.java
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/fragment/Renderer.java?rev=898739&r1=898738&r2=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/fragment/Renderer.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/fragment/Renderer.java Wed Jan 13 13:23:44 2010
@@ -24,8 +24,6 @@
 import java.util.Map.Entry;
 
 /**
- * TODO support namespaces (inherited from outer/inheriting for inner elements)
- *
  * @author The Apache MINA Project (dev@mina.apache.org)
  */
 public class Renderer {
@@ -65,7 +63,7 @@
     	openElementBuffer.append("<");
         renderElementName(openElementBuffer, element, nsResolver);
         
-        // render namespace delcarations
+        // render namespace declarations
         Map<String, String> nsAttrs = nsResolver.getNamespaceDeclarations();
         for(Entry<String, String> nsAttr : nsAttrs.entrySet()) {
     		openElementBuffer.append(" ");

Modified: mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/DefaultNonBlockingXMLReader.java
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/DefaultNonBlockingXMLReader.java?rev=898739&r1=898738&r2=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/DefaultNonBlockingXMLReader.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/DefaultNonBlockingXMLReader.java Wed Jan 13 13:23:44 2010
@@ -43,7 +43,7 @@
 
 	public static final String FEATURE_NAMESPACES = "http://xml.org/sax/features/namespaces";
 	public static final String FEATURE_NAMESPACE_PREFIXES = "http://xml.org/sax/features/namespace-prefixes";
-	public static final String FEATURE_COMMENTS_FORBIDDEN = "http://mina.apache.org/vysper/features/comments-forbidden";
+	public static final String FEATURE_COMMENTS_ALLOWED = "http://mina.apache.org/vysper/features/comments-allowed";
 	
 	private ErrorHandler errorHandler = new DefaultHandler();
 	private ContentHandler contentHandler = new DefaultHandler();
@@ -56,7 +56,7 @@
 		// set default features
 		features.put(FEATURE_NAMESPACES, true);
 		features.put(FEATURE_NAMESPACE_PREFIXES, false);
-		features.put(FEATURE_COMMENTS_FORBIDDEN, false);
+		features.put(FEATURE_COMMENTS_ALLOWED, true);
 	}
 	
 	/**
@@ -89,8 +89,8 @@
     			// ok
     		} else if(name.equals(FEATURE_NAMESPACE_PREFIXES) && !value) {
     			// ok
-    		} else if(name.equals(FEATURE_COMMENTS_FORBIDDEN)) {
-    			features.put(FEATURE_COMMENTS_FORBIDDEN, value);
+    		} else if(name.equals(FEATURE_COMMENTS_ALLOWED)) {
+    			features.put(FEATURE_COMMENTS_ALLOWED, value);
     		} else {
     			throw new SAXNotSupportedException("Not supported");
     		}

Modified: mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLParser.java
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLParser.java?rev=898739&r1=898738&r2=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLParser.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLParser.java Wed Jan 13 13:23:44 2010
@@ -67,7 +67,16 @@
 		IN_EMPTY_TAG,
 		AFTER_ATTRIBUTE_NAME,
 		AFTER_ATTRIBUTE_EQUALS,
-		IN_COMMENT,
+		AFTER_ATTRIBUTE_FIRST_QUOTE,
+		AFTER_ATTRIBUTE_VALUE,
+		AFTER_COMMENT_BANG,
+		AFTER_COMMENT_DASH1,
+		AFTER_COMMENT_DASH2,
+		AFTER_COMMENT,
+		AFTER_COMMENT_CLOSING_DASH1,
+		AFTER_COMMENT_CLOSING_DASH2,
+		AFTER_COMMENT_ENDING_DASH1,
+		AFTER_COMMENT_ENDING_DASH2,
 		CLOSED
 	}
 	
@@ -81,16 +90,17 @@
 
 	// element names as {uri}qname
 	private Stack<String> elements = new Stack<String>();
+	private boolean sentStartDocument = false; 
 	
 	// features
-	boolean commentsForbidden = false;
+	boolean commentsAllowed = true;
 
 	
 	public XMLParser(ContentHandler contentHandler, ErrorHandler errorHandler, Map<String, Boolean> features) {
 		this.contentHandler = contentHandler;
 		this.errorHandler = errorHandler;
 		
-		commentsForbidden = feature(features, DefaultNonBlockingXMLReader.FEATURE_COMMENTS_FORBIDDEN, false);
+		commentsAllowed = feature(features, DefaultNonBlockingXMLReader.FEATURE_COMMENTS_ALLOWED, true);
 		
 		this.tokenizer = new XMLTokenizer(this);
 	}
@@ -106,13 +116,17 @@
     public void parse(ByteBuffer byteBuffer, CharsetDecoder charsetDecoder) throws SAXException {
     	if(state == State.CLOSED) throw new SAXException("Parser is closed");
     	
-    	tokenizer.parse(byteBuffer, charsetDecoder);
+    	try {
+    		tokenizer.parse(byteBuffer, charsetDecoder);
+    	} catch(RuntimeException e) {
+    		fatalError(e.getMessage());
+    	}
     }
 
 	public void token(char c, String token) throws SAXException {
 		if(log.isDebugEnabled()) {
 			String s = (token == null) ? Character.toString(c) : token;
-			log.debug("Parser got token {} in state {}", s, state);
+			log.trace("Parser got token {} in state {}", s, state);
 		}
 		
 		switch(state) {
@@ -128,16 +142,21 @@
 			// token must be element name or / for a end tag
 			if(c == '/') {
 				state = State.IN_END_TAG;
-			} else if(token.equals("!--")) {
-				if(commentsForbidden) {
+			} else if(c == '!') {
+				if(commentsAllowed) {
+					state = State.AFTER_COMMENT_BANG;
+				} else {
 					fatalError("Comments not allowed");
 					return;
-				} else {
-					state = State.IN_COMMENT;
 				}
 			} else {
-				qname = token;
-				state = State.AFTER_START_NAME;
+				if(isValidName(token)) {
+					qname = token;
+					state = State.AFTER_START_NAME;
+				} else {
+					fatalError("Invalid element name: " + qname);
+					return;
+				}
 			}
 			break;
 		case IN_END_TAG:
@@ -172,9 +191,23 @@
 			}
 			break;
 		case AFTER_ATTRIBUTE_EQUALS:
+			// token must be " or '
+			if(c == '"' || c == '\'') {
+				state = State.AFTER_ATTRIBUTE_FIRST_QUOTE;
+			}
+			break;
+		case AFTER_ATTRIBUTE_FIRST_QUOTE:
 			// token must be attribute value
 			attributes.put(attributeName, unescape(token));
-			state = State.AFTER_START_NAME;
+			state = State.AFTER_ATTRIBUTE_VALUE;
+			break;
+		case AFTER_ATTRIBUTE_VALUE:
+			// token must be " or '
+			if(c == '"' || c == '\'') {
+				state = State.AFTER_START_NAME;
+			} else {
+				fatalError("Not wellformed");
+			}
 			break;
 		case AFTER_END_NAME:
 			// token must be >
@@ -195,9 +228,60 @@
 				}
 			}
 			break;
-		case IN_COMMENT:
-			log.debug("Comment: {}", token);
-			state = State.START;
+		case AFTER_COMMENT_BANG:
+			// token must be -
+			if(c == '-') {
+				state = State.AFTER_COMMENT_DASH1;
+			} else {
+				fatalError("Comment not wellformed");
+				return;
+			}
+			break;
+		case AFTER_COMMENT_DASH1:
+			// token must be -
+			if(c == '-') {
+				state = State.AFTER_COMMENT_DASH2;
+			} else {
+				fatalError("Comment not wellformed");
+				return;
+			}
+			break;
+		case AFTER_COMMENT_DASH2:
+			// we should now get the comment content, ignore
+			if(c == '-') {
+				state = State.AFTER_COMMENT_CLOSING_DASH1;
+			} else {
+				state = State.AFTER_COMMENT;
+			}
+			break;
+		case AFTER_COMMENT:
+			// token must be - or some text
+			if(c == '-') {
+				state = State.AFTER_COMMENT_CLOSING_DASH1;
+			} else if(c == '>') {
+				fatalError("Comment not wellformed");
+				return;
+			} else {
+				// ignore
+			}
+			break;
+		case AFTER_COMMENT_CLOSING_DASH1:
+			// token must be -
+			if(c == '-') {
+				state = State.AFTER_COMMENT_CLOSING_DASH2;
+			} else {
+				fatalError("Comment not wellformed");
+				return;
+			}
+			break;
+		case AFTER_COMMENT_CLOSING_DASH2:
+			// token must be >
+			if(c == '>') {
+				state = State.START;
+			} else {
+				fatalError("Comment not wellformed");
+				return;
+			}
 			break;
 		}
 	}
@@ -210,30 +294,31 @@
 			contentHandler.characters(unescaped.toCharArray(), 0, unescaped.length());
 		} else {
 			// must start document, even that document is not wellformed
-			contentHandler.startDocument();
+			startDocument();
 			fatalError("Text only allowed in element");
 		}
 	}
 	
+	private boolean isValidName(String name) {
+		// element names must only contain valid characters
+        // element names must not begin with "xml" in any casing
+        return NAME_PATTERN.matcher(name).find() && !NAME_PREFIX_PATTERN.matcher(name).find();
+	}
+	
+	private void startDocument() throws SAXException {
+		if(!sentStartDocument) {
+			contentHandler.startDocument();
+			sentStartDocument = true;
+		}
+	}
+	
 	private void startElement() throws SAXException {
 		log.debug("StartElement {}", qname);
 		
 		if(elements.isEmpty()) {
-			contentHandler.startDocument();
+			startDocument();
 		}
 		
-        if(!NAME_PATTERN.matcher(qname).find()) {
-        	fatalError("Invalid element name: " + qname);
-        	return;
-        }
-
-        // element names must not begin with "xml" in any casing
-        if(NAME_PREFIX_PATTERN.matcher(qname).find()) {
-        	fatalError("Names must not start with 'xml': " + qname);
-        	return;
-        }
-
-
 		// find all namespace declarations so we can populate the NS resolver
 		Map<String, String> nsDeclarations = new HashMap<String, String>();
 		for(Entry<String, String> attribute: attributes.entrySet()) {
@@ -347,6 +432,9 @@
 		state = State.CLOSED;
 		tokenizer.close();
 		
+		// make sure we send a start document event
+		startDocument();
+		
 		errorHandler.fatalError(new SAXParseException(message, null));
 	}
 	

Modified: mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLTokenizer.java
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLTokenizer.java?rev=898739&r1=898738&r2=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLTokenizer.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/main/java/org/apache/vysper/xml/sax/impl/XMLTokenizer.java Wed Jan 13 13:23:44 2010
@@ -23,6 +23,7 @@
 import java.nio.charset.CharsetDecoder;
 
 import org.apache.mina.common.ByteBuffer;
+import org.apache.vysper.charset.CharsetUtil;
 import org.xml.sax.SAXException;
 
 /**
@@ -30,16 +31,15 @@
  * @author The Apache MINA Project (dev@mina.apache.org)
  */
 public class XMLTokenizer {
+
+    private static final char NO_CHAR = (char) -1;
 	
 	private enum State { 
 		START, 
 		IN_TAG,
+		IN_STRING,
 		IN_DOUBLE_ATTRIBUTE_VALUE,
 		IN_SINGLE_ATTRIBUTE_VALUE,
-		AFTER_COMMENT_BANG,
-		AFTER_COMMENT_DASH1,
-		AFTER_COMMENT_DASH2,
-		IN_COMMENT,
 		IN_TEXT,
 		CLOSED
 	}
@@ -65,102 +65,67 @@
      */
     public void parse(ByteBuffer byteBuffer, CharsetDecoder decoder) throws SAXException {
         lastPosition = byteBuffer.position();
-//        StringBuffer sb = new StringBuffer();
+
         while (byteBuffer.hasRemaining() && state != State.CLOSED) {
             char c = (char)byteBuffer.get();
 
             if(state == State.START) {
             	if(c == '<') {
-            		emit('<', byteBuffer);
+            		emit(c, byteBuffer);
             		state = State.IN_TAG;
             	} else {
             		state = State.IN_TEXT;
-//            		sb.append(c);
             	}
             } else if(state == State.IN_TEXT) {
             	if(c == '<') {
             		emit(byteBuffer, decoder);
-            		emit('<', byteBuffer);
+            		emit(c, byteBuffer);
             		state = State.IN_TAG;
-            	} else {
-//            		sb.append(c);
             	}
             } else if(state == State.IN_TAG) {
-            	if(c == '/') {
-            		if(checkEmit(byteBuffer)) {
-            			emit(byteBuffer, decoder);
-            		}
-            		emit('/', byteBuffer);
-            	} else if(c == '>') {
-            		if(checkEmit(byteBuffer)) {
-            			emit(byteBuffer, decoder);
-            		}
-                	emit('>', byteBuffer);
-                	state = State.START;
-            	} else if(Character.isWhitespace(c)) {
-            		if(checkEmit(byteBuffer)) {
-            			emit(byteBuffer, decoder);
-            		} else {
-            			// ignore whitespace
-            			lastPosition = byteBuffer.position();
-            		}
-            	} else if(c == '=') {
-            		emit(byteBuffer, decoder);
-            		emit('=', byteBuffer);
+            	if(c == '>') {
+            		emit(c, byteBuffer);
+            		state = State.START;
             	} else if(c == '"') {
-            		lastPosition = byteBuffer.position();
-//            		emit("\"", byteBuffer);
-            		state = State.IN_DOUBLE_ATTRIBUTE_VALUE;
+                		emit(c, byteBuffer);
+                		state = State.IN_DOUBLE_ATTRIBUTE_VALUE;
             	} else if(c == '\'') {
-//            		emit("\'", byteBuffer);
-            		lastPosition = byteBuffer.position();
+            		emit(c, byteBuffer);
             		state = State.IN_SINGLE_ATTRIBUTE_VALUE;
-            	} else if(c == '!') {
-//            		emit("!", byteBuffer);
-            		state = State.AFTER_COMMENT_BANG;
+            	} else if(isControlChar(c)) {
+            		emit(c, byteBuffer);
+            	} else if(Character.isWhitespace(c)) {
+            		lastPosition = byteBuffer.position();
+            	} else {
+            		state = State.IN_STRING;
+            	}
+            } else if(state == State.IN_STRING) {
+            	if(c == '>') {
+            		emit(byteBuffer, CharsetUtil.UTF8_DECODER);
+            		emit(c, byteBuffer);
+            		state = State.START;
+            	} else if(isControlChar(c)) {
+            		emit(byteBuffer, CharsetUtil.UTF8_DECODER);
+            		emit(c, byteBuffer);
+            		state = State.IN_TAG;
+            	} else if(Character.isWhitespace(c)) {
+            		emit(byteBuffer, CharsetUtil.UTF8_DECODER);
+            		state = State.IN_TAG;
             	} else {
-            		// non-whitespace char
+            		// do nothing
             	}
             } else if(state == State.IN_DOUBLE_ATTRIBUTE_VALUE) {
             	if(c == '"') {
             		emit(byteBuffer, decoder);
-//            		emit("\"", byteBuffer);
+            		emit(c, byteBuffer);
             		state = State.IN_TAG;
             	}
             } else if(state == State.IN_SINGLE_ATTRIBUTE_VALUE) {
             	if(c == '\'') {
             		emit(byteBuffer, decoder);
-//            		emit("'", byteBuffer);
+            		emit(c, byteBuffer);
             		state = State.IN_TAG;
             	}
-            } else if(state == State.AFTER_COMMENT_BANG) {
-            	if(c == '-') {
-            		state = State.AFTER_COMMENT_DASH1;
-            	} else {
-            		throw new SAXException("XML not well-formed");
-            	}
-            } else if(state == State.AFTER_COMMENT_DASH1) {
-            	if(c == '-') {
-            		state = State.AFTER_COMMENT_DASH2;
-            	} else {
-            		throw new SAXException("XML not well-formed");
-            	}
-            } else if(state == State.AFTER_COMMENT_DASH2) {
-            	if(Character.isWhitespace(c)) {
-            		emit("!--", byteBuffer);
-            		state = State.IN_COMMENT;
-            		lastPosition = byteBuffer.position();
-            	} else {
-            		throw new SAXException("XML not well-formed");
-            	}
-            } else if(state == State.IN_COMMENT) {
-            	if(Character.isWhitespace(c)) {
-            		emit(byteBuffer, decoder);
-            	} else if(c == '>') {
-            		// TODO handle verification of closing dashes
-            		state = State.START;
-            		lastPosition = byteBuffer.position();
-            	}
             } 
         }
 
@@ -171,8 +136,8 @@
     	state = State.CLOSED;
     }
     
-    private boolean checkEmit(ByteBuffer buffer) {
-    	return buffer.position() > lastPosition + 1;
+    private boolean isControlChar(char c) {
+    	return c == '<' || c == '>' || c == '-' || c == '!' || c == '/' || c == '?' || c == '='; 
     }
 
     private void emit(char token, ByteBuffer byteBuffer) throws SAXException {
@@ -181,14 +146,6 @@
     	lastPosition = byteBuffer.position();
     }
     
-    public static final char NO_CHAR = (char) -1;
-    
-    private void emit(String token, ByteBuffer byteBuffer) throws SAXException {
-    	listener.token(NO_CHAR, token);
-    	
-    	lastPosition = byteBuffer.position();
-    }
-
     private void emit(ByteBuffer byteBuffer, CharsetDecoder decoder) throws SAXException {
     	int endPosition = byteBuffer.position();
     	int oldLimit = byteBuffer.limit();

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/AbstractAsyncXMLReaderTestCase.java (from r896357, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/AbstractAsyncXMLReaderTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/AbstractAsyncXMLReaderTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/AbstractAsyncXMLReaderTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/AbstractAsyncXMLReaderTestCase.java&r1=896357&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/AbstractAsyncXMLReaderTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/AbstractAsyncXMLReaderTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,7 +17,7 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -29,16 +29,17 @@
 
 import org.apache.mina.common.ByteBuffer;
 import org.apache.vysper.charset.CharsetUtil;
-import org.apache.vysper.xml.sax.TestHandler.CharacterEvent;
-import org.apache.vysper.xml.sax.TestHandler.EndDocumentEvent;
-import org.apache.vysper.xml.sax.TestHandler.EndElementEvent;
-import org.apache.vysper.xml.sax.TestHandler.FatalErrorEvent;
-import org.apache.vysper.xml.sax.TestHandler.StartDocumentEvent;
-import org.apache.vysper.xml.sax.TestHandler.StartElementEvent;
-import org.apache.vysper.xml.sax.TestHandler.TestEvent;
+import org.apache.vysper.xml.sax.NonBlockingXMLReader;
 import org.apache.vysper.xml.sax.impl.Attribute;
 import org.apache.vysper.xml.sax.impl.DefaultAttributes;
 import org.apache.vysper.xml.sax.impl.DefaultNonBlockingXMLReader;
+import org.apache.vysper.xml.sax.impl.TestHandler.CharacterEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.EndDocumentEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.EndElementEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.FatalErrorEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.StartDocumentEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.StartElementEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.TestEvent;
 import org.xml.sax.Attributes;
 
 
@@ -93,7 +94,7 @@
 	}
 
 	protected void assertStartDocument(TestEvent actual) {
-		if(!(actual instanceof StartDocumentEvent)) fail("Event must be StartDocumentEvent");
+		if(!(actual instanceof StartDocumentEvent)) fail("Event must be StartDocumentEvent but is " + actual.getClass());
 	}
 
 	protected void assertEndDocument(TestEvent actual) {

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/DefaultAsyncXMLReaderTestCase.java (from r895485, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/DefaultAsyncXMLReaderTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/DefaultAsyncXMLReaderTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/DefaultAsyncXMLReaderTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/DefaultAsyncXMLReaderTestCase.java&r1=895485&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/DefaultAsyncXMLReaderTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/DefaultAsyncXMLReaderTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,10 +17,11 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import org.apache.mina.common.ByteBuffer;
 import org.apache.vysper.charset.CharsetUtil;
+import org.apache.vysper.xml.sax.NonBlockingXMLReader;
 import org.apache.vysper.xml.sax.impl.DefaultNonBlockingXMLReader;
 import org.xml.sax.DTDHandler;
 import org.xml.sax.SAXException;

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/NameTestCase.java (from r895495, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/NameTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/NameTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/NameTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/NameTestCase.java&r1=895495&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/NameTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/NameTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,7 +17,7 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import junit.framework.TestCase;
 

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseAttributesTestCase.java (from r895481, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseAttributesTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseAttributesTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseAttributesTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseAttributesTestCase.java&r1=895481&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseAttributesTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseAttributesTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,12 +17,12 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.Iterator;
 
-import org.apache.vysper.xml.sax.TestHandler.TestEvent;
 import org.apache.vysper.xml.sax.impl.Attribute;
+import org.apache.vysper.xml.sax.impl.TestHandler.TestEvent;
 
 
 

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseCommentsTestCase.java (from r896357, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseCommentsTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseCommentsTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseCommentsTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseCommentsTestCase.java&r1=896357&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseCommentsTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseCommentsTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,13 +17,13 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 
-import org.apache.vysper.xml.sax.TestHandler.TestEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.TestEvent;
 
 
 /**
@@ -66,9 +66,56 @@
 		assertNoMoreevents(events);
 	}
 
+	public void testCommentAdvancedContent() throws Exception {
+		Iterator<TestEvent> events = parse("<root><!-- 3 comment with multiple words --></root>").iterator();
+
+		assertStartDocument(events.next());
+		assertStartElement("", "root", "root", events.next());
+		assertEndElement("", "root", "root", events.next());
+		assertEndDocument(events.next());
+		
+		assertNoMoreevents(events);
+	}
+
+	public void testNotWellformedComment1() throws Exception {
+		Iterator<TestEvent> events = parse("<root><!- comment --></root>").iterator();
+
+		assertStartDocument(events.next());
+		assertStartElement("", "root", "root", events.next());
+		assertFatalError(events.next());
+		assertNoMoreevents(events);
+	}
+
+	public void testNotWellformedComment2() throws Exception {
+		Iterator<TestEvent> events = parse("<root><!-- comment -></root>").iterator();
+
+		assertStartDocument(events.next());
+		assertStartElement("", "root", "root", events.next());
+		assertFatalError(events.next());
+		assertNoMoreevents(events);
+	}
+
+	public void testNotWellformedComment3() throws Exception {
+		Iterator<TestEvent> events = parse("<root><!-- comment ></root>").iterator();
+
+		assertStartDocument(events.next());
+		assertStartElement("", "root", "root", events.next());
+		assertFatalError(events.next());
+		assertNoMoreevents(events);
+	}
+
+	public void testNotWellformedComment4() throws Exception {
+		Iterator<TestEvent> events = parse("<root><! comment --></root>").iterator();
+
+		assertStartDocument(events.next());
+		assertStartElement("", "root", "root", events.next());
+		assertFatalError(events.next());
+		assertNoMoreevents(events);
+	}
+	
 	public void testCommentNotAllowed() throws Exception {
 		Map<String, Boolean> features = new HashMap<String, Boolean>();
-		features.put("http://mina.apache.org/vysper/features/comments-forbidden", true);
+		features.put("http://mina.apache.org/vysper/features/comments-allowed", false);
 		
 		Iterator<TestEvent> events = parse("<root><!-- comment --></root>", features).iterator();
 

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseElementsTestCase.java (from r895076, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseElementsTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseElementsTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseElementsTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseElementsTestCase.java&r1=895076&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseElementsTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseElementsTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,11 +17,11 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.Iterator;
 
-import org.apache.vysper.xml.sax.TestHandler.TestEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.TestEvent;
 
 
 /**

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseNamespacesTestCase.java (from r895495, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseNamespacesTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseNamespacesTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseNamespacesTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseNamespacesTestCase.java&r1=895495&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseNamespacesTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseNamespacesTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,12 +17,12 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.Iterator;
 
-import org.apache.vysper.xml.sax.TestHandler.TestEvent;
 import org.apache.vysper.xml.sax.impl.Attribute;
+import org.apache.vysper.xml.sax.impl.TestHandler.TestEvent;
 
 
 /**

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseTextTestCase.java (from r895076, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseTextTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseTextTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseTextTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseTextTestCase.java&r1=895076&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/ParseTextTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/ParseTextTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,11 +17,11 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.Iterator;
 
-import org.apache.vysper.xml.sax.TestHandler.TestEvent;
+import org.apache.vysper.xml.sax.impl.TestHandler.TestEvent;
 
 
 

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/TestHandler.java (from r895076, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/TestHandler.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/TestHandler.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/TestHandler.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/TestHandler.java&r1=895076&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/TestHandler.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/TestHandler.java Wed Jan 13 13:23:44 2010
@@ -17,7 +17,7 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.ArrayList;
 import java.util.List;

Copied: mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/XMPPContentHandlerTestCase.java (from r895485, mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/XMPPContentHandlerTestCase.java)
URL: http://svn.apache.org/viewvc/mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/XMPPContentHandlerTestCase.java?p2=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/XMPPContentHandlerTestCase.java&p1=mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/XMPPContentHandlerTestCase.java&r1=895485&r2=898739&rev=898739&view=diff
==============================================================================
--- mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/XMPPContentHandlerTestCase.java (original)
+++ mina/sandbox/vysper/branches/nbxml-sax/src/test/java/org/apache/vysper/xml/sax/impl/XMPPContentHandlerTestCase.java Wed Jan 13 13:23:44 2010
@@ -17,7 +17,7 @@
  *  under the License.
  *
  */
-package org.apache.vysper.xml.sax;
+package org.apache.vysper.xml.sax.impl;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -30,6 +30,7 @@
 import org.apache.vysper.xml.decoder.XMPPContentHandler;
 import org.apache.vysper.xml.decoder.XMPPContentHandler.StanzaListener;
 import org.apache.vysper.xml.fragment.XMLElement;
+import org.apache.vysper.xml.sax.NonBlockingXMLReader;
 import org.apache.vysper.xml.sax.impl.DefaultNonBlockingXMLReader;
 
 /**