You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2011/08/03 18:52:03 UTC
svn commit: r1153579 -
/incubator/jena/Jena2/ARQ/trunk/src/org/openjena/riot/tokens/TokenizerText.java
Author: andy
Date: Wed Aug 3 16:52:02 2011
New Revision: 1153579
URL: http://svn.apache.org/viewvc?rev=1153579&view=rev
Log:
Handle symbols more carefully.
Modified:
incubator/jena/Jena2/ARQ/trunk/src/org/openjena/riot/tokens/TokenizerText.java
Modified: incubator/jena/Jena2/ARQ/trunk/src/org/openjena/riot/tokens/TokenizerText.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/ARQ/trunk/src/org/openjena/riot/tokens/TokenizerText.java?rev=1153579&r1=1153578&r2=1153579&view=diff
==============================================================================
--- incubator/jena/Jena2/ARQ/trunk/src/org/openjena/riot/tokens/TokenizerText.java (original)
+++ incubator/jena/Jena2/ARQ/trunk/src/org/openjena/riot/tokens/TokenizerText.java Wed Aug 3 16:52:02 2011
@@ -15,6 +15,7 @@ import org.openjena.atlas.AtlasException
import org.openjena.atlas.io.IO ;
import org.openjena.atlas.io.PeekReader ;
import org.openjena.riot.RiotParseException ;
+import org.openjena.riot.system.RiotChars ;
/** Tokenizer for all sorts of things RDF-ish */
@@ -247,7 +248,10 @@ public final class TokenizerText impleme
ch = reader.readChar() ;
if ( ch == EOF )
exception("EOF found after "+CTRL_CHAR) ;
- token.cntrlCode = (char)ch ;
+ if ( RiotChars.isWhitespace(ch) )
+ token.cntrlCode = -1 ;
+ else
+ token.cntrlCode = (char)ch ;
if ( Checking ) checkControl(token.cntrlCode) ;
return token ;
}