svn commit: r1590598 - in /jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang: TestLangNTriples.java TestLangNTuples.java

[rv...@apache.org]

Author: rvesse
Date: Mon Apr 28 10:55:50 2014
New Revision: 1590598

URL: http://svn.apache.org/r1590598
Log:
Tests to show that you can enforce the stricter ASCII mode for NTriples when desired (JENA-688)

Modified:
    jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java
    jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java

Modified: jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java?rev=1590598&r1=1590597&r2=1590598&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java (original)
+++ jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java Mon Apr 28 10:55:50 2014
@@ -23,9 +23,11 @@ import java.io.StringReader ;
 import org.apache.jena.atlas.lib.StrUtils ;
 import org.apache.jena.riot.RDFLanguages ;
 import org.apache.jena.riot.RDFDataMgr ;
+import org.apache.jena.riot.RiotException;
 import org.apache.jena.riot.RiotReader ;
 import org.apache.jena.riot.ErrorHandlerTestLib.ErrorHandlerEx ;
 import org.apache.jena.riot.ErrorHandlerTestLib.ExFatal ;
+import org.apache.jena.riot.out.CharSpace;
 import org.apache.jena.riot.system.StreamRDF ;
 import org.apache.jena.riot.system.StreamRDFLib ;
 import org.apache.jena.riot.tokens.Tokenizer ;
@@ -89,11 +91,21 @@ public class TestLangNTriples extends Te
         parseCount("<x> <p> \"é\" .") ; 
     }
     
+    @Test(expected = RiotException.class)
+    public void nt_only_5b()
+    {
+        parseCount(CharSpace.ASCII, "<x> <p> \"é\" .") ; 
+    }
+    
     @Override
-    protected long parseCount(String... strings)
+    protected long parseCount(String... strings) {
+        return parseCount(CharSpace.UTF8, strings);
+    }
+    
+    private long parseCount(CharSpace charSpace, String... strings)
     {
         String string = StrUtils.strjoin("\n", strings) ;
-        Tokenizer tokenizer = tokenizer(string) ;
+        Tokenizer tokenizer = tokenizer(charSpace, string) ;
         StreamRDFCounting sink = StreamRDFLib.count() ;
         LangNTriples x = RiotReader.createParserNTriples(tokenizer, sink) ;
         x.getProfile().setHandler(new ErrorHandlerEx()) ;

Modified: jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java?rev=1590598&r1=1590597&r2=1590598&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java (original)
+++ jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java Mon Apr 28 10:55:50 2014
@@ -19,14 +19,16 @@
 package org.apache.jena.riot.lang;
 
 import java.io.ByteArrayInputStream ;
+import java.nio.charset.Charset;
 
 import org.apache.jena.atlas.junit.BaseTest ;
 import org.apache.jena.atlas.lib.StrUtils ;
-import org.apache.jena.riot.ErrorHandlerTestLib.ErrorHandlerEx ;
-import org.apache.jena.riot.ErrorHandlerTestLib.ExError ;
-import org.apache.jena.riot.ErrorHandlerTestLib.ExFatal ;
-import org.apache.jena.riot.ErrorHandlerTestLib.ExWarning ;
+import org.apache.jena.riot.ErrorHandlerTestLib.ErrorHandlerEx;
+import org.apache.jena.riot.ErrorHandlerTestLib.ExError;
+import org.apache.jena.riot.ErrorHandlerTestLib.ExFatal;
+import org.apache.jena.riot.ErrorHandlerTestLib.ExWarning;
 import org.apache.jena.riot.* ;
+import org.apache.jena.riot.out.CharSpace;
 import org.apache.jena.riot.system.RiotLib ;
 import org.apache.jena.riot.system.StreamRDF ;
 import org.apache.jena.riot.system.StreamRDFLib ;
@@ -154,6 +156,14 @@ abstract public class TestLangNTuples ex
         parseCheck("<http://example/é> <http://example/p> \"é\" .") ; 
     }
     
+    static protected Tokenizer tokenizer(CharSpace charSpace, String string)
+    {
+        byte b[] = StrUtils.asUTF8bytes(string) ;
+        ByteArrayInputStream in = new ByteArrayInputStream(b) ;
+        Tokenizer tokenizer = charSpace == CharSpace.ASCII ? TokenizerFactory.makeTokenizerASCII(in) : TokenizerFactory.makeTokenizerUTF8(in) ;
+        return tokenizer ;
+    }
+    
     static protected Tokenizer tokenizer(String string)
     {
         // UTF-8