You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2026/05/26 12:34:55 UTC
(jena) branch main updated (fbff38a1dd -> 12c0dfb57d)
This is an automated email from the ASF dual-hosted git repository.
afs pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/jena.git
from fbff38a1dd GH-3941: Fixed spliterator characteristics
new aaaf6ee590 GH-3931: No Surrogates - RIOT
new 12c0dfb57d GH-3931: No surrogates - SPARQL
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/main/java/org/apache/jena/query/ARQ.java | 51 ++--
.../src/main/java/org/apache/jena/riot/RIOT.java | 5 +
.../org/apache/jena/riot/tokens/TokenizerText.java | 121 +++------
...nizerText.java => TokenizerTextSurrogates.java} | 35 ++-
.../org/apache/jena/riot/lang/TestLangTurtle.java | 14 +-
.../AbstractTestTokenizerTextSurrogates.java | 293 +++++++++++++++++++++
.../org/apache/jena/riot/tokens/TS_Tokens.java | 6 +-
.../apache/jena/riot/tokens/TestTokenizerText.java | 257 +-----------------
.../TestTokenizerTextAllowEscSurrogates.java | 80 ++++++
.../tokens/TestTokenizerTextNoSurrogates.java} | 45 +---
.../sparql/expr/TestSPARQLKeywordFunctions.java | 25 +-
.../apache/jena/sparql/syntax/TestQueryParser.java | 32 ++-
.../main/java/org/apache/jena/atlas/lib/Chars.java | 7 +
.../java/org/apache/jena/atlas/lib/EscapeStr.java | 61 ++---
.../org/apache/jena/atlas/lib/TestEscapeStr.java | 175 ++++++------
15 files changed, 664 insertions(+), 543 deletions(-)
copy jena-arq/src/main/java/org/apache/jena/riot/tokens/{TokenizerText.java => TokenizerTextSurrogates.java} (97%)
create mode 100644 jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java
create mode 100644 jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java
copy jena-arq/src/{main/java/org/apache/jena/riot/tokens/TokenizerWrapper.java => test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java} (55%)
(jena) 02/02: GH-3931: No surrogates - SPARQL
Posted by an...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
afs pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/jena.git
commit 12c0dfb57da36a0fef8378d362d3876fcdb36372
Author: Andy Seaborne <an...@apache.org>
AuthorDate: Fri May 22 08:32:25 2026 +0100
GH-3931: No surrogates - SPARQL
---
.../sparql/expr/TestSPARQLKeywordFunctions.java | 25 ++-
.../apache/jena/sparql/syntax/TestQueryParser.java | 32 ++--
.../java/org/apache/jena/atlas/lib/EscapeStr.java | 61 +++----
.../org/apache/jena/atlas/lib/TestEscapeStr.java | 175 ++++++++++++---------
4 files changed, 160 insertions(+), 133 deletions(-)
diff --git a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java
index 4502fe4639..529f392260 100644
--- a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java
+++ b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java
@@ -24,7 +24,6 @@ package org.apache.jena.sparql.expr;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -199,17 +198,15 @@ public class TestSPARQLKeywordFunctions
// As surrogate pair: 0xD83D 0xDC6A
// Written here in forms which protect against binary file corruption.
- //@Test public void substr_30() { test("substr('👪', 1)", "'👪'"); }
+ @Test public void substr_30() { test("substr('👪', 1)", "'👪'"); }
+ // Written using \-u escapes in SPARQL. Tests for supplemental codepoint handling.
+ @Test public void substr_31() { test("substr('\uD83D\uDC6A', 1)", "'\uD83D\uDC6A'"); }
+ @Test public void substr_32() { test("substr('\uD83D\uDC6A', 2)", kwEmptyString); }
+ @Test public void substr_33() { test("substr('ABC\uD83D\uDC6ADEF', 4, 1)", "'\uD83D\uDC6A'"); }
+ @Test public void substr_34() { test("substr('\uD83D\uDC6A!', -1, 3)", "'\uD83D\uDC6A'"); }
+ @Test public void substr_35() { test("substr('\uD83D\uDC6A!', -1, 4)", "'\uD83D\uDC6A!'"); }
- // Written using \-u escapes in SPARQL.
- @Test public void substr_30() { test("substr('\\uD83D\\uDC6A', 1)", "'\\uD83D\\uDC6A'"); }
- // Same using Java string escapes.
- @Test public void substr_30b() { test("substr('\uD83D\uDC6A', 1)", "'\uD83D\uDC6A'"); }
- @Test public void substr_31() { test("substr('\\uD83D\\uDC6A', 2)", kwEmptyString); }
-
- @Test public void substr_32() { test("substr('ABC\\uD83D\\uDC6ADEF', 4, 1)", "'\\uD83D\\uDC6A'"); }
- @Test public void substr_33() { test("substr('\\uD83D\\uDC6A!', -1, 3)", "'\\uD83D\\uDC6A'"); }
- @Test public void substr_34() { test("substr('\\uD83D\\uDC6A!', -1, 4)", "'\\uD83D\\uDC6A!'"); }
+ // Now (1.2) numeric escape sequences do not allow surrogates. See TestQueryParser.
// STRLEN
@Test public void strlen_01() { test("strlen('abc')", "3"); }
@@ -701,10 +698,6 @@ public class TestSPARQLKeywordFunctions
private void testEvalException(String exprStr) {
Expr expr = ExprUtils.parse(exprStr);
- try {
- NodeValue r = expr.eval(null, LibTestExpr.createTest());
- fail("No exception raised");
- }
- catch (ExprEvalException ex) {}
+ assertThrows(ExprEvalException.class,()->expr.eval(null, LibTestExpr.createTest()));
}
}
diff --git a/jena-arq/src/test/java/org/apache/jena/sparql/syntax/TestQueryParser.java b/jena-arq/src/test/java/org/apache/jena/sparql/syntax/TestQueryParser.java
index 8d16dd1b03..dd40139037 100644
--- a/jena-arq/src/test/java/org/apache/jena/sparql/syntax/TestQueryParser.java
+++ b/jena-arq/src/test/java/org/apache/jena/sparql/syntax/TestQueryParser.java
@@ -45,39 +45,51 @@ public class TestQueryParser {
LogCtl.withLevel(loggerSPARQL, "fatal", action);
}
- // Single backslash so a Java string escape, raw surrogate in the string.
+ // These are U+0001F46A 👪 - FAMILY
+ // As surrogate pair: 0xD83D 0xDC6A
+
+ @Test
+ public void syntax_unicode_raw_uri() {
+ testParse("SELECT * { <http://example/👪> ?p ?o}");
+ }
+
+ @Test
+ public void syntax_unicode_raw_string() {
+ testParse("SELECT * { ?s ?p '👪'}");
+ }
+
+ // Single backslash so a Java string escape, surrogate in the string.
@Test
public void syntax_unicode_raw_surrogate_uri() {
- QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { <http://example/\uD800> ?p ?o}"));
+ QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { <http://example/\uD83D> ?p ?o}"));
assertTrue(ex.getMessage().contains("surrogate"));
}
@Test
public void syntax_unicode_raw_surrogate_string() {
- QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { ?s ?p '\uD800' }"));
+ QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { ?s ?p '\uDC6A' }"));
assertTrue(ex.getMessage().contains("surrogate"));
}
// Double backslash so the query string has an escape in it.
@Test
public void syntax_unicode_escaped_surrogate_uri() {
- QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { <http://example/\\uD800> ?p ?o}"));
+ QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { <http://example/\\uD83D> ?p ?o}"));
assertTrue(ex.getMessage().contains("surrogate"));
}
@Test
public void syntax_unicode_escaped_surrogate_strings() {
- QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { ?s ?p '\\uD800'}"));
+ QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { ?s ?p '\\uD83D'}"));
assertTrue(ex.getMessage().contains("surrogate"));
}
@Test
public void syntax_unicode_surrogate_pair_by_unicode_escape() {
- // Allow - because Java strings may have surrogate pairs so we allow them in unicode escapes if paired.
- testParse("SELECT * { ?s ?p '\\uD801\\uDC37'}");
-
-// QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { ?s ?p '\\uD801\\uDC37'}"));
-// assertTrue(ex.getMessage().contains("surrogate"));
+// // Allow - because Java strings may have surrogate pairs so we allow them in unicode escapes if paired.
+// testParse("SELECT * { ?s ?p '\\uD801\\uDC37'}");
+ QueryParseException ex = assertThrows(QueryParseException.class, ()->testParse("SELECT * { ?s ?p '\\uD801\\uDC37'}"));
+ assertTrue(ex.getMessage().contains("Surrogate"));
}
private static void testParse(String string) {
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/EscapeStr.java b/jena-base/src/main/java/org/apache/jena/atlas/lib/EscapeStr.java
index 5c2dde137d..61cf515af8 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/lib/EscapeStr.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/EscapeStr.java
@@ -21,6 +21,8 @@
package org.apache.jena.atlas.lib;
+import static org.apache.jena.atlas.lib.Chars.isSurrogate;
+
import org.apache.jena.atlas.AtlasException;
import org.apache.jena.atlas.io.AWriter;
import org.apache.jena.atlas.io.OutputUtils;
@@ -222,7 +224,7 @@ public class EscapeStr
char ch = s.charAt(i);
if ( ch != escape ) {
- sb.append(ch);
+ insertCodepoint(sb, ch);
continue;
}
@@ -244,7 +246,10 @@ public class EscapeStr
if ( i+4 >= s.length() )
throw new AtlasException("\\u escape too short");
int x4 = Hex.hexStringToInt(s, i+1, 4);
- sb.append((char)x4);
+ if ( isSurrogate(x4) )
+ throw new AtlasException(String.format("Surrogate code point in \\u sequence. Value: 0x%04X", x4));
+ // Do direct.
+ insertCodepoint(sb, (char)x4);
// Jump 1 2 3 4 -- already skipped \ and u
i = i+4;
continue;
@@ -253,19 +258,9 @@ public class EscapeStr
if ( i+8 >= s.length() )
throw new AtlasException("\\U escape too short");
int ch8 = Hex.hexStringToInt(s, i+1, 8);
- if ( Character.charCount(ch8) == 1 )
- sb.append((char)ch8);
- else {
- // See also TokenerText.insertCodepoint and TokenerText.readUnicodeEscape
- // Convert to UTF-16. Note that the rest of any system this is used
- // in must also respect codepoints and surrogate pairs.
- if ( !Character.isDefined(ch8) && !Character.isSupplementaryCodePoint(ch8) )
- throw new AtlasException(String.format("Illegal codepoint: 0x%04X", ch8));
- if ( ch8 > Character.MAX_CODE_POINT )
- throw new AtlasException(String.format("Illegal code point in \\U sequence value: 0x%08X", ch8));
- char[] chars = Character.toChars(ch8);
- sb.append(chars);
- }
+ if ( isSurrogate(ch8) )
+ throw new AtlasException(String.format("Surrogate code point in \\U sequence. Value: 0x%08X", ch8));
+ insertCodepoint(sb, ch8);
// Jump 1 2 3 4 5 6 7 8 -- already skipped \ and U
i = i+8;
continue;
@@ -275,8 +270,8 @@ public class EscapeStr
// If so, \X-anything else is legal as a literal "\" and "X"
if ( pointCodeOnly ) {
- sb.append('\\');
- sb.append(ch2);
+ insertCodepoint(sb, '\\');
+ insertCodepoint(sb, ch2);
continue;
}
@@ -294,7 +289,7 @@ public class EscapeStr
}
if ( actualCh != 0 ) {
- sb.append(actualCh);
+ insertCodepoint(sb, actualCh);
continue;
}
@@ -336,20 +331,26 @@ public class EscapeStr
}
if ( j == 0 )
throw new AtlasException("Empty \\u{} sequence");
+ if ( isSurrogate(value) )
+ throw new AtlasException(String.format("Surrogate code point in \\u{} sequence: 0x%08X", value));
+ insertCodepoint(sb, value);
+ // Looking at the closing '}'
+ i = i+j;
+ return i;
+ }
- int ch8 = value;
- if ( Character.charCount(ch8) == 1 )
- sb.append((char)ch8);
+ private static void insertCodepoint(StringBuilder sb, int cp) {
+ if ( Character.charCount(cp) == 1 )
+ sb.append((char)cp);
else {
- if ( !Character.isDefined(ch8) && !Character.isSupplementaryCodePoint(ch8) )
- throw new AtlasException(String.format("Illegal codepoint: 0x%04X", ch8));
- if ( ch8 > Character.MAX_CODE_POINT )
- throw new AtlasException(String.format("Illegal code point in \\u{..} sequence value: 0x%08X", ch8));
- char[] chars = Character.toChars(ch8);
+ if ( !Character.isDefined(cp) && !Character.isSupplementaryCodePoint(cp) )
+ throw new AtlasException(String.format("Illegal codepoint: 0x%04X", cp));
+ char[] chars = Character.toChars(cp);
sb.append(chars);
}
- // Looking at the closing '}'
- i = i+j;
- return i;
}
-}
\ No newline at end of file
+
+ private static void insertCodepoint(StringBuilder sb, char cp) {
+ sb.append(cp);
+ }
+}
diff --git a/jena-base/src/test/java/org/apache/jena/atlas/lib/TestEscapeStr.java b/jena-base/src/test/java/org/apache/jena/atlas/lib/TestEscapeStr.java
index bac02e4dd5..95c525f3c2 100644
--- a/jena-base/src/test/java/org/apache/jena/atlas/lib/TestEscapeStr.java
+++ b/jena-base/src/test/java/org/apache/jena/atlas/lib/TestEscapeStr.java
@@ -26,7 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.StringWriter;
-import org.junit.jupiter.api.Test ;
+import org.junit.jupiter.api.Test;
import org.apache.jena.atlas.AtlasException;
import org.apache.jena.atlas.io.AWriter;
@@ -41,32 +41,32 @@ public class TestEscapeStr {
private static char D_QUOTE = '"';
// General, for double quoted strings.
- @Test public void escape_str_01() { test_esc("", "") ; }
- @Test public void escape_str_02() { test_esc("A", "A") ; }
- @Test public void escape_str_03() { test_esc("\n", "\\n") ; }
- @Test public void escape_str_04() { test_esc("A\tB", "A\\tB") ; }
- @Test public void escape_str_05() { test_esc("\"", "\\\"") ; }
- @Test public void escape_str_06() { test_esc("'", "'") ; }
+ @Test public void escape_str_01() { test_esc("", ""); }
+ @Test public void escape_str_02() { test_esc("A", "A"); }
+ @Test public void escape_str_03() { test_esc("\n", "\\n"); }
+ @Test public void escape_str_04() { test_esc("A\tB", "A\\tB"); }
+ @Test public void escape_str_05() { test_esc("\"", "\\\""); }
+ @Test public void escape_str_06() { test_esc("'", "'"); }
private static void test_esc(String input, String expected) {
- String output = EscapeStr.stringEsc(input) ;
+ String output = EscapeStr.stringEsc(input);
assertEquals(expected, output, ()->"Failed at escape");
String output2 = EscapeStr.unescapeStr(output);
assertEquals(input, output2, ()->"Failed at unescape");
}
// Single line
- @Test public void escape_str_single_01() { test_esc1("a'x", S_QUOTE, "a\\'x") ; }
- @Test public void escape_str_single_02() { test_esc1("a\"x", D_QUOTE, "a\\\"x") ; }
+ @Test public void escape_str_single_01() { test_esc1("a'x", S_QUOTE, "a\\'x"); }
+ @Test public void escape_str_single_02() { test_esc1("a\"x", D_QUOTE, "a\\\"x"); }
- @Test public void escape_str_single_03() { test_esc1("a\"x", S_QUOTE, "a\"x") ; }
- @Test public void escape_str_single_04() { test_esc1("a'x", D_QUOTE, "a'x") ; }
+ @Test public void escape_str_single_03() { test_esc1("a\"x", S_QUOTE, "a\"x"); }
+ @Test public void escape_str_single_04() { test_esc1("a'x", D_QUOTE, "a'x"); }
- @Test public void escape_str_single_05() { test_esc1("a'", S_QUOTE, "a\\'") ; }
- @Test public void escape_str_single_06() { test_esc1("a\"", D_QUOTE, "a\\\"") ; }
+ @Test public void escape_str_single_05() { test_esc1("a'", S_QUOTE, "a\\'"); }
+ @Test public void escape_str_single_06() { test_esc1("a\"", D_QUOTE, "a\\\""); }
- @Test public void escape_str_single_07() { test_esc1("\"", S_QUOTE, "\"") ; }
- @Test public void escape_str_single_08() { test_esc1("'", D_QUOTE, "'") ; }
+ @Test public void escape_str_single_07() { test_esc1("\"", S_QUOTE, "\""); }
+ @Test public void escape_str_single_08() { test_esc1("'", D_QUOTE, "'"); }
private static void test_esc1(String input, char quoteChar, String expected) {
StringWriter sw = new StringWriter();
@@ -79,43 +79,43 @@ public class TestEscapeStr {
// Multiline quoting.
// One character
- @Test public void escape_str_multi_01() { test_esc3("a'x", S_QUOTE, "a'x") ; }
- @Test public void escape_str_multi_02() { test_esc3("a\"x", D_QUOTE, "a\"x") ; }
+ @Test public void escape_str_multi_01() { test_esc3("a'x", S_QUOTE, "a'x"); }
+ @Test public void escape_str_multi_02() { test_esc3("a\"x", D_QUOTE, "a\"x"); }
- @Test public void escape_str_multi_03() { test_esc3("'x", S_QUOTE, "'x") ; }
- @Test public void escape_str_multi_04() { test_esc3("'x", D_QUOTE, "'x") ; }
+ @Test public void escape_str_multi_03() { test_esc3("'x", S_QUOTE, "'x"); }
+ @Test public void escape_str_multi_04() { test_esc3("'x", D_QUOTE, "'x"); }
// Last character
- @Test public void escape_str_multi_05() { test_esc3("a'", S_QUOTE, "a\\'") ; }
- @Test public void escape_str_multi_06() { test_esc3("a'", D_QUOTE, "a'") ; }
+ @Test public void escape_str_multi_05() { test_esc3("a'", S_QUOTE, "a\\'"); }
+ @Test public void escape_str_multi_06() { test_esc3("a'", D_QUOTE, "a'"); }
- @Test public void escape_str_multi_07() { test_esc3("a\"", S_QUOTE, "a\"") ; }
- @Test public void escape_str_multi_08() { test_esc3("a\"", D_QUOTE, "a\\\"") ; }
+ @Test public void escape_str_multi_07() { test_esc3("a\"", S_QUOTE, "a\""); }
+ @Test public void escape_str_multi_08() { test_esc3("a\"", D_QUOTE, "a\\\""); }
- @Test public void escape_str_multi_09() { test_esc3("'", S_QUOTE, "\\'") ; }
- @Test public void escape_str_multi_10() { test_esc3("'", D_QUOTE, "'") ; }
+ @Test public void escape_str_multi_09() { test_esc3("'", S_QUOTE, "\\'"); }
+ @Test public void escape_str_multi_10() { test_esc3("'", D_QUOTE, "'"); }
- @Test public void escape_str_multi_11() { test_esc3("\"", S_QUOTE, "\"") ; }
- @Test public void escape_str_multi_12() { test_esc3("\"", D_QUOTE, "\\\"") ; }
+ @Test public void escape_str_multi_11() { test_esc3("\"", S_QUOTE, "\""); }
+ @Test public void escape_str_multi_12() { test_esc3("\"", D_QUOTE, "\\\""); }
// 2 in a row
- @Test public void escape_str_multi_2q_1() { test_esc3("a''z", S_QUOTE, "a''z") ; }
- @Test public void escape_str_multi_2q_2() { test_esc3("a''z", D_QUOTE, "a''z") ; }
- @Test public void escape_str_multi_2q_3() { test_esc3("a''", S_QUOTE, "a'\\'") ; }
- @Test public void escape_str_multi_2q_4() { test_esc3("a''", D_QUOTE, "a''") ; }
+ @Test public void escape_str_multi_2q_1() { test_esc3("a''z", S_QUOTE, "a''z"); }
+ @Test public void escape_str_multi_2q_2() { test_esc3("a''z", D_QUOTE, "a''z"); }
+ @Test public void escape_str_multi_2q_3() { test_esc3("a''", S_QUOTE, "a'\\'"); }
+ @Test public void escape_str_multi_2q_4() { test_esc3("a''", D_QUOTE, "a''"); }
// 3 in a row.
- @Test public void escape_str_multi_3q_1() { test_esc3("a'''z", S_QUOTE, "a''\\'z") ; }
- @Test public void escape_str_multi_3q_2() { test_esc3("a'''z", D_QUOTE, "a'''z") ; }
- @Test public void escape_str_multi_3q_3() { test_esc3("a'''", S_QUOTE, "a''\\'") ; }
- @Test public void escape_str_multi_3q_4() { test_esc3("a'''", D_QUOTE, "a'''") ; }
+ @Test public void escape_str_multi_3q_1() { test_esc3("a'''z", S_QUOTE, "a''\\'z"); }
+ @Test public void escape_str_multi_3q_2() { test_esc3("a'''z", D_QUOTE, "a'''z"); }
+ @Test public void escape_str_multi_3q_3() { test_esc3("a'''", S_QUOTE, "a''\\'"); }
+ @Test public void escape_str_multi_3q_4() { test_esc3("a'''", D_QUOTE, "a'''"); }
// 4 in a row.
- @Test public void escape_str_multi_4q_1() { test_esc3("a''''z", S_QUOTE, "a''\\''z") ; }
- @Test public void escape_str_multi_4q_2() { test_esc3("a''''z", D_QUOTE, "a''''z") ; }
+ @Test public void escape_str_multi_4q_1() { test_esc3("a''''z", S_QUOTE, "a''\\''z"); }
+ @Test public void escape_str_multi_4q_2() { test_esc3("a''''z", D_QUOTE, "a''''z"); }
- @Test public void escape_str_multi_4q_3() { test_esc3("a''''", S_QUOTE, "a''\\'\\'") ; }
- @Test public void escape_str_multi_4q_4() { test_esc3("a''''", D_QUOTE, "a''''") ; }
+ @Test public void escape_str_multi_4q_3() { test_esc3("a''''", S_QUOTE, "a''\\'\\'"); }
+ @Test public void escape_str_multi_4q_4() { test_esc3("a''''", D_QUOTE, "a''''"); }
// Unicode replacement char U+FFFD - write in Unicode escape form.
@Test public void escape_str_repacementChar_1() { test_esc1("abc\uFFFDdef", S_QUOTE, "abc\\uFFFDdef"); }
@@ -131,71 +131,92 @@ public class TestEscapeStr {
}
// Unescape
- @Test public void unescape_str_10() { test_unesc("\\u0041", "A") ; }
- @Test public void unescape_str_11() { test_unesc("\\U00000041", "A") ; }
- @Test public void unescape_str_12() { test_unesc("12\\u004134", "12A34") ; }
- @Test public void unescape_str_13() { test_unesc("12\\U0000004134", "12A34") ; }
+ @Test public void unescape_str_10() { test_unesc("\\u0041", "A"); }
+ @Test public void unescape_str_11() { test_unesc("\\U00000041", "A"); }
+ @Test public void unescape_str_12() { test_unesc("12\\u004134", "12A34"); }
+ @Test public void unescape_str_13() { test_unesc("12\\U0000004134", "12A34"); }
private void test_unesc(String input, String expected) {
- String output = EscapeStr.unescapeStr(input) ;
+ String output = EscapeStr.unescapeStr(input);
assertEquals(expected, output);
}
- private void test_escape(String input, String expected) {
+ private void test_escape_string(String input, String expected) {
String output = EscapeStr.stringEsc(input);
assertEquals(expected, output);
}
- @Test public void unescape_unicode_1() { test_unesc_unicode("", "") ; }
- @Test public void unescape_unicode_2() { test_unesc_unicode("abc\\u0020def", "abc def") ; }
- @Test public void unescape_unicode_3() { test_unesc_unicode("\\u0020", " ") ; }
- @Test public void unescape_unicode_4() { test_unesc_unicode("abc\\U00000020def", "abc def") ; }
- @Test public void unescape_unicode_5() { test_unesc_unicode("\\U00000020", " ") ; }
+
+ @Test public void unescape_unicode_1() { test_unesc_unicode("", ""); }
+ @Test public void unescape_unicode_2() { test_unesc_unicode("abc\\u0020def", "abc def"); }
+ @Test public void unescape_unicode_3() { test_unesc_unicode("\\u0020", " "); }
+ @Test public void unescape_unicode_4() { test_unesc_unicode("abc\\U00000020def", "abc def"); }
+ @Test public void unescape_unicode_5() { test_unesc_unicode("\\U00000020", " "); }
// Leaves non-unicode untouched.
- @Test public void unescape_unicode_10() { test_unesc_unicode("\\1\\2", "\\1\\2") ; }
- @Test public void unescape_unicode_11() { test_unesc_unicode("\\n\\t", "\\n\\t") ; }
- @Test public void unescape_unicode_12() { test_unesc_unicode("\\(\\)", "\\(\\)") ; }
- @Test public void unescape_unicode_13() { test_unesc_unicode("\\\\", "\\\\") ; }
+ @Test public void unescape_unicode_10() { test_unesc_unicode("\\1\\2", "\\1\\2"); }
+ @Test public void unescape_unicode_11() { test_unesc_unicode("\\n\\t", "\\n\\t"); }
+ @Test public void unescape_unicode_12() { test_unesc_unicode("\\(\\)", "\\(\\)"); }
+ @Test public void unescape_unicode_13() { test_unesc_unicode("\\\\", "\\\\"); }
// \-u{...} style Unicode escapes
- @Test public void unescape_unicode_20() { test_unesc_unicode("\\u{41}", "A") ; }
- @Test public void unescape_unicode_21() { test_unesc_unicode("\\u{000000}", "\u0000") ; }
- @Test public void unescape_unicode_22() { test_unesc_unicode("\\u{1F0A1}", "🂡") ; }
- @Test public void unescape_unicode_23() { test_unesc_unicode("\\u{01F0A1}", "🂡") ; }
- @Test public void unescape_unicode_24() { test_unesc_unicode("\\u{10FFFF}", 0x10FFFF) ; }
-
- @Test public void unescape_unicode_30() { assertThrows(AtlasException.class, ()->test_unesc_unicode("\\u{}", "")) ; }
- @Test public void unescape_unicode_31() { assertThrows(AtlasException.class, ()->test_unesc_unicode("\\u{123456789}", "")) ; }
- @Test public void unescape_unicode_32() { assertThrows(AtlasException.class, ()->test_unesc_unicode("\\u{000000000}", "")) ; }
+ @Test public void unescape_unicode_20() { test_unesc_unicode("\\u{41}", "A"); }
+ @Test public void unescape_unicode_21() { test_unesc_unicode("\\u{000000}", "\u0000"); }
+ @Test public void unescape_unicode_22() { test_unesc_unicode("\\u{1F0A1}", "🂡"); }
+ @Test public void unescape_unicode_23() { test_unesc_unicode("\\u{01F0A1}", "🂡"); }
+ @Test public void unescape_unicode_24() { test_unesc_unicode("\\u{10FFFF}", 0x10FFFF); }
+
+ @Test public void unescape_unicode_30() { test_unesc_unicode_bad("\\u{}"); }
+ @Test public void unescape_unicode_31() { test_unesc_unicode_bad("\\u{123456789}"); }
+ @Test public void unescape_unicode_32() { test_unesc_unicode_bad("\\u{000000000}"); }
// If the limit is 6
- @Test public void unescape_unicode_33() { assertThrows(AtlasException.class, ()->test_unesc_unicode("\\u{1234567}", "")) ; }
- @Test public void unescape_unicode_34() { assertThrows(AtlasException.class, ()->test_unesc_unicode("\\u{0000000}", "")) ; }
+ @Test public void unescape_unicode_33() { test_unesc_unicode_bad("\\u{1234567}"); }
+ @Test public void unescape_unicode_34() { test_unesc_unicode_bad("\\u{0000000}"); }
+
+ // Surrogates via \-u, \-U and \-u{}
+ @Test public void unescape_unicode_40() { test_unesc_unicode_bad("\\uD83C\\uDCA1"); }
+ @Test public void unescape_unicode_41() { test_unesc_unicode_bad("\\U0000D83C\\U0000DCA1"); }
+ @Test public void unescape_unicode_42() { test_unesc_unicode_bad("\\u{00D83C}\\u{00DCA1}"); }
+
+ // Not valid surrogate pairs.
+ @Test public void unescape_unicode_50() { test_unesc_unicode_bad("\\uD83Cxyz"); }
+ @Test public void unescape_unicode_51() { test_unesc_unicode_bad("\\U0000D83Cxyz"); }
+ @Test public void unescape_unicode_52() { test_unesc_unicode_bad("\\u{00D83C}xyz"); }
+
+ @Test public void unescape_unicode_60() { test_unesc_unicode_bad("abc\\uDCA1xyz"); }
+ @Test public void unescape_unicode_61() { test_unesc_unicode_bad("abc\\U0000DCA1xyz"); }
+ @Test public void unescape_unicode_62() { test_unesc_unicode_bad("abc\\u{00DCA1}xyz"); }
// Escaped surrogates, good and bad.
// Use java character escapes to put the surrogates into the java string.
// 🂡 is U+D83C U+DCA1
- @Test public void escape_unicode_50() { test_escape("\uD83C\uDCA1", "🂡"); }
- @Test public void escape_unicode_51() { test_escape("abc\uD83C\uDCA1xyz", "abc🂡xyz"); }
+ @Test public void escape_unicode_70() { test_escape_string("\uD83C\uDCA1", "🂡"); }
+ @Test public void escape_unicode_71() { test_escape_string("abc\uD83C\uDCA1xyz", "abc🂡xyz"); }
// low, then high -> illegal
- @Test public void escape_unicode_55() { test_escape("\uDCA1\uD83C", "\\uFFFD\\uFFFD"); }
- @Test public void escape_unicode_56() { test_escape("\uDCA1\uD83C@", "\\uFFFD\\uFFFD@"); }
+ @Test public void escape_unicode_75() { test_escape_string("\uDCA1\uD83C", "\\uFFFD\\uFFFD"); }
+ @Test public void escape_unicode_76() { test_escape_string("\uDCA1\uD83C@", "\\uFFFD\\uFFFD@"); }
// Lone surrogate
- @Test public void escape_unicode_60() { test_escape("\uD83C", "\\uFFFD"); }
- @Test public void escape_unicode_61() { test_escape("abc\uD83Cxyz", "abc\\uFFFDxyz"); }
- @Test public void escape_unicode_62() { test_escape("\uDCA1", "\\uFFFD"); }
- @Test public void escape_unicode_63() { test_escape("abc\uDCA1xyz", "abc\\uFFFDxyz"); }
+ @Test public void escape_unicode_80() { test_escape_string("\uD83C", "\\uFFFD"); }
+ @Test public void escape_unicode_81() { test_escape_string("abc\uD83Cxyz", "abc\\uFFFDxyz"); }
+ @Test public void escape_unicode_82() { test_escape_string("\uDCA1", "\\uFFFD"); }
+ @Test public void escape_unicode_83() { test_escape_string("abc\uDCA1xyz", "abc\\uFFFDxyz"); }
+ // Illegal then legal
+ @Test public void escape_unicode_84() { test_escape_string("\uDCA1\uD83C\uDCA1", "\\uFFFD🂡"); }
+
// low, then high/low -> one illegal, encode legal pair.
- @Test public void escape_unicode_59() { test_escape("\uDCA1\uD83C\uDCA1", "\\uFFFD🂡"); }
private void test_unesc_unicode(String input, String expected) {
- String output = EscapeStr.unescapeUnicode(input) ;
+ String output = EscapeStr.unescapeUnicode(input);
assertEquals(expected, output);
}
private void test_unesc_unicode(String input, int expected) {
- String output = EscapeStr.unescapeUnicode(input) ;
+ String output = EscapeStr.unescapeUnicode(input);
int codepoint = output.codePointAt(0);
assertEquals(expected, codepoint);
}
+
+ private void test_unesc_unicode_bad(String input) {
+ assertThrows(AtlasException.class, ()->EscapeStr.unescapeUnicode(input));
+ }
}
(jena) 01/02: GH-3931: No Surrogates - RIOT
Posted by an...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
afs pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/jena.git
commit aaaf6ee5908876631c60168727bc4badce593c07
Author: Andy Seaborne <an...@apache.org>
AuthorDate: Mon May 18 18:32:24 2026 +0100
GH-3931: No Surrogates - RIOT
---
.../src/main/java/org/apache/jena/query/ARQ.java | 51 ++--
.../src/main/java/org/apache/jena/riot/RIOT.java | 5 +
.../org/apache/jena/riot/tokens/TokenizerText.java | 121 +++------
...nizerText.java => TokenizerTextSurrogates.java} | 35 ++-
.../org/apache/jena/riot/lang/TestLangTurtle.java | 14 +-
.../AbstractTestTokenizerTextSurrogates.java | 293 +++++++++++++++++++++
.../org/apache/jena/riot/tokens/TS_Tokens.java | 6 +-
.../apache/jena/riot/tokens/TestTokenizerText.java | 257 +-----------------
.../TestTokenizerTextAllowEscSurrogates.java | 80 ++++++
...ens.java => TestTokenizerTextNoSurrogates.java} | 33 ++-
.../sparql/expr/TestSPARQLKeywordFunctions.java | 2 +-
.../main/java/org/apache/jena/atlas/lib/Chars.java | 7 +
12 files changed, 515 insertions(+), 389 deletions(-)
diff --git a/jena-arq/src/main/java/org/apache/jena/query/ARQ.java b/jena-arq/src/main/java/org/apache/jena/query/ARQ.java
index 8667faccab..9b6277e758 100644
--- a/jena-arq/src/main/java/org/apache/jena/query/ARQ.java
+++ b/jena-arq/src/main/java/org/apache/jena/query/ARQ.java
@@ -139,47 +139,38 @@ public class ARQ
/** Prefix for ARQ-defined parameter names */
public static final String arqSymbolPrefix = "arq";
- /** Stick exactly to the spec.
- */
+ /** Stick exactly to the spec. */
public static final Symbol strictSPARQL =
SystemARQ.allocSymbol("strictSPARQL");
- /** Controls bNode labels as <_:...> or not -
- * that is a pseudo URIs.
- * This does not affect [] or _:a bNodes as variables in queries.
- */
-
- public static final Symbol constantBNodeLabels =
- SystemARQ.allocSymbol("constantBNodeLabels");
-
- /** Enable built-in property functions - also called "magic properties".
- * These are properties in triple patterns that need
- * calculation, not matching. See ARQ documentation for more details.
- * rdfs:member and http://jena.apache.org/ARQ/list#member are provided.
+ /**
+ * Controls bNode labels as <_:...> or not - that is a pseudo URIs. This
+ * does not affect [] or _:a bNodes as variables in queries.
*/
+ public static final Symbol constantBNodeLabels = SystemARQ.allocSymbol("constantBNodeLabels");
- public static final Symbol enablePropertyFunctions =
- SystemARQ.allocSymbol("enablePropertyFunctions");
-
- /** Enable logging of execution timing.
+ /**
+ * Enable built-in property functions - also called "magic properties". These are
+ * properties in triple patterns that need calculation, not matching. See ARQ
+ * documentation for more details. rdfs:member and
+ * http://jena.apache.org/ARQ/list#member are provided.
*/
+ public static final Symbol enablePropertyFunctions = SystemARQ.allocSymbol("enablePropertyFunctions");
- public static final Symbol enableExecutionTimeLogging =
- SystemARQ.allocSymbol("enableExecutionTimeLogging");
+ /** Enable logging of execution timing. */
+ public static final Symbol enableExecutionTimeLogging = SystemARQ.allocSymbol("enableExecutionTimeLogging");
- /** If true, XML result sets written will contain the graph bNode label
- * See also inputGraphBNodeLabels
+ /**
+ * If true, XML result sets written will contain the graph bNode label.
+ * See also {@link #inputGraphBNodeLabels}.
*/
+ public static final Symbol outputGraphBNodeLabels = SystemARQ.allocSymbol("outputGraphBNodeLabels");
- public static final Symbol outputGraphBNodeLabels =
- SystemARQ.allocSymbol("outputGraphBNodeLabels");
-
- /** If true, XML result sets will use the bNode label in the result set itself.
- * See also outputGraphBNodeLabels
+ /**
+ * If true, XML result sets will use the bNode label in the result set itself.
+ * See also {@link #outputGraphBNodeLabels}.
*/
-
- public static final Symbol inputGraphBNodeLabels =
- SystemARQ.allocSymbol("inputGraphBNodeLabels");
+ public static final Symbol inputGraphBNodeLabels = SystemARQ.allocSymbol("inputGraphBNodeLabels");
/** Turn on processing of blank node labels in queries */
public static void enableBlankNodeResultLabels() { enableBlankNodeResultLabels(true); }
diff --git a/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java b/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java
index 6ff7eaa7c4..f61ae0858a 100644
--- a/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java
+++ b/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java
@@ -22,6 +22,7 @@
package org.apache.jena.riot ;
import org.apache.jena.atlas.lib.Version;
+import org.apache.jena.query.ARQ;
import org.apache.jena.riot.resultset.ResultSetLang;
import org.apache.jena.riot.system.StreamRDF;
import org.apache.jena.riot.writer.DirectiveStyle;
@@ -60,6 +61,10 @@ public class RIOT {
return systemGlobalContext;
}
+ public static boolean isStrictMode() {
+ return ARQ.isStrictMode();
+ }
+
public static void init() {
if ( initialized )
return ;
diff --git a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
index 3792b9288d..962da824c2 100644
--- a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
+++ b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
@@ -32,6 +32,7 @@ import org.apache.jena.atlas.io.IO;
import org.apache.jena.atlas.io.PeekReader;
import org.apache.jena.atlas.lib.Chars;
import org.apache.jena.atlas.lib.Hex;
+import org.apache.jena.atlas.lib.InternalErrorException;
import org.apache.jena.riot.RiotParseException;
import org.apache.jena.riot.system.ErrorHandler;
import org.apache.jena.riot.system.RiotChars;
@@ -99,9 +100,6 @@ public final class TokenizerText implements Tokenizer
// CHECK_CODEPOINTS = true;
// CHECK_RDFSTRING = false;
- // Default=true. Whether to check for legal codepoint (i.e. only high-low surrogate pairs) while building strings.
- private static final boolean CHECK_CODEPOINTS = true;
-
// Default=false. Whether to check for legal RDF strings (no ill formed use of surrogates) after building strings (normally done during string parsing).
private static final boolean CHECK_RDFSTRING = false;
@@ -611,9 +609,7 @@ public final class TokenizerText implements Tokenizer
// It should be possible to rename stringBuilder with no changes to the code anywhere outside these operations.s
private final StringBuilder stringBuilder = new StringBuilder(200);
- private static final int NO_CODEPOINT = '\u0000';
-
- // -- Unicode sequences with the possibility of codepoints beyond U+FFFF
+ // -- Unicode sequences with the possibility of codepoints beyond U+FFFF
/**
* String with the possibility of a unicode surrogate or unicode escape.
*
@@ -627,9 +623,7 @@ public final class TokenizerText implements Tokenizer
* Check terminates correctly and return string.
* Pair with {@link #startStringU()}
*/
- private String finishStringU(int finalCodepoint) {
- if ( finalCodepoint != NO_CODEPOINT )
- fatal("Bad unpaired surrogate at end of string");
+ private String finishStringU() {
return stringBuilder.toString();
}
@@ -659,56 +653,25 @@ public final class TokenizerText implements Tokenizer
private void deleteCharAt(int idx) { stringBuilder.deleteCharAt(idx); }
/** Insert codepoint. */
- private int insertCodepoint(int previousCP,int ch) {
+ private void insertCodepoint(int ch) {
if ( Character.charCount(ch) == 1 ) {
char ch16 = (char)ch; // Safe, not truncating, because count = 1
- char rtn = 0;
- if ( CHECK_CODEPOINTS )
- rtn = checkCodepoint((char)previousCP, ch16);
insertCodepointDirect(ch16);
- return rtn;
- } else {
- // Surrogate waiting?
- if ( CHECK_CODEPOINTS && (previousCP != NO_CODEPOINT) )
- fatal("Lone surrogate");
- if ( !Character.isDefined(ch) && !Character.isSupplementaryCodePoint(ch) )
- fatal("Illegal codepoint: 0x%04X", ch);
- // Only legal surrogate pairs at this point.
- char[] chars = Character.toChars(ch);
- stringBuilder.append(chars);
- return NO_CODEPOINT;
- }
- }
-
- // Only high then low is allowed.
- // Casting int to char is a 16 bit silent truncation (bytecode "i2c").
- private char checkCodepoint(char previousCP, char ch) {
- if ( ! Character.isSurrogate(ch) ) {
- if ( previousCP == NO_CODEPOINT )
- return NO_CODEPOINT;
- fatal("Bad surrogate (high surrogate not followed by a low surrogate): 0x%04X", (int)previousCP);
- }
- // Surrogate.
- if ( previousCP == NO_CODEPOINT ) { // Effectively: is previousCodePoint a high surrogate?
- if ( Character.isHighSurrogate(ch) ) {
- // Park it
- return ch;
- }
- fatal("Bad surrogate (low surrogate not preceded by a high surrogate): 0x%04X", (int)ch);
- }
- // previousCodePoint != NO_CODEPOINT
- // Previous is a high surrogate
-
- if ( Character.isLowSurrogate(ch) ) {
- // high-low -- OK! Clear previous.
- return NO_CODEPOINT;
+ return;
}
- fatal("Bad surrogate (high surrogate not followed by low surrogate): 0x%04X", (int)previousCP);
- return NO_CODEPOINT;
+ if ( !Character.isDefined(ch) && !Character.isSupplementaryCodePoint(ch) )
+ fatal("Illegal codepoint: 0x%04X", ch);
+ char[] chars = Character.toChars(ch);
+ stringBuilder.append(chars);
}
+ // XXX
// Insert codepoint, knowing that 'ch' is 16 bit and not a surrogate.
private void insertCodepointDirect(int ch) {
+ if ( ch >= Character.MIN_SUPPLEMENTARY_CODE_POINT ) {
+ String msg = String .format("Internal error: Attempt to insert a supplementary codepoint directly 0x04X", ch);
+ throw new InternalErrorException(msg);
+ }
insertCodepointDirect((char)ch);
}
@@ -725,7 +688,6 @@ public final class TokenizerText implements Tokenizer
// [8] IRIREF ::= '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>'
private String readIRI() {
startStringU();
- int prevCP = NO_CODEPOINT;
for (;;) {
int ch = reader.readChar();
switch(ch) {
@@ -737,7 +699,7 @@ public final class TokenizerText implements Tokenizer
fatal("Broken IRI (CR): %s", currentString()); return null;
case CH_GT:
// Done!
- String str = finishStringU(prevCP);
+ String str = finishStringU();
if ( CHECK_RDFSTRING )
checkRDFString(str);
return str;
@@ -769,7 +731,7 @@ public final class TokenizerText implements Tokenizer
if ( ch <= 0x19 )
warning("Illegal character in IRI (control char 0x%02X): <%s[0x%02X]...>", ch, currentString(), ch);
}
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
}
}
@@ -836,8 +798,6 @@ public final class TokenizerText implements Tokenizer
// RiotChars has isPNChars_U_N for ( PN_CHARS_U | [0-9] )
- int prevCP = NO_CODEPOINT;
-
// -- Test first character
int ch = reader.peekChar();
if ( ch == EOF )
@@ -848,7 +808,7 @@ public final class TokenizerText implements Tokenizer
if ( isLocalPart ) {
if ( ch == CH_COLON ) {
reader.readChar();
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
} else if ( ch == CH_PERCENT || ch == CH_RSLASH ) {
// processPLX
// read % or \
@@ -860,18 +820,18 @@ public final class TokenizerText implements Tokenizer
if ( ch == REPLACEMENT )
warning("Unicode replacement character U+FFFD in prefixed name");
}
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
reader.readChar();
} else {
- finishStringU(prevCP);
+ finishStringU();
return "";
}
} else {
if ( !RiotChars.isPNCharsBase(ch) ) {
- finishStringU(prevCP);
+ finishStringU();
return "";
}
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
reader.readChar();
}
// Done first character
@@ -885,7 +845,6 @@ public final class TokenizerText implements Tokenizer
if ( chDot != 0 )
insertCodepointDirect(chDot);
processPLX(ch);
- prevCP = NO_CODEPOINT;
chDot = 0;
continue;
}
@@ -915,7 +874,7 @@ public final class TokenizerText implements Tokenizer
if ( ch == REPLACEMENT )
warning("Unicode replacement character U+FFFD in prefixed name");
}
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
} else {
// DOT - delay until next loop.
chDot = ch;
@@ -928,7 +887,7 @@ public final class TokenizerText implements Tokenizer
if ( chDot == CH_DOT )
// Unread it.
reader.pushbackChar(chDot);
- return finishStringU(prevCP);
+ return finishStringU();
}
// Process PLX (percent or character escape for a prefixed name)
@@ -990,7 +949,6 @@ public final class TokenizerText implements Tokenizer
// Reads the terminating delimiter.
startStringU();
- int prevCP = NO_CODEPOINT;
for (;;) {
int ch = reader.readChar();
if ( WarnOnReplacmentCharInString ) {
@@ -1004,7 +962,7 @@ public final class TokenizerText implements Tokenizer
fatal("Broken token: %s", currentString());
else if ( ch == endCh ) {
// Done!
- String str = finishStringU(prevCP);
+ String str = finishStringU();
if ( CHECK_RDFSTRING )
checkRDFString(str);
return str;
@@ -1020,7 +978,7 @@ public final class TokenizerText implements Tokenizer
else if ( ch == CH_RSLASH )
// Allow escaped replacement character.
ch = readLiteralEscape();
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
}
}
@@ -1028,7 +986,6 @@ public final class TokenizerText implements Tokenizer
// Assumes the 3 character starting delimiter has been read.
// Reads the terminating delimiter.
startStringU();
- int prevCP = NO_CODEPOINT;
for (;;) {
int ch = reader.readChar();
if ( WarnOnReplacmentCharInString ) {
@@ -1040,7 +997,7 @@ public final class TokenizerText implements Tokenizer
fatal("Broken long string");
} else if ( ch == quoteChar ) {
if ( threeQuotes(quoteChar) ) {
- String str = finishStringU(prevCP);
+ String str = finishStringU();
if ( CHECK_RDFSTRING )
checkRDFString(str);
return str;
@@ -1048,7 +1005,7 @@ public final class TokenizerText implements Tokenizer
// quote, not triple. It is a normal character.
} else if ( ch == CH_RSLASH )
ch = readLiteralEscape();
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
}
}
@@ -1126,8 +1083,6 @@ public final class TokenizerText implements Tokenizer
private String readBlankNodeLabel() {
startStringU();
-
- int prevCP = NO_CODEPOINT;
// First character.
{
int ch = reader.peekChar();
@@ -1143,7 +1098,7 @@ public final class TokenizerText implements Tokenizer
if ( ch == REPLACEMENT )
warning("Unicode replacement character U+FFFD in blank node label");
}
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
}
// Remainder. DOT can't be last so do a delay on that.
@@ -1162,7 +1117,6 @@ public final class TokenizerText implements Tokenizer
if ( chDot != 0 ) {
insertCodepointDirect(chDot);
- prevCP = NO_CODEPOINT;
chDot = 0;
}
@@ -1172,7 +1126,7 @@ public final class TokenizerText implements Tokenizer
if ( ch == REPLACEMENT )
warning("Unicode replacement character U+FFFD in blank node label");
}
- prevCP = insertCodepoint(prevCP, ch);
+ insertCodepoint(ch);
} else
// DOT - delay until next loop.
chDot = ch;
@@ -1184,7 +1138,7 @@ public final class TokenizerText implements Tokenizer
// if ( ! seen )
// exception("Blank node label missing");
- return finishStringU(prevCP);
+ return finishStringU();
}
/*
@@ -1398,7 +1352,7 @@ public final class TokenizerText implements Tokenizer
} else
break;
}
- return finishStringU(NO_CODEPOINT).intern();
+ return finishStringU().intern();
}
// ASCII-only e.g. in lang tags.
@@ -1497,11 +1451,12 @@ public final class TokenizerText implements Tokenizer
// Either \-uXXXX or \-u{...}
private final int readUnicode4OrDelimEscape() {
int nextCh = reader.peekChar();
- if ( nextCh == CH_LBRACE ) {
+ if ( nextCh == CH_LBRACE )
return readUnicodeDelimitedHex();
- } else {
- return readUnicode4Escape();
- }
+ int ch = readUnicode4Escape();
+ if ( isSurrogate(ch) )
+ fatal("Surrogate codepoint from Unicode \\u escape sequence: 0x%04X", ch);
+ return ch;
}
private final int readUnicodeDelimitedHex() {
@@ -1530,6 +1485,8 @@ public final class TokenizerText implements Tokenizer
}
if ( n == 0 )
fatal("Empty delimited hex escape sequence");
+ if ( isSurrogate(value) )
+ fatal("Surrogate codepoint from delimited Unicode escape sequence: 0x%04X", value);
return value;
}
@@ -1542,6 +1499,8 @@ public final class TokenizerText implements Tokenizer
// Check > 0x10FFFF
if ( ! Character.isValidCodePoint(ch8) )
fatal("Illegal code point from \\U sequence value: 0x%08X", ch8);
+ if ( isSurrogate(ch8) )
+ fatal("Surrogate codepoint from Unicode \\U escape sequence: 0x%04X", ch8);
return ch8;
}
diff --git a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerTextSurrogates.java
similarity index 97%
copy from jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
copy to jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerTextSurrogates.java
index 3792b9288d..eb29663c3f 100644
--- a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
+++ b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerTextSurrogates.java
@@ -33,15 +33,29 @@ import org.apache.jena.atlas.io.PeekReader;
import org.apache.jena.atlas.lib.Chars;
import org.apache.jena.atlas.lib.Hex;
import org.apache.jena.riot.RiotParseException;
+import org.apache.jena.riot.SysRIOT;
import org.apache.jena.riot.system.ErrorHandler;
+import org.apache.jena.riot.system.ErrorHandlerFactory;
import org.apache.jena.riot.system.RiotChars;
import org.apache.jena.sparql.ARQInternalErrorException;
+import org.slf4j.Logger;
+
+
+// TokenizerText from Jena 6.1.0
+//
+// This version accepts surrogate pairs written using numeric escape sequences
+// and rejects surrogates ill-formed using numeric escape sequences
+//
+// Changing back to this tokenizer will affect tests.
+
+// TokenizerText now rejects all surrogates inline with the RDF 1.2 clarifications.
+// Jena has not written surrogates numeric escape sequences in any version.
/**
* Tokenizer for the Turtle family of syntaxes.
* Supports additional tokens.
*/
-public final class TokenizerText implements Tokenizer
+/*public*/ final class TokenizerTextSurrogates implements Tokenizer
{
// This class is performance critical.
@@ -148,15 +162,24 @@ public final class TokenizerText implements Tokenizer
private Token token = null;
private boolean finished = false;
- public static TokenizerTextBuilder create() { return new TokenizerTextBuilder(); }
+ // public static TokenizerTextBuilder create() { return new TokenizerTextBuilder(); }
+ //
+ // public static Tokenizer fromString(String string) { return create().fromString(string).build(); }
- public static Tokenizer fromString(String string) { return create().fromString(string).build(); }
+ // For tests
+ public static Tokenizer fromString(String string) {
+ PeekReader reader = PeekReader.readString(string);
+ Logger LOG = SysRIOT.getLogger();
+ // Default - errors are exceptions, warning are logged.
+ ErrorHandler errorHandlerDft = ErrorHandlerFactory.errorHandlerWarnOrExceptions(LOG);
+ return new TokenizerTextSurrogates(reader, false, errorHandlerDft);
+ }
- /*package*/ static TokenizerText internal(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) {
- return new TokenizerText(reader, singleLineMode, errorHandler);
+ /*package*/ static TokenizerTextSurrogates internal(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) {
+ return new TokenizerTextSurrogates(reader, singleLineMode, errorHandler);
}
- private TokenizerText(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) {
+ private TokenizerTextSurrogates(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) {
this.reader = Objects.requireNonNull(reader, "PeekReader");
this.singleLineMode = singleLineMode;
this.errorHandler = Objects.requireNonNull(errorHandler, "ErrorHandler");
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java b/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java
index 55b2d635df..1f60db7628 100644
--- a/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java
+++ b/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java
@@ -298,12 +298,18 @@ public class TestLangTurtle
@Test public void turtle_surrogate_1() {
// escaped high, escaped low
- parseOneTriple("<x:s> <x:p> '\\ud800\\udc00' . ");
+ // If accepting escaped surrogates.
+ // parseOneTriple("<x:s> <x:p> '\\ud800\\udc00' . ");
+ // If not accepting escaped surrogates
+ parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\ud800\\udc00' . "));
}
@Test public void turtle_surrogate_2() {
// escaped high, raw low
- parseOneTriple("<x:s> <x:p> '\\ud800\udc00' . ");
+ // If accepting escaped surrogates.
+ // parseOneTriple("<x:s> <x:p> '\\ud800\udc00' . ");
+ // If not accepting escaped surrogates
+ parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\ud800\udc00' . "));
}
// Compilation failure. (maven+openjdk - OK in Eclipse, and test correct)
@@ -339,7 +345,7 @@ public class TestLangTurtle
@Test
public void turtle_bad_surrogate_5() {
// Wrong way round: low-high
- parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\uc800\\ud800' . "));
+ parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\udc00\\ud800' . "));
}
// Compilation failure. Can't write \ud800
@@ -352,7 +358,7 @@ public class TestLangTurtle
@Test
public void turtle_bad_surrogate_7() {
// escaped low - raw high
- parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\uc800\ud800' . "));
+ parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\ud800\ud800' . "));
}
// No Formulae. Not trig.
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java
new file mode 100644
index 0000000000..475ea00c6e
--- /dev/null
+++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.apache.jena.riot.tokens;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.jena.riot.RiotException;
+
+public abstract class AbstractTestTokenizerTextSurrogates {
+
+ abstract protected Tokenizer tokenizer(String string);
+
+ // Classification of tests
+
+ // Structurally bad.
+ abstract protected void surrogateBad(String string);
+
+ // Validly encoded into escape sequences.
+ abstract protected void surrogateValidEsc(String string);
+
+ protected final void surrogateTest(String string) {
+ Tokenizer tokenizer = tokenizer(string);
+ assertTrue(tokenizer.hasNext());
+ Token t1 = tokenizer.next();
+ assertFalse(tokenizer.hasNext());
+ }
+
+ // Literal surrogates, i.e. java escape used to put into the tokenizer input.
+ // These don't get to the tokenizer escape processing code.
+ // They should be as if the UTF-8 has supplemental codepoints.
+ protected void surrogateRawGood(String string) {
+ surrogateTest(string);
+ }
+
+ // Literal surrogates, i.e. java escape used to put into the tokenizer input but illegal in some way.
+ protected final void surrogateRawBad(String string) {
+ assertThrows(RiotException.class, ()->surrogateTest(string));
+ }
+
+ // ----
+
+ // U+D800-U+DBFF is a high surrogate (first part of a pair)
+ // U+DC00-U+DFFF is a low surrogate (second part of a pair)
+ // so D800-DC00 is legal.
+
+ @Test public void turtle_surrogate_pair_esc_esc_01() {
+ // escaped high, escaped low
+ surrogateValidEsc("'\\ud800\\udc00'");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_esc_02() {
+ // escaped high, escaped low
+ surrogateValidEsc("'''\\ud800\\udc00'''");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_esc_03() {
+ // escaped high, escaped low
+ surrogateValidEsc("<\\ud800\\udc00>");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_raw_01() {
+ // escaped high, raw low
+ surrogateValidEsc("'\\ud800\udc00'");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_raw_02() {
+ // escaped high, raw low
+ surrogateValidEsc("'''\\ud800\udc00'''");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_raw_03() {
+ // escaped high, raw low
+ surrogateRawBad("<\\ud800\udc00>");
+ }
+
+ // Compilation failure - illegal escape character
+// @Test public void turtle_surrogate_pair_raw_esc_01() {
+// // raw high, escaped low
+// surrogate("'\ud800\\udc00'");
+// }
+
+ @Test public void turtle_surrogate_pair_raw_raw_01() {
+ // raw high, raw low
+ surrogateRawGood("'\ud800\udc00'");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw_02() {
+ // raw high, raw low
+ surrogateRawGood("'''\ud800\udc00'''");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw_03() {
+ // raw high, raw low
+ surrogateRawGood("<\ud800\udc00>");
+ }
+
+ // Blank nodes label allow unicode but not unicode escapes.
+ @Test public void turtle_surrogate_pair_raw_raw_04() {
+ // raw high, raw low
+ surrogateRawGood("_:b\ud800\udc00");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw_05() {
+ surrogateRawGood("ns:\ud800\udc00");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw_06() {
+ surrogateRawGood("\ud800\udc00:local");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_esc_internal_01() {
+ // escaped high, escaped low
+ surrogateValidEsc("'a\\ud800\\udc00x'");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_esc_internal_02() {
+ // escaped high, escaped low
+ surrogateValidEsc("'''a\\ud800\\udc00x'''");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_esc_internal_03() {
+ // escaped high, escaped low
+ surrogateValidEsc("<a\\ud800\\udc00x>");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_raw_internal_01() {
+ // escaped high, raw low
+ surrogateRawBad("'z\\ud800\udc00z'");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_raw_internal_02() {
+ // escaped high, raw low
+ surrogateRawBad("'''z\\ud800\udc00z'''");
+ }
+
+ @Test public void turtle_surrogate_pair_esc_raw_internal_03() {
+ // escaped high, raw low
+ surrogateRawBad("<z\\ud800\udc00z>");
+ }
+
+ // Compilation failure - illegal escape character
+// @Test public void turtle_surrogate_pair_raw_esc() {
+// // raw high, escaped low
+// surrogate("'a\ud800\\udc00'z");
+// }
+
+ // Java, not RDF, surrogates.
+ @Test public void turtle_surrogate_pair_raw_raw_internal_01() {
+ // raw high, raw low
+ surrogateRawGood("'a\ud800\udc00z'");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw_internal_02() {
+ // raw high, raw low
+ surrogateRawGood("'''a\ud800\udc00z'''");
+ }
+
+ // XXX Overlap
+ @Test public void turtle_surrogate_pair_raw_raw_internal_03() {
+ // raw high, raw low
+ surrogateRawGood("<a\ud800\udc00z>");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw_internal_04() {
+ // raw high, raw low
+ surrogateRawGood("_:ba\ud800\udc00z");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw__internal05() {
+ surrogateRawGood("ns:x\ud800\udc00y");
+ }
+
+ @Test public void turtle_surrogate_pair_raw_raw__internal06() {
+ surrogateRawGood("x\ud800\udc00y:local");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_01() {
+ surrogateBad("'\\ud800'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_01a() {
+ surrogateBad("'''\\ud800''''");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_02() {
+ surrogateBad("'a\\ud800z'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_02a() {
+ surrogateBad("'''a\\ud800z'''");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_02b() {
+ surrogateBad("<a\\ud800z>");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_03() {
+ surrogateBad("'\\udfff'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_04() {
+ surrogateBad("'a\\udfffz'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_05() {
+ surrogateBad("'\\U0000d800'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_06() {
+ surrogateBad("'a\\U0000d800z'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_07() {
+ surrogateBad("'\\U0000dfff'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_08() {
+ surrogateBad("'a\\U0000dfffz'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_09() {
+ // Wrong way round: low-high;
+ surrogateBad("'\\udc00\\ud800'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_10() {
+ // Wrong way round: low-high
+ surrogateBad("'a\\udc00\ud800z'");
+ }
+
+ // Compilation failure - illegal escape character
+// @Test
+// public void turtle_bad_surrogate_11() {
+// // raw low - escaped high
+// assertThrows(RiotParseException.class, ()->
+// surrogate("'\ud800\\ud800'");
+// }
+//
+// @Test
+// public void turtle_bad_surrogate_12() {
+// // raw low - escaped high
+// assertThrows(RiotParseException.class, ()->
+// surrogate("'a\ud800\\ud800z'");
+// }
+
+ @Test
+ public void turtle_bad_surrogate_13() {
+ // escaped low - raw high
+ surrogateBad("'\\udc00\ud800'");
+ }
+
+ @Test
+ public void turtle_bad_surrogate_14() {
+ // escaped low - raw high
+ surrogateBad("'a\\dc00\ud800z'");
+ }
+}
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java
index 9d4f87226c..044468dff6 100644
--- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java
+++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java
@@ -26,8 +26,10 @@ import org.junit.platform.suite.api.Suite;
@Suite
@SelectClasses({
- TestTokenizerText.class
- , TestTokenForNode.class
+ TestTokenForNode.class
+ , TestTokenizerText.class
+ , TestTokenizerTextNoSurrogates.class
+ , TestTokenizerTextAllowEscSurrogates.class
})
public class TS_Tokens
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java
index 64af6b8ad7..c82b2aca52 100644
--- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java
+++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java
@@ -39,6 +39,7 @@ import org.apache.jena.riot.system.ErrorHandlerFactory.ErrorHandlerRecorder;
import org.apache.jena.sparql.ARQConstants;
public class TestTokenizerText {
+ // Surrogate related tests in TestTokenizerTextNoSurrogates.
private static Tokenizer tokenizer(String string) {
return tokenizer(string, false);
@@ -1380,262 +1381,6 @@ public class TestTokenizerText {
assertFalse(tokenizer.hasNext());
}
- // U+D800-U+DBFF is a high surrogate (first part of a pair)
- // U+DC00-U+DFFF is a low surrogate (second part of a pair)
- // so D800-DC00 is legal.
-
- @Test public void turtle_surrogate_pair_esc_esc_01() {
- // escaped high, escaped low
- surrogate("'\\ud800\\udc00'");
- }
-
- @Test public void turtle_surrogate_pair_esc_esc_02() {
- // escaped high, escaped low
- surrogate("'''\\ud800\\udc00'''");
- }
-
- @Test public void turtle_surrogate_pair_esc_esc_03() {
- // escaped high, escaped low
- surrogate("<\\ud800\\udc00>");
- }
-
- @Test public void turtle_surrogate_pair_esc_raw_01() {
- // escaped high, raw low
- surrogate("'\\ud800\udc00'");
- }
-
- @Test public void turtle_surrogate_pair_esc_raw_02() {
- // escaped high, raw low
- surrogate("'''\\ud800\udc00'''");
- }
- @Test public void turtle_surrogate_pair_esc_raw_03() {
- // escaped high, raw low
- surrogate("<\\ud800\udc00>");
- }
- @Test public void turtle_surrogate_pair_esc_raw_04() {
- // escaped high, raw low
- surrogate("_:b\ud800\udc00");
- }
-
- // Compilation failure - illegal escape character
-// @Test public void turtle_surrogate_pair_raw_esc_01() {
-// // raw high, escaped low
-// surrogate("'\ud800\\udc00'");
-// }
-
- @Test public void turtle_surrogate_pair_raw_raw_01() {
- // raw high, raw low
- surrogate("'\ud800\udc00'");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_02() {
- // raw high, raw low
- surrogate("'''\ud800\udc00'''");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_03() {
- // raw high, raw low
- surrogate("<\ud800\udc00>");
- }
-
- // Blank nodes label allow unicode but not unicode escapes.
- @Test public void turtle_surrogate_pair_raw_raw_04() {
- // raw high, raw low
- surrogate("_:b\ud800\udc00");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_05() {
- // escaped high, escaped low
- surrogate("ns:\ud800\udc00");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_06() {
- // escaped high, escaped low
- surrogate("\ud800\udc00:local");
- }
-
- @Test public void turtle_surrogate_pair_esc_esc_internal_01() {
- // escaped high, escaped low
- surrogate("'a\\ud800\\udc00x'");
- }
-
- @Test public void turtle_surrogate_pair_esc_esc_internal_02() {
- // escaped high, escaped low
- surrogate("'''a\\ud800\\udc00x'''");
- }
-
- @Test public void turtle_surrogate_pair_esc_esc_internal_03() {
- // escaped high, escaped low
- surrogate("<a\\ud800\\udc00x>");
- }
-
- @Test public void turtle_surrogate_pair_esc_raw_internal_01() {
- // escaped high, raw low
- surrogate("'z\\ud800\udc00z'");
- }
-
- @Test public void turtle_surrogate_pair_esc_raw_internal_02() {
- // escaped high, raw low
- surrogate("'''z\\ud800\udc00z'''");
- }
-
- @Test public void turtle_surrogate_pair_esc_raw_internal_03() {
- // escaped high, raw low
- surrogate("<z\\ud800\udc00z>");
- }
-
- // Compilation failure - illegal escape character
-// @Test public void turtle_surrogate_pair_raw_esc() {
-// // raw high, escaped low
-// surrogate("'a\ud800\\udc00'z");
-// }
-
- @Test public void turtle_surrogate_pair_raw_raw_internal_01() {
- // raw high, raw low
- surrogate("'a\ud800\udc00z'");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_internal_02() {
- // raw high, raw low
- surrogate("'''a\ud800\udc00z'''");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_internal_03() {
- // raw high, raw low
- surrogate("<a\ud800\udc00z>");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw_internal_04() {
- // raw high, raw low
- surrogate("_:ba\ud800\udc00z");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw__internal05() {
- // escaped high, escaped low
- surrogate("ns:x\ud800\udc00y");
- }
-
- @Test public void turtle_surrogate_pair_raw_raw__internal06() {
- // escaped high, escaped low
- surrogate("x\ud800\udc00y:local");
- }
-
- @Test
- public void turtle_bad_surrogate_01() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'\\ud800'"));
- }
-
- @Test
- public void turtle_bad_surrogate_01a() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'''\\ud800''''"));
- }
-
- @Test
- public void turtle_bad_surrogate_02() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'a\\ud800z'"));
- }
-
- @Test
- public void turtle_bad_surrogate_02a() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'''a\\ud800z'''"));
- }
-
- @Test
- public void turtle_bad_surrogate_02b() {
- assertThrows(RiotParseException.class, ()->
- surrogate("<a\\ud800z>"));
- }
-
- @Test
- public void turtle_bad_surrogate_03() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'\\udfff'"));
- }
-
- @Test
- public void turtle_bad_surrogate_04() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'a\\udfffz'"));
- }
-
- @Test
- public void turtle_bad_surrogate_05() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'\\U0000d800'"));
- }
-
- @Test
- public void turtle_bad_surrogate_06() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'a\\U0000d800z'"));
- }
-
- @Test
- public void turtle_bad_surrogate_07() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'\\U0000dfff'"));
- }
-
- @Test
- public void turtle_bad_surrogate_08() {
- assertThrows(RiotParseException.class, ()->
- surrogate("'a\\U0000dfffz'"));
- }
-
- @Test
- public void turtle_bad_surrogate_09() {
- // Wrong way round: low-hig);
- assertThrows(RiotParseException.class, ()->
- surrogate("'\\uc800\\ud800'"));
- }
-
- @Test
- public void turtle_bad_surrogate_10() {
- assertThrows(RiotParseException.class, ()->
- // Wrong way round: low-high
- surrogate("'a\\uc800\\ud800z'"));
- }
-
- // Compilation failure - illegal escape character
-// @Test
-// public void turtle_bad_surrogate_11() {
-// // raw low - escaped high
-// assertThrows(RiotParseException.class, ()->
-// surrogate("'\ud800\\ud800'"));
-// }
-//
-// @Test
-// public void turtle_bad_surrogate_12() {
-// // raw low - escaped high
-// assertThrows(RiotParseException.class, ()->
-// surrogate("'a\ud800\\ud800z'"));
-// }
-
- @Test
- public void turtle_bad_surrogate_13() {
- // escaped low - raw high
- assertThrows(RiotParseException.class, ()->
- surrogate("'\\uc800\ud800'"));
- }
-
- @Test
- public void turtle_bad_surrogate_14() {
- // escaped low - raw high
- assertThrows(RiotParseException.class, ()->
- surrogate("'a\\uc800\ud800z'"));
- }
-
- private void surrogate(String string) {
- Tokenizer tokenizer = tokenizer(string);
- assertTrue(tokenizer.hasNext());
- tokenizer.next();
- assertFalse(tokenizer.hasNext());
- }
-
// \-u{...} style Unicode escapes
@Test public void unescape_unicode_20() { test_unesc_unicode("\\u{41}", "A") ; }
@Test public void unescape_unicode_21() { test_unesc_unicode("\\u{000000}", "\u0000") ; }
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java
new file mode 100644
index 0000000000..13196c5176
--- /dev/null
+++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.apache.jena.riot.tokens;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.jena.riot.RiotException;
+
+// Tests for the variant of TokenizerText that handles escaped surrogates
+public class TestTokenizerTextAllowEscSurrogates extends AbstractTestTokenizerTextSurrogates {
+
+ @Override
+ protected Tokenizer tokenizer(String string) {
+ return TokenizerTextSurrogates.fromString(string);
+ }
+
+ // Structurally bad.
+ @Override
+ protected void surrogateBad(String string) {
+ assertThrows(RiotException.class, ()->surrogateTest(string));
+ }
+
+ // Validly encoded into escape sequences.
+ // Jena now rejects all surrogates, via any route.
+ // Testing the legacy TokenizerTextSurrogates
+ @Override
+ protected void surrogateValidEsc(String string) {
+ surrogateTest(string);
+ }
+
+ // These are "surrogateRawBad" in the "no surrogates" test suite
+ // but valid (and junk) if accepting surrogates.
+ // 6.1.0 compatibility.
+ // The esc-raw case is wrong in TokenizerTextSurrogates.checkCodepoint.
+ // It inserts a high surrogate early which then gets a real raw low surrogate and "works".
+ // Probably not what was intended!
+
+ @Override
+ @Test public void turtle_surrogate_pair_esc_raw_internal_01() {
+ //surrogateValidEsc("'z\\ud800\udc00z'");
+ surrogateValidEsc("'\\ud800\udc00z'");
+ }
+
+ @Override
+ @Test public void turtle_surrogate_pair_esc_raw_internal_02() {
+ surrogateValidEsc("'''z\\ud800\udc00z'''");
+ }
+
+ @Override
+ @Test public void turtle_surrogate_pair_esc_raw_internal_03() {
+ // escaped high, raw low
+ surrogateValidEsc("<z\\ud800\udc00z>");
+ }
+
+ @Override
+ @Test public void turtle_surrogate_pair_esc_raw_03() {
+ surrogateValidEsc("<\\ud800\udc00>");
+ }
+}
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java
similarity index 51%
copy from jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java
copy to jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java
index 9d4f87226c..87a6748155 100644
--- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java
+++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java
@@ -21,15 +21,30 @@
package org.apache.jena.riot.tokens;
-import org.junit.platform.suite.api.SelectClasses;
-import org.junit.platform.suite.api.Suite;
+import static org.junit.jupiter.api.Assertions.assertThrows;
-@Suite
-@SelectClasses({
- TestTokenizerText.class
- , TestTokenForNode.class
-})
+import org.apache.jena.riot.RiotException;
-public class TS_Tokens
-{ }
+// Tests for the variant of TokenizerText that handles escaped surrogates
+public class TestTokenizerTextNoSurrogates extends AbstractTestTokenizerTextSurrogates {
+ @Override
+ protected Tokenizer tokenizer(String string) {
+ return TokenizerText.fromString(string);
+ }
+
+ // Reject all use of surrogates.
+
+ // Structurally bad.
+ @Override
+ protected void surrogateBad(String string) {
+ assertThrows(RiotException.class, ()->surrogateTest(string));
+ }
+
+ // Validly encoded into escape sequences.
+ // Jena now rejects all surrogates, via any route.
+ @Override
+ protected void surrogateValidEsc(String string) {
+ assertThrows(RiotException.class, ()->surrogateTest(string));
+ }
+}
diff --git a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java
index 3ec13e2e9b..4502fe4639 100644
--- a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java
+++ b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java
@@ -205,7 +205,7 @@ public class TestSPARQLKeywordFunctions
@Test public void substr_30() { test("substr('\\uD83D\\uDC6A', 1)", "'\\uD83D\\uDC6A'"); }
// Same using Java string escapes.
@Test public void substr_30b() { test("substr('\uD83D\uDC6A', 1)", "'\uD83D\uDC6A'"); }
- @Test public void substr_31() { test("substr('\\uD83D\\uDC6A', 2)", kwEmptyString); }
+ @Test public void substr_31() { test("substr('\\uD83D\\uDC6A', 2)", kwEmptyString); }
@Test public void substr_32() { test("substr('ABC\\uD83D\\uDC6ADEF', 4, 1)", "'\\uD83D\\uDC6A'"); }
@Test public void substr_33() { test("substr('\\uD83D\\uDC6A!', -1, 3)", "'\\uD83D\\uDC6A'"); }
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java b/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java
index b77861901a..0ef6b9faff 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java
@@ -180,6 +180,13 @@ public class Chars
// REPLACEMENT CHARACTER
public static final char REPLACEMENT = 0xFFFD ;
+ /** Test for a surrogate codepoint */
+ public static boolean isSurrogate(int ch) {
+ // The Character class only has char tests.
+ // The range D800-DFFF (high/leading: D800-DBFF, low/trailing: DC00-DFFF)
+ return ch >= Character.MIN_HIGH_SURROGATE && ch <= Character.MAX_LOW_SURROGATE;
+ }
+
// "Not a character" - a Unicode non-character codepoint that is not legal in UTF-8.
public static final char NotACharacter = 0xFFFF ;
// Detect byte order by contrast (BOM reversed) - a Unicode non-character codepoint that is not legal in UTF-8.