You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hc.apache.org by ol...@apache.org on 2020/09/12 12:39:00 UTC
[httpcomponents-core] 07/18: Moved generic token parser out of
o.a.hc.core5.http.message to o.a.hc.core5.util package
This is an automated email from the ASF dual-hosted git repository.
olegk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/httpcomponents-core.git
commit 2e92ba7c273561f843fdee7eab09ea9bd00fe0dd
Author: Oleg Kalnichevski <ol...@apache.org>
AuthorDate: Fri Jul 24 11:13:32 2020 +0200
Moved generic token parser out of o.a.hc.core5.http.message to o.a.hc.core5.util package
---
.../core5/http/message/BasicHeaderValueParser.java | 15 +-
.../hc/core5/http/message/BasicLineParser.java | 39 ++--
.../hc/core5/http/message/BasicTokenIterator.java | 9 +-
.../hc/core5/http/message/BufferedHeader.java | 3 +-
.../hc/core5/http/message/MessageSupport.java | 5 +-
.../apache/hc/core5/http/message/ParserCursor.java | 54 +----
.../apache/hc/core5/http/message/TokenParser.java | 217 ++-------------------
.../java/org/apache/hc/core5/http/ssl/TLS.java | 4 +-
.../apache/hc/core5/http/ssl/TlsVersionParser.java | 13 +-
.../org/apache/hc/core5/net/URLEncodedUtils.java | 9 +-
.../TokenParser.java => util/Tokenizer.java} | 111 +++++++++--
.../hc/core5/http/ssl/TestTlsVersionParser.java | 7 +-
.../TestTokenizer.java} | 47 +++--
13 files changed, 189 insertions(+), 344 deletions(-)
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java
index 4cb6ba5..b1a063c 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java
@@ -36,6 +36,7 @@ import org.apache.hc.core5.annotation.ThreadingBehavior;
import org.apache.hc.core5.http.HeaderElement;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.util.Args;
+import org.apache.hc.core5.util.Tokenizer;
/**
* Default {@link org.apache.hc.core5.http.message.HeaderValueParser} implementation.
@@ -52,13 +53,13 @@ public class BasicHeaderValueParser implements HeaderValueParser {
// IMPORTANT!
// These private static variables must be treated as immutable and never exposed outside this class
- private static final BitSet TOKEN_DELIMS = TokenParser.INIT_BITSET('=', PARAM_DELIMITER, ELEM_DELIMITER);
- private static final BitSet VALUE_DELIMS = TokenParser.INIT_BITSET(PARAM_DELIMITER, ELEM_DELIMITER);
+ private static final BitSet TOKEN_DELIMS = Tokenizer.INIT_BITSET('=', PARAM_DELIMITER, ELEM_DELIMITER);
+ private static final BitSet VALUE_DELIMS = Tokenizer.INIT_BITSET(PARAM_DELIMITER, ELEM_DELIMITER);
- private final TokenParser tokenParser;
+ private final Tokenizer tokenizer;
public BasicHeaderValueParser() {
- this.tokenParser = TokenParser.INSTANCE;
+ this.tokenizer = Tokenizer.INSTANCE;
}
@Override
@@ -94,7 +95,7 @@ public class BasicHeaderValueParser implements HeaderValueParser {
public NameValuePair[] parseParameters(final CharSequence buffer, final ParserCursor cursor) {
Args.notNull(buffer, "Char sequence");
Args.notNull(cursor, "Parser cursor");
- tokenParser.skipWhiteSpace(buffer, cursor);
+ tokenizer.skipWhiteSpace(buffer, cursor);
final List<NameValuePair> params = new ArrayList<>();
while (!cursor.atEnd()) {
final NameValuePair param = parseNameValuePair(buffer, cursor);
@@ -112,7 +113,7 @@ public class BasicHeaderValueParser implements HeaderValueParser {
Args.notNull(buffer, "Char sequence");
Args.notNull(cursor, "Parser cursor");
- final String name = tokenParser.parseToken(buffer, cursor, TOKEN_DELIMS);
+ final String name = tokenizer.parseToken(buffer, cursor, TOKEN_DELIMS);
if (cursor.atEnd()) {
return new BasicNameValuePair(name, null);
}
@@ -121,7 +122,7 @@ public class BasicHeaderValueParser implements HeaderValueParser {
if (delim != '=') {
return new BasicNameValuePair(name, null);
}
- final String value = tokenParser.parseValue(buffer, cursor, VALUE_DELIMS);
+ final String value = tokenizer.parseValue(buffer, cursor, VALUE_DELIMS);
if (!cursor.atEnd()) {
cursor.updatePos(cursor.getPos() + 1);
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java
index 9b2e805..04dd847 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java
@@ -38,6 +38,7 @@ import org.apache.hc.core5.http.ProtocolVersion;
import org.apache.hc.core5.util.Args;
import org.apache.hc.core5.util.CharArrayBuffer;
import org.apache.hc.core5.util.TextUtils;
+import org.apache.hc.core5.util.Tokenizer;
/**
* Default {@link org.apache.hc.core5.http.message.LineParser} implementation.
@@ -51,16 +52,16 @@ public class BasicLineParser implements LineParser {
// IMPORTANT!
// These private static variables must be treated as immutable and never exposed outside this class
- private static final BitSet FULL_STOP = TokenParser.INIT_BITSET('.');
- private static final BitSet BLANKS = TokenParser.INIT_BITSET(' ', '\t');
- private static final BitSet COLON = TokenParser.INIT_BITSET(':');
+ private static final BitSet FULL_STOP = Tokenizer.INIT_BITSET('.');
+ private static final BitSet BLANKS = Tokenizer.INIT_BITSET(' ', '\t');
+ private static final BitSet COLON = Tokenizer.INIT_BITSET(':');
/**
* A version of the protocol to parse.
* The version is typically not relevant, but the protocol name.
*/
private final ProtocolVersion protocol;
- private final TokenParser tokenParser;
+ private final Tokenizer tokenizer;
/**
* Creates a new line parser for the given HTTP-like protocol.
@@ -71,7 +72,7 @@ public class BasicLineParser implements LineParser {
*/
public BasicLineParser(final ProtocolVersion proto) {
this.protocol = proto != null? proto : HttpVersion.HTTP_1_1;
- this.tokenParser = TokenParser.INSTANCE;
+ this.tokenizer = Tokenizer.INSTANCE;
}
/**
@@ -87,7 +88,7 @@ public class BasicLineParser implements LineParser {
final String protoname = this.protocol.getProtocol();
final int protolength = protoname.length();
- this.tokenParser.skipWhiteSpace(buffer, cursor);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
final int pos = cursor.getPos();
@@ -112,7 +113,7 @@ public class BasicLineParser implements LineParser {
cursor.updatePos(pos + protolength + 1);
- final String token1 = this.tokenParser.parseToken(buffer, cursor, FULL_STOP);
+ final String token1 = this.tokenizer.parseToken(buffer, cursor, FULL_STOP);
final int major;
try {
major = Integer.parseInt(token1);
@@ -125,7 +126,7 @@ public class BasicLineParser implements LineParser {
buffer, cursor.getLowerBound(), cursor.getUpperBound(), cursor.getPos());
}
cursor.updatePos(cursor.getPos() + 1);
- final String token2 = this.tokenParser.parseToken(buffer, cursor, BLANKS);
+ final String token2 = this.tokenizer.parseToken(buffer, cursor, BLANKS);
final int minor;
try {
minor = Integer.parseInt(token2);
@@ -150,20 +151,20 @@ public class BasicLineParser implements LineParser {
Args.notNull(buffer, "Char array buffer");
final ParserCursor cursor = new ParserCursor(0, buffer.length());
- this.tokenParser.skipWhiteSpace(buffer, cursor);
- final String method = this.tokenParser.parseToken(buffer, cursor, BLANKS);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
+ final String method = this.tokenizer.parseToken(buffer, cursor, BLANKS);
if (TextUtils.isEmpty(method)) {
throw new ParseException("Invalid request line",
buffer, cursor.getLowerBound(), cursor.getUpperBound(), cursor.getPos());
}
- this.tokenParser.skipWhiteSpace(buffer, cursor);
- final String uri = this.tokenParser.parseToken(buffer, cursor, BLANKS);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
+ final String uri = this.tokenizer.parseToken(buffer, cursor, BLANKS);
if (TextUtils.isEmpty(uri)) {
throw new ParseException("Invalid request line",
buffer, cursor.getLowerBound(), cursor.getUpperBound(), cursor.getPos());
}
final ProtocolVersion ver = parseProtocolVersion(buffer, cursor);
- this.tokenParser.skipWhiteSpace(buffer, cursor);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
if (!cursor.atEnd()) {
throw new ParseException("Invalid request line",
buffer, cursor.getLowerBound(), cursor.getUpperBound(), cursor.getPos());
@@ -176,10 +177,10 @@ public class BasicLineParser implements LineParser {
Args.notNull(buffer, "Char array buffer");
final ParserCursor cursor = new ParserCursor(0, buffer.length());
- this.tokenParser.skipWhiteSpace(buffer, cursor);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
final ProtocolVersion ver = parseProtocolVersion(buffer, cursor);
- this.tokenParser.skipWhiteSpace(buffer, cursor);
- final String s = this.tokenParser.parseToken(buffer, cursor, BLANKS);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
+ final String s = this.tokenizer.parseToken(buffer, cursor, BLANKS);
for (int i = 0; i < s.length(); i++) {
if (!Character.isDigit(s.charAt(i))) {
throw new ParseException("Status line contains invalid status code",
@@ -202,12 +203,12 @@ public class BasicLineParser implements LineParser {
Args.notNull(buffer, "Char array buffer");
final ParserCursor cursor = new ParserCursor(0, buffer.length());
- this.tokenParser.skipWhiteSpace(buffer, cursor);
- final String name = this.tokenParser.parseToken(buffer, cursor, COLON);
+ this.tokenizer.skipWhiteSpace(buffer, cursor);
+ final String name = this.tokenizer.parseToken(buffer, cursor, COLON);
if (cursor.getPos() == cursor.getLowerBound() || cursor.getPos() == cursor.getUpperBound() ||
buffer.charAt(cursor.getPos()) != ':' ||
TextUtils.isEmpty(name) ||
- TokenParser.isWhitespace(buffer.charAt(cursor.getPos() - 1))) {
+ Tokenizer.isWhitespace(buffer.charAt(cursor.getPos() - 1))) {
throw new ParseException("Invalid header",
buffer, cursor.getLowerBound(), cursor.getUpperBound(), cursor.getPos());
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java
index d58207b..45ca98d 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java
@@ -32,6 +32,7 @@ import java.util.Iterator;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.util.TextUtils;
+import org.apache.hc.core5.util.Tokenizer;
/**
* {@link java.util.Iterator} of {@link org.apache.hc.core5.http.Header} tokens..
@@ -40,9 +41,9 @@ import org.apache.hc.core5.util.TextUtils;
*/
public class BasicTokenIterator extends AbstractHeaderElementIterator<String> {
- private static final BitSet COMMA = TokenParser.INIT_BITSET(',');
+ private static final BitSet COMMA = Tokenizer.INIT_BITSET(',');
- private final TokenParser parser;
+ private final Tokenizer tokenizer;
/**
* Creates a new instance of {@link BasicTokenIterator}.
@@ -51,12 +52,12 @@ public class BasicTokenIterator extends AbstractHeaderElementIterator<String> {
*/
public BasicTokenIterator(final Iterator<Header> headerIterator) {
super(headerIterator);
- this.parser = TokenParser.INSTANCE;
+ this.tokenizer = Tokenizer.INSTANCE;
}
@Override
String parseHeaderElement(final CharSequence buf, final ParserCursor cursor) {
- final String token = this.parser.parseToken(buf, cursor, COMMA);
+ final String token = this.tokenizer.parseToken(buf, cursor, COMMA);
if (!cursor.atEnd()) {
final int pos = cursor.getPos();
if (buf.charAt(pos) == ',') {
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BufferedHeader.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BufferedHeader.java
index b4ff32d..6148e09 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BufferedHeader.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BufferedHeader.java
@@ -33,6 +33,7 @@ import org.apache.hc.core5.http.FormattedHeader;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.util.Args;
import org.apache.hc.core5.util.CharArrayBuffer;
+import org.apache.hc.core5.util.Tokenizer;
/**
* This class represents a raw HTTP header whose content is parsed 'on demand'
@@ -90,7 +91,7 @@ public class BufferedHeader implements FormattedHeader, Serializable {
if (colon <= 0) {
throw new ParseException("Invalid header", buffer, 0, buffer.length());
}
- if (strict && TokenParser.isWhitespace(buffer.charAt(colon - 1))) {
+ if (strict && Tokenizer.isWhitespace(buffer.charAt(colon - 1))) {
throw new ParseException("Invalid header", buffer, 0, buffer.length(), colon - 1);
}
final String s = buffer.substringTrimmed(0, colon);
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java
index b875527..263cc2d 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java
@@ -45,6 +45,7 @@ import org.apache.hc.core5.http.Method;
import org.apache.hc.core5.util.Args;
import org.apache.hc.core5.util.CharArrayBuffer;
import org.apache.hc.core5.util.TextUtils;
+import org.apache.hc.core5.util.Tokenizer;
/**
* Support methods for HTTP message processing.
@@ -100,7 +101,7 @@ public class MessageSupport {
return BufferedHeader.create(buffer);
}
- private static final BitSet COMMA = TokenParser.INIT_BITSET(',');
+ private static final BitSet COMMA = Tokenizer.INIT_BITSET(',');
public static Set<String> parseTokens(final CharSequence src, final ParserCursor cursor) {
Args.notNull(src, "Source");
@@ -111,7 +112,7 @@ public class MessageSupport {
if (src.charAt(pos) == ',') {
cursor.updatePos(pos + 1);
}
- final String token = TokenParser.INSTANCE.parseToken(src, cursor, COMMA);
+ final String token = Tokenizer.INSTANCE.parseToken(src, cursor, COMMA);
if (!TextUtils.isBlank(token)) {
tokens.add(token);
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/ParserCursor.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/ParserCursor.java
index fcbd2e4..dcf51df 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/ParserCursor.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/ParserCursor.java
@@ -27,7 +27,7 @@
package org.apache.hc.core5.http.message;
-import org.apache.hc.core5.util.Args;
+import org.apache.hc.core5.util.Tokenizer;
/**
* This class represents a context of a parsing operation:
@@ -38,58 +38,10 @@ import org.apache.hc.core5.util.Args;
*
* @since 4.0
*/
-public class ParserCursor {
-
- private final int lowerBound;
- private final int upperBound;
- private int pos;
+public class ParserCursor extends Tokenizer.Cursor {
public ParserCursor(final int lowerBound, final int upperBound) {
- super();
- Args.notNegative(lowerBound, "lowerBound");
- Args.check(lowerBound <= upperBound, "lowerBound cannot be greater than upperBound");
- this.lowerBound = lowerBound;
- this.upperBound = upperBound;
- this.pos = lowerBound;
- }
-
- public int getLowerBound() {
- return this.lowerBound;
- }
-
- public int getUpperBound() {
- return this.upperBound;
- }
-
- public int getPos() {
- return this.pos;
- }
-
- public void updatePos(final int pos) {
- if (pos < this.lowerBound) {
- throw new IndexOutOfBoundsException("pos: "+pos+" < lowerBound: "+this.lowerBound);
- }
- if (pos > this.upperBound) {
- throw new IndexOutOfBoundsException("pos: "+pos+" > upperBound: "+this.upperBound);
- }
- this.pos = pos;
- }
-
- public boolean atEnd() {
- return this.pos >= this.upperBound;
- }
-
- @Override
- public String toString() {
- final StringBuilder buffer = new StringBuilder();
- buffer.append('[');
- buffer.append(Integer.toString(this.lowerBound));
- buffer.append('>');
- buffer.append(Integer.toString(this.pos));
- buffer.append('>');
- buffer.append(Integer.toString(this.upperBound));
- buffer.append(']');
- return buffer.toString();
+ super(lowerBound, upperBound);
}
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java
index cdd6a21..e8dcbd0 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java
@@ -31,8 +31,7 @@ import java.util.BitSet;
import org.apache.hc.core5.annotation.Contract;
import org.apache.hc.core5.annotation.ThreadingBehavior;
-import org.apache.hc.core5.http.Chars;
-import org.apache.hc.core5.util.Args;
+import org.apache.hc.core5.util.Tokenizer;
/**
* Low level parser for header field elements. The parsing routines of this class are designed
@@ -41,17 +40,14 @@ import org.apache.hc.core5.util.Args;
* This class is immutable and thread safe.
*
* @since 4.4
+ *
+ * @deprecated Use {@link Tokenizer}
*/
+@Deprecated
@Contract(threading = ThreadingBehavior.IMMUTABLE)
-public class TokenParser {
+public class TokenParser extends Tokenizer {
- public static BitSet INIT_BITSET(final int ... b) {
- final BitSet bitset = new BitSet();
- for (final int aB : b) {
- bitset.set(aB);
- }
- return bitset;
- }
+ public static final TokenParser INSTANCE = new TokenParser();
/** Double quote */
public static final char DQUOTE = '\"';
@@ -59,211 +55,30 @@ public class TokenParser {
/** Backward slash / escape character */
public static final char ESCAPE = '\\';
- public static boolean isWhitespace(final char ch) {
- return ch == Chars.SP || ch == Chars.HT || ch == Chars.CR || ch == Chars.LF;
- }
-
- public static final TokenParser INSTANCE = new TokenParser();
-
- /**
- * Extracts from the sequence of chars a token terminated with any of the given delimiters
- * discarding semantically insignificant whitespace characters.
- *
- * @param buf buffer with the sequence of chars to be parsed
- * @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the token
- * is not delimited by any character.
- */
public String parseToken(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters) {
- Args.notNull(buf, "Char sequence");
- Args.notNull(cursor, "Parser cursor");
- final StringBuilder dst = new StringBuilder();
- boolean whitespace = false;
- while (!cursor.atEnd()) {
- final char current = buf.charAt(cursor.getPos());
- if (delimiters != null && delimiters.get(current)) {
- break;
- } else if (isWhitespace(current)) {
- skipWhiteSpace(buf, cursor);
- whitespace = true;
- } else {
- if (whitespace && dst.length() > 0) {
- dst.append(' ');
- }
- copyContent(buf, cursor, delimiters, dst);
- whitespace = false;
- }
- }
- return dst.toString();
+ return super.parseToken(buf, cursor, delimiters);
}
- /**
- * Extracts from the sequence of chars a value which can be enclosed in quote marks and
- * terminated with any of the given delimiters discarding semantically insignificant
- * whitespace characters.
- *
- * @param buf buffer with the sequence of chars to be parsed
- * @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the value
- * is not delimited by any character.
- */
public String parseValue(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters) {
- Args.notNull(buf, "Char sequence");
- Args.notNull(cursor, "Parser cursor");
- final StringBuilder dst = new StringBuilder();
- boolean whitespace = false;
- while (!cursor.atEnd()) {
- final char current = buf.charAt(cursor.getPos());
- if (delimiters != null && delimiters.get(current)) {
- break;
- } else if (isWhitespace(current)) {
- skipWhiteSpace(buf, cursor);
- whitespace = true;
- } else if (current == DQUOTE) {
- if (whitespace && dst.length() > 0) {
- dst.append(' ');
- }
- copyQuotedContent(buf, cursor, dst);
- whitespace = false;
- } else {
- if (whitespace && dst.length() > 0) {
- dst.append(' ');
- }
- copyUnquotedContent(buf, cursor, delimiters, dst);
- whitespace = false;
- }
- }
- return dst.toString();
+ return super.parseValue(buf, cursor, delimiters);
}
- /**
- * Skips semantically insignificant whitespace characters and moves the cursor to the closest
- * non-whitespace character.
- *
- * @param buf buffer with the sequence of chars to be parsed
- * @param cursor defines the bounds and current position of the buffer
- */
public void skipWhiteSpace(final CharSequence buf, final ParserCursor cursor) {
- Args.notNull(buf, "Char sequence");
- Args.notNull(cursor, "Parser cursor");
- int pos = cursor.getPos();
- final int indexFrom = cursor.getPos();
- final int indexTo = cursor.getUpperBound();
- for (int i = indexFrom; i < indexTo; i++) {
- final char current = buf.charAt(i);
- if (!isWhitespace(current)) {
- break;
- }
- pos++;
- }
- cursor.updatePos(pos);
+ super.skipWhiteSpace(buf, cursor);
}
- /**
- * Transfers content into the destination buffer until a whitespace character or any of
- * the given delimiters is encountered.
- *
- * @param buf buffer with the sequence of chars to be parsed
- * @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the value
- * is delimited by a whitespace only.
- * @param dst destination buffer
- */
public void copyContent(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters,
- final StringBuilder dst) {
- Args.notNull(buf, "Char sequence");
- Args.notNull(cursor, "Parser cursor");
- Args.notNull(dst, "String builder");
- int pos = cursor.getPos();
- final int indexFrom = cursor.getPos();
- final int indexTo = cursor.getUpperBound();
- for (int i = indexFrom; i < indexTo; i++) {
- final char current = buf.charAt(i);
- if ((delimiters != null && delimiters.get(current)) || isWhitespace(current)) {
- break;
- }
- pos++;
- dst.append(current);
- }
- cursor.updatePos(pos);
+ final StringBuilder dst) {
+ super.copyContent(buf, cursor, delimiters, dst);
}
- /**
- * Transfers content into the destination buffer until a whitespace character, a quote,
- * or any of the given delimiters is encountered.
- *
- * @param buf buffer with the sequence of chars to be parsed
- * @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the value
- * is delimited by a whitespace or a quote only.
- * @param dst destination buffer
- */
- public void copyUnquotedContent(final CharSequence buf, final ParserCursor cursor,
- final BitSet delimiters, final StringBuilder dst) {
- Args.notNull(buf, "Char sequence");
- Args.notNull(cursor, "Parser cursor");
- Args.notNull(dst, "String builder");
- int pos = cursor.getPos();
- final int indexFrom = cursor.getPos();
- final int indexTo = cursor.getUpperBound();
- for (int i = indexFrom; i < indexTo; i++) {
- final char current = buf.charAt(i);
- if ((delimiters != null && delimiters.get(current))
- || isWhitespace(current) || current == DQUOTE) {
- break;
- }
- pos++;
- dst.append(current);
- }
- cursor.updatePos(pos);
+ public void copyUnquotedContent(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters,
+ final StringBuilder dst) {
+ super.copyUnquotedContent(buf, cursor, delimiters, dst);
}
- /**
- * Transfers content enclosed with quote marks into the destination buffer.
- *
- * @param buf buffer with the sequence of chars to be parsed
- * @param cursor defines the bounds and current position of the buffer
- * @param dst destination buffer
- */
- public void copyQuotedContent(final CharSequence buf, final ParserCursor cursor,
- final StringBuilder dst) {
- Args.notNull(buf, "Char sequence");
- Args.notNull(cursor, "Parser cursor");
- Args.notNull(dst, "String builder");
- if (cursor.atEnd()) {
- return;
- }
- int pos = cursor.getPos();
- int indexFrom = cursor.getPos();
- final int indexTo = cursor.getUpperBound();
- char current = buf.charAt(pos);
- if (current != DQUOTE) {
- return;
- }
- pos++;
- indexFrom++;
- boolean escaped = false;
- for (int i = indexFrom; i < indexTo; i++, pos++) {
- current = buf.charAt(i);
- if (escaped) {
- if (current != DQUOTE && current != ESCAPE) {
- dst.append(ESCAPE);
- }
- dst.append(current);
- escaped = false;
- } else {
- if (current == DQUOTE) {
- pos++;
- break;
- }
- if (current == ESCAPE) {
- escaped = true;
- } else if (current != Chars.CR && current != Chars.LF) {
- dst.append(current);
- }
- }
- }
- cursor.updatePos(pos);
+ public void copyQuotedContent(final CharSequence buf, final ParserCursor cursor, final StringBuilder dst) {
+ super.copyQuotedContent(buf, cursor, dst);
}
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TLS.java b/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TLS.java
index c84cb8c..6836cdf 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TLS.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TLS.java
@@ -32,7 +32,7 @@ import java.util.List;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.ProtocolVersion;
-import org.apache.hc.core5.http.message.ParserCursor;
+import org.apache.hc.core5.util.Tokenizer;
/**
* Supported {@code TLS} protocol versions.
@@ -74,7 +74,7 @@ public enum TLS {
if (s == null) {
return null;
}
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
return TlsVersionParser.INSTANCE.parse(s, cursor, null);
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java
index d207da8..6ad359c 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java
@@ -31,22 +31,21 @@ import java.util.BitSet;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.ProtocolVersion;
-import org.apache.hc.core5.http.message.ParserCursor;
-import org.apache.hc.core5.http.message.TokenParser;
+import org.apache.hc.core5.util.Tokenizer;
final class TlsVersionParser {
public final static TlsVersionParser INSTANCE = new TlsVersionParser();
- private final TokenParser tokenParser;
+ private final Tokenizer tokenizer;
TlsVersionParser() {
- this.tokenParser = TokenParser.INSTANCE;
+ this.tokenizer = Tokenizer.INSTANCE;
}
ProtocolVersion parse(
final CharSequence buffer,
- final ParserCursor cursor,
+ final Tokenizer.Cursor cursor,
final BitSet delimiters) throws ParseException {
final int lowerBound = cursor.getLowerBound();
final int upperBound = cursor.getUpperBound();
@@ -64,7 +63,7 @@ final class TlsVersionParser {
if (cursor.atEnd()) {
throw new ParseException("Invalid TLS version", buffer, lowerBound, upperBound, pos);
}
- final String s = this.tokenParser.parseToken(buffer, cursor, delimiters);
+ final String s = this.tokenizer.parseToken(buffer, cursor, delimiters);
final int idx = s.indexOf('.');
if (idx == -1) {
final int major;
@@ -97,7 +96,7 @@ final class TlsVersionParser {
if (s == null) {
return null;
}
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
return parse(s, cursor, null);
}
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/net/URLEncodedUtils.java b/httpcore5/src/main/java/org/apache/hc/core5/net/URLEncodedUtils.java
index 93376be..c2131ea 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/net/URLEncodedUtils.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/net/URLEncodedUtils.java
@@ -40,9 +40,8 @@ import java.util.List;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.message.BasicNameValuePair;
-import org.apache.hc.core5.http.message.ParserCursor;
-import org.apache.hc.core5.http.message.TokenParser;
import org.apache.hc.core5.util.Args;
+import org.apache.hc.core5.util.Tokenizer;
/**
* A collection of utilities for encoding URLs.
@@ -110,12 +109,12 @@ public class URLEncodedUtils {
public static List<NameValuePair> parse(
final CharSequence s, final Charset charset, final char... separators) {
Args.notNull(s, "Char sequence");
- final TokenParser tokenParser = TokenParser.INSTANCE;
+ final Tokenizer tokenParser = Tokenizer.INSTANCE;
final BitSet delimSet = new BitSet();
for (final char separator: separators) {
delimSet.set(separator);
}
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
final List<NameValuePair> list = new ArrayList<>();
while (!cursor.atEnd()) {
delimSet.set('=');
@@ -142,7 +141,7 @@ public class URLEncodedUtils {
}
static List<String> splitSegments(final CharSequence s, final BitSet separators) {
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
// Skip leading separator
if (cursor.atEnd()) {
return Collections.emptyList();
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java b/httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java
similarity index 71%
copy from httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java
copy to httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java
index cdd6a21..ee66fd3 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/TokenParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java
@@ -25,25 +25,80 @@
*
*/
-package org.apache.hc.core5.http.message;
+package org.apache.hc.core5.util;
import java.util.BitSet;
import org.apache.hc.core5.annotation.Contract;
import org.apache.hc.core5.annotation.ThreadingBehavior;
-import org.apache.hc.core5.http.Chars;
-import org.apache.hc.core5.util.Args;
/**
- * Low level parser for header field elements. The parsing routines of this class are designed
- * to produce near zero intermediate garbage and make no intermediate copies of input data.
+ * Tokenizer that can be used as a foundation for more complex parsing routines.
+ * Methods of this class are designed to produce near zero intermediate garbage
+ * and make no intermediate copies of input data.
* <p>
* This class is immutable and thread safe.
*
- * @since 4.4
+ * @since 5.1
*/
@Contract(threading = ThreadingBehavior.IMMUTABLE)
-public class TokenParser {
+public class Tokenizer {
+
+ public static class Cursor {
+
+ private final int lowerBound;
+ private final int upperBound;
+ private int pos;
+
+ public Cursor(final int lowerBound, final int upperBound) {
+ super();
+ Args.notNegative(lowerBound, "lowerBound");
+ Args.check(lowerBound <= upperBound, "lowerBound cannot be greater than upperBound");
+ this.lowerBound = lowerBound;
+ this.upperBound = upperBound;
+ this.pos = lowerBound;
+ }
+
+ public int getLowerBound() {
+ return this.lowerBound;
+ }
+
+ public int getUpperBound() {
+ return this.upperBound;
+ }
+
+ public int getPos() {
+ return this.pos;
+ }
+
+ public void updatePos(final int pos) {
+ if (pos < this.lowerBound) {
+ throw new IndexOutOfBoundsException("pos: "+pos+" < lowerBound: "+this.lowerBound);
+ }
+ if (pos > this.upperBound) {
+ throw new IndexOutOfBoundsException("pos: "+pos+" > upperBound: "+this.upperBound);
+ }
+ this.pos = pos;
+ }
+
+ public boolean atEnd() {
+ return this.pos >= this.upperBound;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder();
+ buffer.append('[');
+ buffer.append(Integer.toString(this.lowerBound));
+ buffer.append('>');
+ buffer.append(Integer.toString(this.pos));
+ buffer.append('>');
+ buffer.append(Integer.toString(this.upperBound));
+ buffer.append(']');
+ return buffer.toString();
+ }
+
+ }
public static BitSet INIT_BITSET(final int ... b) {
final BitSet bitset = new BitSet();
@@ -59,11 +114,33 @@ public class TokenParser {
/** Backward slash / escape character */
public static final char ESCAPE = '\\';
+ public static final int CR = 13; // <US-ASCII CR, carriage return (13)>
+ public static final int LF = 10; // <US-ASCII LF, linefeed (10)>
+ public static final int SP = 32; // <US-ASCII SP, space (32)>
+ public static final int HT = 9; // <US-ASCII HT, horizontal-tab (9)>
+
public static boolean isWhitespace(final char ch) {
- return ch == Chars.SP || ch == Chars.HT || ch == Chars.CR || ch == Chars.LF;
+ return ch == SP || ch == HT || ch == CR || ch == LF;
}
- public static final TokenParser INSTANCE = new TokenParser();
+ public static final Tokenizer INSTANCE = new Tokenizer();
+
+ /**
+ * Extracts from the sequence of chars a token terminated with any of the given delimiters
+ * or a whitespace characters.
+ *
+ * @param buf buffer with the sequence of chars to be parsed
+ * @param cursor defines the bounds and current position of the buffer
+ * @param delimiters set of delimiting characters. Can be {@code null} if the token
+ * is not delimited by any character.
+ */
+ public String parseContent(final CharSequence buf, final Cursor cursor, final BitSet delimiters) {
+ Args.notNull(buf, "Char sequence");
+ Args.notNull(cursor, "Parser cursor");
+ final StringBuilder dst = new StringBuilder();
+ copyContent(buf, cursor, delimiters, dst);
+ return dst.toString();
+ }
/**
* Extracts from the sequence of chars a token terminated with any of the given delimiters
@@ -74,7 +151,7 @@ public class TokenParser {
* @param delimiters set of delimiting characters. Can be {@code null} if the token
* is not delimited by any character.
*/
- public String parseToken(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters) {
+ public String parseToken(final CharSequence buf, final Cursor cursor, final BitSet delimiters) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
final StringBuilder dst = new StringBuilder();
@@ -107,7 +184,7 @@ public class TokenParser {
* @param delimiters set of delimiting characters. Can be {@code null} if the value
* is not delimited by any character.
*/
- public String parseValue(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters) {
+ public String parseValue(final CharSequence buf, final Cursor cursor, final BitSet delimiters) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
final StringBuilder dst = new StringBuilder();
@@ -143,7 +220,7 @@ public class TokenParser {
* @param buf buffer with the sequence of chars to be parsed
* @param cursor defines the bounds and current position of the buffer
*/
- public void skipWhiteSpace(final CharSequence buf, final ParserCursor cursor) {
+ public void skipWhiteSpace(final CharSequence buf, final Cursor cursor) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
int pos = cursor.getPos();
@@ -169,8 +246,8 @@ public class TokenParser {
* is delimited by a whitespace only.
* @param dst destination buffer
*/
- public void copyContent(final CharSequence buf, final ParserCursor cursor, final BitSet delimiters,
- final StringBuilder dst) {
+ public void copyContent(final CharSequence buf, final Cursor cursor, final BitSet delimiters,
+ final StringBuilder dst) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
Args.notNull(dst, "String builder");
@@ -198,7 +275,7 @@ public class TokenParser {
* is delimited by a whitespace or a quote only.
* @param dst destination buffer
*/
- public void copyUnquotedContent(final CharSequence buf, final ParserCursor cursor,
+ public void copyUnquotedContent(final CharSequence buf, final Cursor cursor,
final BitSet delimiters, final StringBuilder dst) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
@@ -225,7 +302,7 @@ public class TokenParser {
* @param cursor defines the bounds and current position of the buffer
* @param dst destination buffer
*/
- public void copyQuotedContent(final CharSequence buf, final ParserCursor cursor,
+ public void copyQuotedContent(final CharSequence buf, final Cursor cursor,
final StringBuilder dst) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
@@ -258,7 +335,7 @@ public class TokenParser {
}
if (current == ESCAPE) {
escaped = true;
- } else if (current != Chars.CR && current != Chars.LF) {
+ } else if (current != CR && current != LF) {
dst.append(current);
}
}
diff --git a/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java b/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java
index a6baccc..221f49d 100644
--- a/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java
+++ b/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java
@@ -29,8 +29,7 @@ package org.apache.hc.core5.http.ssl;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.ProtocolVersion;
-import org.apache.hc.core5.http.message.ParserCursor;
-import org.apache.hc.core5.http.message.TokenParser;
+import org.apache.hc.core5.util.Tokenizer;
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Before;
@@ -59,8 +58,8 @@ public class TestTlsVersionParser {
@Test
public void testParseBuffer() throws Exception {
- final ParserCursor cursor = new ParserCursor(1, 13);
- Assert.assertThat(impl.parse(" TLSv1.2,0000", cursor, TokenParser.INIT_BITSET(',')),
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(1, 13);
+ Assert.assertThat(impl.parse(" TLSv1.2,0000", cursor, Tokenizer.INIT_BITSET(',')),
CoreMatchers.equalTo(TLS.V_1_2.version));
Assert.assertThat(cursor.getPos(), CoreMatchers.equalTo(8));
}
diff --git a/httpcore5/src/test/java/org/apache/hc/core5/http/message/TestTokenParser.java b/httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java
similarity index 83%
rename from httpcore5/src/test/java/org/apache/hc/core5/http/message/TestTokenParser.java
rename to httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java
index 1db53f1..e77cd28 100644
--- a/httpcore5/src/test/java/org/apache/hc/core5/http/message/TestTokenParser.java
+++ b/httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java
@@ -25,20 +25,19 @@
*
*/
-package org.apache.hc.core5.http.message;
+package org.apache.hc.core5.util;
-import org.apache.hc.core5.util.CharArrayBuffer;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-public class TestTokenParser {
+public class TestTokenizer {
- private TokenParser parser;
+ private Tokenizer parser;
@Before
public void setUp() throws Exception {
- parser = new TokenParser();
+ parser = new Tokenizer();
}
private static CharArrayBuffer createBuffer(final String value) {
@@ -54,7 +53,7 @@ public class TestTokenParser {
public void testBasicTokenParsing() throws Exception {
final String s = " raw: \" some stuff \"";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
parser.skipWhiteSpace(raw, cursor);
@@ -62,7 +61,7 @@ public class TestTokenParser {
Assert.assertEquals(3, cursor.getPos());
final StringBuilder strbuf1 = new StringBuilder();
- parser.copyContent(raw, cursor, TokenParser.INIT_BITSET(':'), strbuf1);
+ parser.copyContent(raw, cursor, Tokenizer.INIT_BITSET(':'), strbuf1);
Assert.assertFalse(cursor.atEnd());
Assert.assertEquals(6, cursor.getPos());
@@ -92,7 +91,7 @@ public class TestTokenParser {
public void testTokenParsingWithQuotedPairs() throws Exception {
final String s = "raw: \"\\\"some\\stuff\\\\\"";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
parser.skipWhiteSpace(raw, cursor);
@@ -100,7 +99,7 @@ public class TestTokenParser {
Assert.assertEquals(0, cursor.getPos());
final StringBuilder strbuf1 = new StringBuilder();
- parser.copyContent(raw, cursor, TokenParser.INIT_BITSET(':'), strbuf1);
+ parser.copyContent(raw, cursor, Tokenizer.INIT_BITSET(':'), strbuf1);
Assert.assertFalse(cursor.atEnd());
Assert.assertEquals("raw", strbuf1.toString());
@@ -122,7 +121,7 @@ public class TestTokenParser {
public void testTokenParsingIncompleteQuote() throws Exception {
final String s = "\"stuff and more stuff ";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
final StringBuilder strbuf1 = new StringBuilder();
parser.copyQuotedContent(raw, cursor, strbuf1);
Assert.assertEquals("stuff and more stuff ", strbuf1.toString());
@@ -132,8 +131,8 @@ public class TestTokenParser {
public void testTokenParsingTokensWithUnquotedBlanks() throws Exception {
final String s = " stuff and \tsome\tmore stuff ;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseToken(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseToken(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuff and some more stuff", result);
}
@@ -141,8 +140,8 @@ public class TestTokenParser {
public void testTokenParsingMixedValuesAndQuotedValues() throws Exception {
final String s = " stuff and \" some more \" \"stuff ;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuff and some more stuff ;", result);
}
@@ -150,8 +149,8 @@ public class TestTokenParser {
public void testTokenParsingMixedValuesAndQuotedValues2() throws Exception {
final String s = "stuff\"more\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuffmorestuff", result);
}
@@ -159,8 +158,8 @@ public class TestTokenParser {
public void testTokenParsingEscapedQuotes() throws Exception {
final String s = "stuff\"\\\"more\\\"\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuff\"more\"stuff", result);
}
@@ -168,8 +167,8 @@ public class TestTokenParser {
public void testTokenParsingEscapedDelimiter() throws Exception {
final String s = "stuff\"\\\"more\\\";\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuff\"more\";stuff", result);
}
@@ -177,8 +176,8 @@ public class TestTokenParser {
public void testTokenParsingEscapedSlash() throws Exception {
final String s = "stuff\"\\\"more\\\";\\\\\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuff\"more\";\\stuff", result);
}
@@ -186,8 +185,8 @@ public class TestTokenParser {
public void testTokenParsingSlashOutsideQuotes() throws Exception {
final String s = "stuff\\; more stuff;";
final CharArrayBuffer raw = createBuffer(s);
- final ParserCursor cursor = new ParserCursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, TokenParser.INIT_BITSET(';'));
+ final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
+ final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
Assert.assertEquals("stuff\\", result);
}
}