You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hc.apache.org by ol...@apache.org on 2024/01/18 16:07:17 UTC
(httpcomponents-core) branch 5.2.x updated: Replaced token delimitation based on BitSet with a predicate function
This is an automated email from the ASF dual-hosted git repository.
olegk pushed a commit to branch 5.2.x
in repository https://gitbox.apache.org/repos/asf/httpcomponents-core.git
The following commit(s) were added to refs/heads/5.2.x by this push:
new 279e88e72 Replaced token delimitation based on BitSet with a predicate function
279e88e72 is described below
commit 279e88e7265ba9441cb093255ebfbfbea5923073
Author: Oleg Kalnichevski <ol...@apache.org>
AuthorDate: Thu Jan 18 16:36:23 2024 +0100
Replaced token delimitation based on BitSet with a predicate function
---
.../core5/http/message/BasicHeaderValueParser.java | 5 +-
.../hc/core5/http/message/BasicLineParser.java | 8 +-
.../hc/core5/http/message/BasicTokenIterator.java | 3 +-
.../hc/core5/http/message/MessageSupport.java | 3 +-
.../apache/hc/core5/http/ssl/TlsVersionParser.java | 6 +-
.../main/java/org/apache/hc/core5/net/Host.java | 6 +-
.../java/org/apache/hc/core5/net/URIAuthority.java | 2 +-
.../java/org/apache/hc/core5/net/URIBuilder.java | 17 +--
.../java/org/apache/hc/core5/net/URISupport.java | 13 ++-
.../java/org/apache/hc/core5/util/Tokenizer.java | 130 ++++++++++++++++++---
.../hc/core5/http/ssl/TestTlsVersionParser.java | 2 +-
.../org/apache/hc/core5/util/TestTokenizer.java | 18 +--
12 files changed, 148 insertions(+), 65 deletions(-)
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java
index 5399e529d..4af7cd7cd 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicHeaderValueParser.java
@@ -28,7 +28,6 @@
package org.apache.hc.core5.http.message;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.List;
import org.apache.hc.core5.annotation.Contract;
@@ -53,8 +52,8 @@ public class BasicHeaderValueParser implements HeaderValueParser {
// IMPORTANT!
// These private static variables must be treated as immutable and never exposed outside this class
- private static final BitSet TOKEN_DELIMITER = Tokenizer.INIT_BITSET('=', PARAM_DELIMITER, ELEM_DELIMITER);
- private static final BitSet VALUE_DELIMITER = Tokenizer.INIT_BITSET(PARAM_DELIMITER, ELEM_DELIMITER);
+ private static final Tokenizer.Delimiter TOKEN_DELIMITER = Tokenizer.delimiters('=', PARAM_DELIMITER, ELEM_DELIMITER);
+ private static final Tokenizer.Delimiter VALUE_DELIMITER = Tokenizer.delimiters(PARAM_DELIMITER, ELEM_DELIMITER);
private final Tokenizer tokenizer;
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java
index 04dd847c1..de3b44f48 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicLineParser.java
@@ -27,8 +27,6 @@
package org.apache.hc.core5.http.message;
-import java.util.BitSet;
-
import org.apache.hc.core5.annotation.Contract;
import org.apache.hc.core5.annotation.ThreadingBehavior;
import org.apache.hc.core5.http.Header;
@@ -52,9 +50,9 @@ public class BasicLineParser implements LineParser {
// IMPORTANT!
// These private static variables must be treated as immutable and never exposed outside this class
- private static final BitSet FULL_STOP = Tokenizer.INIT_BITSET('.');
- private static final BitSet BLANKS = Tokenizer.INIT_BITSET(' ', '\t');
- private static final BitSet COLON = Tokenizer.INIT_BITSET(':');
+ private static final Tokenizer.Delimiter FULL_STOP = Tokenizer.delimiters('.');
+ private static final Tokenizer.Delimiter BLANKS = Tokenizer.delimiters(' ', '\t');
+ private static final Tokenizer.Delimiter COLON = Tokenizer.delimiters(':');
/**
* A version of the protocol to parse.
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java
index 45ca98d57..028b20733 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/BasicTokenIterator.java
@@ -27,7 +27,6 @@
package org.apache.hc.core5.http.message;
-import java.util.BitSet;
import java.util.Iterator;
import org.apache.hc.core5.http.Header;
@@ -41,7 +40,7 @@ import org.apache.hc.core5.util.Tokenizer;
*/
public class BasicTokenIterator extends AbstractHeaderElementIterator<String> {
- private static final BitSet COMMA = Tokenizer.INIT_BITSET(',');
+ private static final Tokenizer.Delimiter COMMA = Tokenizer.delimiters(',');
private final Tokenizer tokenizer;
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java b/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java
index aad640e03..f3a483005 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/message/MessageSupport.java
@@ -27,7 +27,6 @@
package org.apache.hc.core5.http.message;
-import java.util.BitSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Set;
@@ -106,7 +105,7 @@ public class MessageSupport {
return BufferedHeader.create(buffer);
}
- private static final BitSet COMMA = Tokenizer.INIT_BITSET(',');
+ private static final Tokenizer.Delimiter COMMA = Tokenizer.delimiters(',');
public static Set<String> parseTokens(final CharSequence src, final ParserCursor cursor) {
Args.notNull(src, "Source");
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java b/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java
index 6ad359cff..296e5f814 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/http/ssl/TlsVersionParser.java
@@ -27,8 +27,6 @@
package org.apache.hc.core5.http.ssl;
-import java.util.BitSet;
-
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.ProtocolVersion;
import org.apache.hc.core5.util.Tokenizer;
@@ -46,7 +44,7 @@ final class TlsVersionParser {
ProtocolVersion parse(
final CharSequence buffer,
final Tokenizer.Cursor cursor,
- final BitSet delimiters) throws ParseException {
+ final Tokenizer.Delimiter delimiterPredicate) throws ParseException {
final int lowerBound = cursor.getLowerBound();
final int upperBound = cursor.getUpperBound();
@@ -63,7 +61,7 @@ final class TlsVersionParser {
if (cursor.atEnd()) {
throw new ParseException("Invalid TLS version", buffer, lowerBound, upperBound, pos);
}
- final String s = this.tokenizer.parseToken(buffer, cursor, delimiters);
+ final String s = this.tokenizer.parseToken(buffer, cursor, delimiterPredicate);
final int idx = s.indexOf('.');
if (idx == -1) {
final int major;
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/net/Host.java b/httpcore5/src/main/java/org/apache/hc/core5/net/Host.java
index 3ef6192f1..68be9c74a 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/net/Host.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/net/Host.java
@@ -63,7 +63,7 @@ public final class Host implements NamedEndpoint, Serializable {
final boolean ipv6Brackets = !cursor.atEnd() && s.charAt(cursor.getPos()) == '[';
if (ipv6Brackets) {
cursor.updatePos(cursor.getPos() + 1);
- hostName = tokenizer.parseContent(s, cursor, URISupport.IPV6_HOST_TERMINATORS);
+ hostName = tokenizer.parseContent(s, cursor, URISupport.IPV6_HOST_DELIMITERS);
if (cursor.atEnd() || !(s.charAt(cursor.getPos()) == ']')) {
throw URISupport.createException(s, cursor, "Expected an IPv6 closing bracket ']'");
}
@@ -72,12 +72,12 @@ public final class Host implements NamedEndpoint, Serializable {
throw URISupport.createException(s, cursor, "Expected an IPv6 address");
}
} else {
- hostName = tokenizer.parseContent(s, cursor, URISupport.PORT_SEPARATORS);
+ hostName = tokenizer.parseContent(s, cursor, URISupport.PORT_DELIMITERS);
}
String portText = null;
if (!cursor.atEnd() && s.charAt(cursor.getPos()) == ':') {
cursor.updatePos(cursor.getPos() + 1);
- portText = tokenizer.parseContent(s, cursor, URISupport.TERMINATORS);
+ portText = tokenizer.parseContent(s, cursor, URISupport.DELIMITERS);
}
final int port;
if (!TextUtils.isBlank(portText)) {
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/net/URIAuthority.java b/httpcore5/src/main/java/org/apache/hc/core5/net/URIAuthority.java
index a9fed758c..3dd0ef0e7 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/net/URIAuthority.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/net/URIAuthority.java
@@ -54,7 +54,7 @@ public final class URIAuthority implements NamedEndpoint, Serializable {
final Tokenizer tokenizer = Tokenizer.INSTANCE;
String userInfo = null;
final int initPos = cursor.getPos();
- final String token = tokenizer.parseContent(s, cursor, URISupport.HOST_SEPARATORS);
+ final String token = tokenizer.parseContent(s, cursor, URISupport.HOST_DELIMITERS);
if (!cursor.atEnd() && s.charAt(cursor.getPos()) == '@') {
cursor.updatePos(cursor.getPos() + 1);
if (!TextUtils.isBlank(token)) {
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/net/URIBuilder.java b/httpcore5/src/main/java/org/apache/hc/core5/net/URIBuilder.java
index f6dbe067a..9fed7c089 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/net/URIBuilder.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/net/URIBuilder.java
@@ -34,7 +34,6 @@ import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.BitSet;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -196,16 +195,8 @@ public class URIBuilder {
private static final char PARAM_VALUE_SEPARATOR = '=';
private static final char PATH_SEPARATOR = '/';
- private static final BitSet QUERY_PARAM_SEPARATORS = new BitSet(256);
- private static final BitSet QUERY_VALUE_SEPARATORS = new BitSet(256);
- private static final BitSet PATH_SEPARATORS = new BitSet(256);
-
- static {
- QUERY_PARAM_SEPARATORS.set(QUERY_PARAM_SEPARATOR);
- QUERY_PARAM_SEPARATORS.set(PARAM_VALUE_SEPARATOR);
- QUERY_VALUE_SEPARATORS.set(QUERY_PARAM_SEPARATOR);
- PATH_SEPARATORS.set(PATH_SEPARATOR);
- }
+ private static final Tokenizer.Delimiter QUERY_PARAM_SEPARATORS = Tokenizer.delimiters(QUERY_PARAM_SEPARATOR, PARAM_VALUE_SEPARATOR);
+ private static final Tokenizer.Delimiter QUERY_VALUE_SEPARATORS = Tokenizer.delimiters(QUERY_PARAM_SEPARATOR);
static List<NameValuePair> parseQuery(final CharSequence s, final Charset charset, final boolean plusAsBlank) {
if (s == null) {
@@ -245,7 +236,7 @@ public class URIBuilder {
if (cursor.atEnd()) {
return new ArrayList<>(0);
}
- if (PATH_SEPARATORS.get(s.charAt(cursor.getPos()))) {
+ if (s.charAt(cursor.getPos()) == PATH_SEPARATOR) {
cursor.updatePos(cursor.getPos() + 1);
}
final List<String> list = new ArrayList<>();
@@ -256,7 +247,7 @@ public class URIBuilder {
break;
}
final char current = s.charAt(cursor.getPos());
- if (PATH_SEPARATORS.get(current)) {
+ if (current == PATH_SEPARATOR) {
list.add(buf.toString());
buf.setLength(0);
} else {
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/net/URISupport.java b/httpcore5/src/main/java/org/apache/hc/core5/net/URISupport.java
index e8847940f..350ca11f9 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/net/URISupport.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/net/URISupport.java
@@ -33,10 +33,9 @@ import org.apache.hc.core5.util.Tokenizer;
final class URISupport {
- static final BitSet HOST_SEPARATORS = new BitSet(256);
- static final BitSet IPV6_HOST_TERMINATORS = new BitSet(256);
- static final BitSet PORT_SEPARATORS = new BitSet(256);
- static final BitSet TERMINATORS = new BitSet(256);
+ private static final BitSet HOST_SEPARATORS = new BitSet(256);
+ private static final BitSet PORT_SEPARATORS = new BitSet(256);
+ private static final BitSet TERMINATORS = new BitSet(256);
static {
TERMINATORS.set('/');
@@ -44,11 +43,15 @@ final class URISupport {
TERMINATORS.set('?');
HOST_SEPARATORS.or(TERMINATORS);
HOST_SEPARATORS.set('@');
- IPV6_HOST_TERMINATORS.set(']');
PORT_SEPARATORS.or(TERMINATORS);
PORT_SEPARATORS.set(':');
}
+ static final Tokenizer.Delimiter DELIMITERS = Tokenizer.delimiters(TERMINATORS);
+ static final Tokenizer.Delimiter HOST_DELIMITERS = Tokenizer.delimiters(HOST_SEPARATORS);
+ static final Tokenizer.Delimiter IPV6_HOST_DELIMITERS = Tokenizer.delimiters(']');
+ static final Tokenizer.Delimiter PORT_DELIMITERS = Tokenizer.delimiters(PORT_SEPARATORS);
+
static URISyntaxException createException(
final CharSequence input, final Tokenizer.Cursor cursor, final String reason) {
return new URISyntaxException(
diff --git a/httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java b/httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java
index 5c9b861ab..13acb4c6f 100644
--- a/httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java
+++ b/httpcore5/src/main/java/org/apache/hc/core5/util/Tokenizer.java
@@ -119,6 +119,60 @@ public class Tokenizer {
return ch == SP || ch == HT || ch == CR || ch == LF;
}
+ /**
+ * Represents a predicate whether the given character is a delimiter.
+ *
+ * @since 5.4
+ */
+ @FunctionalInterface
+ public interface Delimiter {
+
+ boolean test(char ch);
+
+ }
+
+ /**
+ * @since 5.4
+ */
+ public static Delimiter delimiters(final BitSet delimiters) {
+ return delimiters::get;
+ }
+
+ /**
+ * @since 5.4
+ */
+ public static Delimiter delimiters(final char... delimiters) {
+ return ch -> {
+ for (final char delimiter : delimiters) {
+ if (delimiter == ch) {
+ return true;
+ }
+ }
+ return false;
+ };
+ }
+
+ /**
+ * @since 5.4
+ */
+ public static Delimiter delimiters(final char delimiter) {
+ return ch -> ch == delimiter;
+ }
+
+ /**
+ * @since 5.4
+ */
+ public static Delimiter delimiters(final char delimiter1, final char delimiter2) {
+ return ch -> ch == delimiter1 || ch == delimiter2;
+ }
+
+ /**
+ * @since 5.4
+ */
+ public static Delimiter delimiters(final char delimiter1, final char delimiter2, final char delimiter3) {
+ return ch -> ch == delimiter1 || ch == delimiter2 || ch == delimiter3;
+ }
+
public static final Tokenizer INSTANCE = new Tokenizer();
/**
@@ -127,34 +181,42 @@ public class Tokenizer {
*
* @param buf buffer with the sequence of chars to be parsed
* @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the token
+ * @param delimiterPredicate delimiter predicate. Can be {@code null} if the token
* is not delimited by any character.
*/
- public String parseContent(final CharSequence buf, final Cursor cursor, final BitSet delimiters) {
+ public String parseContent(final CharSequence buf, final Cursor cursor, final Delimiter delimiterPredicate) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
final StringBuilder dst = new StringBuilder();
- copyContent(buf, cursor, delimiters, dst);
+ copyContent(buf, cursor, delimiterPredicate, dst);
return dst.toString();
}
+ /**
+ * @deprecated use {@link #parseContent(CharSequence, Cursor, Delimiter)}
+ */
+ @Deprecated
+ public String parseContent(final CharSequence buf, final Cursor cursor, final BitSet bitSet) {
+ return parseContent(buf, cursor, bitSet != null ? bitSet::get : null);
+ }
+
/**
* Extracts from the sequence of chars a token terminated with any of the given delimiters
* discarding semantically insignificant whitespace characters.
*
* @param buf buffer with the sequence of chars to be parsed
* @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the token
+ * @param delimiterPredicate delimiter predicate. Can be {@code null} if the token
* is not delimited by any character.
*/
- public String parseToken(final CharSequence buf, final Cursor cursor, final BitSet delimiters) {
+ public String parseToken(final CharSequence buf, final Cursor cursor, final Delimiter delimiterPredicate) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
final StringBuilder dst = new StringBuilder();
boolean whitespace = false;
while (!cursor.atEnd()) {
final char current = buf.charAt(cursor.getPos());
- if (delimiters != null && delimiters.get(current)) {
+ if (delimiterPredicate != null && delimiterPredicate.test(current)) {
break;
} else if (isWhitespace(current)) {
skipWhiteSpace(buf, cursor);
@@ -163,13 +225,21 @@ public class Tokenizer {
if (whitespace && dst.length() > 0) {
dst.append(' ');
}
- copyContent(buf, cursor, delimiters, dst);
+ copyContent(buf, cursor, delimiterPredicate, dst);
whitespace = false;
}
}
return dst.toString();
}
+ /**
+ * @deprecated use {@link #parseToken(CharSequence, Cursor, Delimiter)}
+ */
+ @Deprecated
+ public String parseToken(final CharSequence buf, final Cursor cursor, final BitSet bitSet) {
+ return parseToken(buf, cursor, bitSet != null ? bitSet::get : null);
+ }
+
/**
* Extracts from the sequence of chars a value which can be enclosed in quote marks and
* terminated with any of the given delimiters discarding semantically insignificant
@@ -177,17 +247,17 @@ public class Tokenizer {
*
* @param buf buffer with the sequence of chars to be parsed
* @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the value
+ * @param delimiterPredicate delimiter predicate. Can be {@code null} if the token
* is not delimited by any character.
*/
- public String parseValue(final CharSequence buf, final Cursor cursor, final BitSet delimiters) {
+ public String parseValue(final CharSequence buf, final Cursor cursor, final Delimiter delimiterPredicate) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
final StringBuilder dst = new StringBuilder();
boolean whitespace = false;
while (!cursor.atEnd()) {
final char current = buf.charAt(cursor.getPos());
- if (delimiters != null && delimiters.get(current)) {
+ if (delimiterPredicate != null && delimiterPredicate.test(current)) {
break;
} else if (isWhitespace(current)) {
skipWhiteSpace(buf, cursor);
@@ -202,13 +272,21 @@ public class Tokenizer {
if (whitespace && dst.length() > 0) {
dst.append(' ');
}
- copyUnquotedContent(buf, cursor, delimiters, dst);
+ copyUnquotedContent(buf, cursor, delimiterPredicate, dst);
whitespace = false;
}
}
return dst.toString();
}
+ /**
+ * @deprecated use {@link #parseValue(CharSequence, Cursor, Delimiter)}
+ */
+ @Deprecated
+ public String parseValue(final CharSequence buf, final Cursor cursor, final BitSet bitSet) {
+ return parseValue(buf, cursor, bitSet != null ? bitSet::get : null);
+ }
+
/**
* Skips semantically insignificant whitespace characters and moves the cursor to the closest
* non-whitespace character.
@@ -238,11 +316,11 @@ public class Tokenizer {
*
* @param buf buffer with the sequence of chars to be parsed
* @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the value
+ * @param delimiterPredicate delimiter predicate. Can be {@code null} if the token
* is delimited by a whitespace only.
* @param dst destination buffer
*/
- public void copyContent(final CharSequence buf, final Cursor cursor, final BitSet delimiters,
+ public void copyContent(final CharSequence buf, final Cursor cursor, final Delimiter delimiterPredicate,
final StringBuilder dst) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
@@ -252,7 +330,7 @@ public class Tokenizer {
final int indexTo = cursor.getUpperBound();
for (int i = indexFrom; i < indexTo; i++) {
final char current = buf.charAt(i);
- if ((delimiters != null && delimiters.get(current)) || isWhitespace(current)) {
+ if ((delimiterPredicate != null && delimiterPredicate.test(current)) || isWhitespace(current)) {
break;
}
pos++;
@@ -261,18 +339,27 @@ public class Tokenizer {
cursor.updatePos(pos);
}
+ /**
+ * @deprecated Use {@link #copyContent(CharSequence, Cursor, Delimiter, StringBuilder)}
+ */
+ @Deprecated
+ public void copyContent(final CharSequence buf, final Cursor cursor, final BitSet bitSet,
+ final StringBuilder dst) {
+ copyContent(buf, cursor, bitSet != null ? bitSet::get : null, dst);
+ }
+
/**
* Transfers content into the destination buffer until a whitespace character, a quote,
* or any of the given delimiters is encountered.
*
* @param buf buffer with the sequence of chars to be parsed
* @param cursor defines the bounds and current position of the buffer
- * @param delimiters set of delimiting characters. Can be {@code null} if the value
+ * @param delimiterPredicate delimiter predicate. Can be {@code null} if the token
* is delimited by a whitespace or a quote only.
* @param dst destination buffer
*/
public void copyUnquotedContent(final CharSequence buf, final Cursor cursor,
- final BitSet delimiters, final StringBuilder dst) {
+ final Delimiter delimiterPredicate, final StringBuilder dst) {
Args.notNull(buf, "Char sequence");
Args.notNull(cursor, "Parser cursor");
Args.notNull(dst, "String builder");
@@ -281,7 +368,7 @@ public class Tokenizer {
final int indexTo = cursor.getUpperBound();
for (int i = indexFrom; i < indexTo; i++) {
final char current = buf.charAt(i);
- if ((delimiters != null && delimiters.get(current))
+ if ((delimiterPredicate != null && delimiterPredicate.test(current))
|| isWhitespace(current) || current == DQUOTE) {
break;
}
@@ -291,6 +378,15 @@ public class Tokenizer {
cursor.updatePos(pos);
}
+ /**
+ * @deprecated Use {@link #copyUnquotedContent(CharSequence, Cursor, Delimiter, StringBuilder)}
+ */
+ @Deprecated
+ public void copyUnquotedContent(final CharSequence buf, final Cursor cursor,
+ final BitSet bitSet, final StringBuilder dst) {
+ copyUnquotedContent(buf, cursor, bitSet != null ? bitSet::get : null, dst);
+ }
+
/**
* Transfers content enclosed with quote marks into the destination buffer.
*
diff --git a/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java b/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java
index 39d393bed..df9bf543e 100644
--- a/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java
+++ b/httpcore5/src/test/java/org/apache/hc/core5/http/ssl/TestTlsVersionParser.java
@@ -61,7 +61,7 @@ public class TestTlsVersionParser {
@Test
public void testParseBuffer() throws Exception {
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(1, 13);
- assertThat(impl.parse(" TLSv1.2,0000", cursor, Tokenizer.INIT_BITSET(',')),
+ assertThat(impl.parse(" TLSv1.2,0000", cursor, Tokenizer.delimiters(',')),
CoreMatchers.equalTo(TLS.V_1_2.getVersion()));
assertThat(cursor.getPos(), CoreMatchers.equalTo(8));
}
diff --git a/httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java b/httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java
index ce0dfd506..b56a78506 100644
--- a/httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java
+++ b/httpcore5/src/test/java/org/apache/hc/core5/util/TestTokenizer.java
@@ -61,7 +61,7 @@ public class TestTokenizer {
Assertions.assertEquals(3, cursor.getPos());
final StringBuilder strbuf1 = new StringBuilder();
- parser.copyContent(raw, cursor, Tokenizer.INIT_BITSET(':'), strbuf1);
+ parser.copyContent(raw, cursor, Tokenizer.delimiters(':'), strbuf1);
Assertions.assertFalse(cursor.atEnd());
Assertions.assertEquals(6, cursor.getPos());
@@ -99,7 +99,7 @@ public class TestTokenizer {
Assertions.assertEquals(0, cursor.getPos());
final StringBuilder strbuf1 = new StringBuilder();
- parser.copyContent(raw, cursor, Tokenizer.INIT_BITSET(':'), strbuf1);
+ parser.copyContent(raw, cursor, Tokenizer.delimiters(':'), strbuf1);
Assertions.assertFalse(cursor.atEnd());
Assertions.assertEquals("raw", strbuf1.toString());
@@ -132,7 +132,7 @@ public class TestTokenizer {
final String s = " stuff and \tsome\tmore stuff ;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseToken(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseToken(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuff and some more stuff", result);
}
@@ -141,7 +141,7 @@ public class TestTokenizer {
final String s = " stuff and \" some more \" \"stuff ;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseValue(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuff and some more stuff ;", result);
}
@@ -150,7 +150,7 @@ public class TestTokenizer {
final String s = "stuff\"more\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseValue(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuffmorestuff", result);
}
@@ -159,7 +159,7 @@ public class TestTokenizer {
final String s = "stuff\"\\\"more\\\"\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseValue(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuff\"more\"stuff", result);
}
@@ -168,7 +168,7 @@ public class TestTokenizer {
final String s = "stuff\"\\\"more\\\";\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseValue(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuff\"more\";stuff", result);
}
@@ -177,7 +177,7 @@ public class TestTokenizer {
final String s = "stuff\"\\\"more\\\";\\\\\"stuff;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseValue(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuff\"more\";\\stuff", result);
}
@@ -186,7 +186,7 @@ public class TestTokenizer {
final String s = "stuff\\; more stuff;";
final CharArrayBuffer raw = createBuffer(s);
final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
- final String result = parser.parseValue(raw, cursor, Tokenizer.INIT_BITSET(';'));
+ final String result = parser.parseValue(raw, cursor, Tokenizer.delimiters(';'));
Assertions.assertEquals("stuff\\", result);
}
}