You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/02/01 09:36:39 UTC
[lucene-solr] branch master updated: LUCENE-9708: Hunspell: support
FLAG UTF-8 in absence of SET UTF-8 (#2270)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 7a7949a LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)
7a7949a is described below
commit 7a7949aed2a1db99c62239e9d8f6d07e0da8a732
Author: Peter Gromov <pe...@jetbrains.com>
AuthorDate: Mon Feb 1 10:36:24 2021 +0100
LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)
---
.../lucene/analysis/hunspell/Dictionary.java | 26 ++++++++++++---
.../lucene/analysis/hunspell/TestDictionary.java | 38 +++++++++++++++-------
2 files changed, 48 insertions(+), 16 deletions(-)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index 9de67eb..58b6f22 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -83,7 +83,7 @@ public class Dictionary {
// TODO: really for suffixes we should reverse the automaton and run them backwards
private static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*";
private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s";
- static final String DEFAULT_ENCODING = StandardCharsets.ISO_8859_1.name();
+ static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
FST<IntsRef> prefixes;
FST<IntsRef> suffixes;
@@ -347,7 +347,7 @@ public class Dictionary {
} else if ("FLAG".equals(firstWord)) {
// Assume that the FLAG line comes before any prefix or suffixes
// Store the strategy so it can be used when parsing the dic file
- flagParsingStrategy = getFlagParsingStrategy(line);
+ flagParsingStrategy = getFlagParsingStrategy(line, decoder.charset());
} else if (line.equals("COMPLEXPREFIXES")) {
complexPrefixes =
true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix
@@ -725,7 +725,7 @@ public class Dictionary {
// this test only at the end as ineffective but would allow lines only containing spaces:
encoding.toString().trim().length() == 0) {
if (ch < 0) {
- return DEFAULT_ENCODING;
+ return DEFAULT_CHARSET.name();
}
continue;
}
@@ -734,7 +734,7 @@ public class Dictionary {
int last = matcher.end();
return encoding.substring(last).trim();
}
- return DEFAULT_ENCODING;
+ return DEFAULT_CHARSET.name();
}
}
@@ -768,7 +768,7 @@ public class Dictionary {
* @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG
* definition
*/
- static FlagParsingStrategy getFlagParsingStrategy(String flagLine) {
+ static FlagParsingStrategy getFlagParsingStrategy(String flagLine, Charset charset) {
String[] parts = flagLine.split("\\s+");
if (parts.length != 2) {
throw new IllegalArgumentException("Illegal FLAG specification: " + flagLine);
@@ -778,6 +778,9 @@ public class Dictionary {
if ("num".equals(flagType)) {
return new NumFlagParsingStrategy();
} else if ("UTF-8".equals(flagType)) {
+ if (DEFAULT_CHARSET.equals(charset)) {
+ return new DefaultAsUtf8FlagParsingStrategy();
+ }
return new SimpleFlagParsingStrategy();
} else if ("long".equals(flagType)) {
return new DoubleASCIIFlagParsingStrategy();
@@ -1271,6 +1274,19 @@ public class Dictionary {
}
}
+ /** Used to read flags as UTF-8 even if the rest of the file is in the default (8-bit) encoding */
+ private static class DefaultAsUtf8FlagParsingStrategy extends FlagParsingStrategy {
+ @Override
+ public char[] parseFlags(String rawFlags) {
+ return new String(rawFlags.getBytes(DEFAULT_CHARSET), StandardCharsets.UTF_8).toCharArray();
+ }
+
+ @Override
+ void appendFlag(char flag, StringBuilder to) {
+ to.append(new String(String.valueOf(flag).getBytes(StandardCharsets.UTF_8), DEFAULT_CHARSET));
+ }
+ }
+
/**
* Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded in its
* numerical form. In the case of multiple flags, each number is separated by a comma.
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index a0ece78..faee6f0 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.Random;
@@ -261,13 +262,24 @@ public class TestDictionary extends LuceneTestCase {
Dictionary.getDictionaryEncoding(
new ByteArrayInputStream("\uFEFFSET\tUTF-8\r\n".getBytes(StandardCharsets.UTF_8))));
assertEquals(
- Dictionary.DEFAULT_ENCODING,
+ Dictionary.DEFAULT_CHARSET.name(),
Dictionary.getDictionaryEncoding(new ByteArrayInputStream(new byte[0])));
}
public void testFlagWithCrazyWhitespace() {
- assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));
- assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8"));
+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", StandardCharsets.UTF_8));
+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8", StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testUtf8Flag() {
+ Dictionary.FlagParsingStrategy strategy =
+ Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", Dictionary.DEFAULT_CHARSET);
+
+ String src = "привет";
+ String asAscii = new String(src.getBytes(StandardCharsets.UTF_8), Dictionary.DEFAULT_CHARSET);
+ assertNotEquals(src, asAscii);
+ assertEquals(src, new String(strategy.parseFlags(asAscii)));
}
@Test
@@ -275,19 +287,23 @@ public class TestDictionary extends LuceneTestCase {
Random r = random();
char[] flags = new char[r.nextInt(10)];
for (int i = 0; i < flags.length; i++) {
- flags[i] = (char) r.nextInt(Character.MAX_VALUE);
+ flags[i] = (char) r.nextInt(Character.MIN_SURROGATE);
}
String[] flagLines = {"FLAG long", "FLAG UTF-8", "FLAG num"};
+ Charset[] charsets = {StandardCharsets.UTF_8, Dictionary.DEFAULT_CHARSET};
for (String flagLine : flagLines) {
- Dictionary.FlagParsingStrategy strategy = Dictionary.getFlagParsingStrategy(flagLine);
- StringBuilder serialized = new StringBuilder();
- for (char flag : flags) {
- strategy.appendFlag(flag, serialized);
+ for (Charset charset : charsets) {
+ Dictionary.FlagParsingStrategy strategy =
+ Dictionary.getFlagParsingStrategy(flagLine, charset);
+ StringBuilder serialized = new StringBuilder();
+ for (char flag : flags) {
+ strategy.appendFlag(flag, serialized);
+ }
+
+ char[] deserialized = strategy.parseFlags(serialized.toString());
+ assertEquals(new String(flags), new String(deserialized));
}
-
- char[] deserialized = strategy.parseFlags(serialized.toString());
- assertEquals(new String(flags), new String(deserialized));
}
}