You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/02/01 09:36:39 UTC

[lucene-solr] branch master updated: LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)

This is an automated email from the ASF dual-hosted git repository.

dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/master by this push:
     new 7a7949a  LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)
7a7949a is described below

commit 7a7949aed2a1db99c62239e9d8f6d07e0da8a732
Author: Peter Gromov <pe...@jetbrains.com>
AuthorDate: Mon Feb 1 10:36:24 2021 +0100

    LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)
---
 .../lucene/analysis/hunspell/Dictionary.java       | 26 ++++++++++++---
 .../lucene/analysis/hunspell/TestDictionary.java   | 38 +++++++++++++++-------
 2 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index 9de67eb..58b6f22 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -83,7 +83,7 @@ public class Dictionary {
   // TODO: really for suffixes we should reverse the automaton and run them backwards
   private static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*";
   private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s";
-  static final String DEFAULT_ENCODING = StandardCharsets.ISO_8859_1.name();
+  static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
 
   FST<IntsRef> prefixes;
   FST<IntsRef> suffixes;
@@ -347,7 +347,7 @@ public class Dictionary {
       } else if ("FLAG".equals(firstWord)) {
         // Assume that the FLAG line comes before any prefix or suffixes
         // Store the strategy so it can be used when parsing the dic file
-        flagParsingStrategy = getFlagParsingStrategy(line);
+        flagParsingStrategy = getFlagParsingStrategy(line, decoder.charset());
       } else if (line.equals("COMPLEXPREFIXES")) {
         complexPrefixes =
             true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix
@@ -725,7 +725,7 @@ public class Dictionary {
           // this test only at the end as ineffective but would allow lines only containing spaces:
           encoding.toString().trim().length() == 0) {
         if (ch < 0) {
-          return DEFAULT_ENCODING;
+          return DEFAULT_CHARSET.name();
         }
         continue;
       }
@@ -734,7 +734,7 @@ public class Dictionary {
         int last = matcher.end();
         return encoding.substring(last).trim();
       }
-      return DEFAULT_ENCODING;
+      return DEFAULT_CHARSET.name();
     }
   }
 
@@ -768,7 +768,7 @@ public class Dictionary {
    * @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG
    *     definition
    */
-  static FlagParsingStrategy getFlagParsingStrategy(String flagLine) {
+  static FlagParsingStrategy getFlagParsingStrategy(String flagLine, Charset charset) {
     String[] parts = flagLine.split("\\s+");
     if (parts.length != 2) {
       throw new IllegalArgumentException("Illegal FLAG specification: " + flagLine);
@@ -778,6 +778,9 @@ public class Dictionary {
     if ("num".equals(flagType)) {
       return new NumFlagParsingStrategy();
     } else if ("UTF-8".equals(flagType)) {
+      if (DEFAULT_CHARSET.equals(charset)) {
+        return new DefaultAsUtf8FlagParsingStrategy();
+      }
       return new SimpleFlagParsingStrategy();
     } else if ("long".equals(flagType)) {
       return new DoubleASCIIFlagParsingStrategy();
@@ -1271,6 +1274,19 @@ public class Dictionary {
     }
   }
 
+  /** Used to read flags as UTF-8 even if the rest of the file is in the default (8-bit) encoding */
+  private static class DefaultAsUtf8FlagParsingStrategy extends FlagParsingStrategy {
+    @Override
+    public char[] parseFlags(String rawFlags) {
+      return new String(rawFlags.getBytes(DEFAULT_CHARSET), StandardCharsets.UTF_8).toCharArray();
+    }
+
+    @Override
+    void appendFlag(char flag, StringBuilder to) {
+      to.append(new String(String.valueOf(flag).getBytes(StandardCharsets.UTF_8), DEFAULT_CHARSET));
+    }
+  }
+
   /**
    * Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded in its
    * numerical form. In the case of multiple flags, each number is separated by a comma.
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index a0ece78..faee6f0 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.util.Random;
@@ -261,13 +262,24 @@ public class TestDictionary extends LuceneTestCase {
         Dictionary.getDictionaryEncoding(
             new ByteArrayInputStream("\uFEFFSET\tUTF-8\r\n".getBytes(StandardCharsets.UTF_8))));
     assertEquals(
-        Dictionary.DEFAULT_ENCODING,
+        Dictionary.DEFAULT_CHARSET.name(),
         Dictionary.getDictionaryEncoding(new ByteArrayInputStream(new byte[0])));
   }
 
   public void testFlagWithCrazyWhitespace() {
-    assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));
-    assertNotNull(Dictionary.getFlagParsingStrategy("FLAG    UTF-8"));
+    assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", StandardCharsets.UTF_8));
+    assertNotNull(Dictionary.getFlagParsingStrategy("FLAG    UTF-8", StandardCharsets.UTF_8));
+  }
+
+  @Test
+  public void testUtf8Flag() {
+    Dictionary.FlagParsingStrategy strategy =
+        Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", Dictionary.DEFAULT_CHARSET);
+
+    String src = "привет";
+    String asAscii = new String(src.getBytes(StandardCharsets.UTF_8), Dictionary.DEFAULT_CHARSET);
+    assertNotEquals(src, asAscii);
+    assertEquals(src, new String(strategy.parseFlags(asAscii)));
   }
 
   @Test
@@ -275,19 +287,23 @@ public class TestDictionary extends LuceneTestCase {
     Random r = random();
     char[] flags = new char[r.nextInt(10)];
     for (int i = 0; i < flags.length; i++) {
-      flags[i] = (char) r.nextInt(Character.MAX_VALUE);
+      flags[i] = (char) r.nextInt(Character.MIN_SURROGATE);
     }
 
     String[] flagLines = {"FLAG long", "FLAG UTF-8", "FLAG num"};
+    Charset[] charsets = {StandardCharsets.UTF_8, Dictionary.DEFAULT_CHARSET};
     for (String flagLine : flagLines) {
-      Dictionary.FlagParsingStrategy strategy = Dictionary.getFlagParsingStrategy(flagLine);
-      StringBuilder serialized = new StringBuilder();
-      for (char flag : flags) {
-        strategy.appendFlag(flag, serialized);
+      for (Charset charset : charsets) {
+        Dictionary.FlagParsingStrategy strategy =
+            Dictionary.getFlagParsingStrategy(flagLine, charset);
+        StringBuilder serialized = new StringBuilder();
+        for (char flag : flags) {
+          strategy.appendFlag(flag, serialized);
+        }
+
+        char[] deserialized = strategy.parseFlags(serialized.toString());
+        assertEquals(new String(flags), new String(deserialized));
       }
-
-      char[] deserialized = strategy.parseFlags(serialized.toString());
-      assertEquals(new String(flags), new String(deserialized));
     }
   }