You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/07/26 13:11:10 UTC

[lucene] 02/02: Clean up writing String to ByteBuffersDataOutput (#12455)

This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 434b4804ebd38d8dac6a284ae0563482aaa94580
Author: Armin Braun <me...@obrown.io>
AuthorDate: Wed Jul 26 14:25:01 2023 +0200

    Clean up writing String to ByteBuffersDataOutput (#12455)
    
    Resolving TODO to use UnicodeUtil instead of a copy of its code here.
    Maybe slightly slower from the extra check for high-surrogate but that
    may be outweigh or better by more compact code and saving the capturing lambda
    that might not inline.
---
 .../apache/lucene/store/ByteBuffersDataOutput.java | 89 +++++-----------------
 1 file changed, 18 insertions(+), 71 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java
index 756a531212d..6f6212fc20e 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java
@@ -28,7 +28,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Consumer;
-import java.util.function.IntConsumer;
 import java.util.function.IntFunction;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BitUtil;
@@ -410,25 +409,17 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountab
     }
   }
 
+  private static final int MAX_CHARS_PER_WINDOW = 1024;
+
   @Override
   public void writeString(String v) {
     try {
-      final int MAX_CHARS_PER_WINDOW = 1024;
       if (v.length() <= MAX_CHARS_PER_WINDOW) {
         final BytesRef utf8 = new BytesRef(v);
         writeVInt(utf8.length);
         writeBytes(utf8.bytes, utf8.offset, utf8.length);
       } else {
-        writeVInt(UnicodeUtil.calcUTF16toUTF8Length(v, 0, v.length()));
-        final byte[] buf = new byte[UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR * MAX_CHARS_PER_WINDOW];
-        UTF16toUTF8(
-            v,
-            0,
-            v.length(),
-            buf,
-            (len) -> {
-              writeBytes(buf, 0, len);
-            });
+        writeLongString(v);
       }
     } catch (IOException e) {
       throw new UncheckedIOException(e);
@@ -549,66 +540,22 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountab
     return blockBits;
   }
 
-  // TODO: move this block-based conversion to UnicodeUtil.
-
-  private static final long HALF_SHIFT = 10;
-  private static final int SURROGATE_OFFSET =
-      Character.MIN_SUPPLEMENTARY_CODE_POINT
-          - (UnicodeUtil.UNI_SUR_HIGH_START << HALF_SHIFT)
-          - UnicodeUtil.UNI_SUR_LOW_START;
-
-  /** A consumer-based UTF16-UTF8 encoder (writes the input string in smaller buffers.). */
-  private static int UTF16toUTF8(
-      final CharSequence s,
-      final int offset,
-      final int length,
-      byte[] buf,
-      IntConsumer bufferFlusher) {
-    int utf8Len = 0;
-    int j = 0;
-    for (int i = offset, end = offset + length; i < end; i++) {
-      final int chr = (int) s.charAt(i);
-
-      if (j + 4 >= buf.length) {
-        bufferFlusher.accept(j);
-        utf8Len += j;
-        j = 0;
-      }
-
-      if (chr < 0x80) buf[j++] = (byte) chr;
-      else if (chr < 0x800) {
-        buf[j++] = (byte) (0xC0 | (chr >> 6));
-        buf[j++] = (byte) (0x80 | (chr & 0x3F));
-      } else if (chr < 0xD800 || chr > 0xDFFF) {
-        buf[j++] = (byte) (0xE0 | (chr >> 12));
-        buf[j++] = (byte) (0x80 | ((chr >> 6) & 0x3F));
-        buf[j++] = (byte) (0x80 | (chr & 0x3F));
-      } else {
-        // A surrogate pair. Confirm valid high surrogate.
-        if (chr < 0xDC00 && (i < end - 1)) {
-          int utf32 = (int) s.charAt(i + 1);
-          // Confirm valid low surrogate and write pair.
-          if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) {
-            utf32 = (chr << 10) + utf32 + SURROGATE_OFFSET;
-            i++;
-            buf[j++] = (byte) (0xF0 | (utf32 >> 18));
-            buf[j++] = (byte) (0x80 | ((utf32 >> 12) & 0x3F));
-            buf[j++] = (byte) (0x80 | ((utf32 >> 6) & 0x3F));
-            buf[j++] = (byte) (0x80 | (utf32 & 0x3F));
-            continue;
-          }
-        }
-        // Replace unpaired surrogate or out-of-order low surrogate
-        // with substitution character.
-        buf[j++] = (byte) 0xEF;
-        buf[j++] = (byte) 0xBF;
-        buf[j++] = (byte) 0xBD;
+  /** Writes a long string in chunks */
+  private void writeLongString(final String s) throws IOException {
+    final int byteLen = UnicodeUtil.calcUTF16toUTF8Length(s, 0, s.length());
+    writeVInt(byteLen);
+    final byte[] buf =
+        new byte[Math.min(byteLen, UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR * MAX_CHARS_PER_WINDOW)];
+    for (int i = 0, end = s.length(); i < end; ) {
+      // do one fewer chars than MAX_CHARS_PER_WINDOW in case we run into an unpaired surrogate
+      // below and need to increase the step to cover the lower surrogate as well
+      int step = Math.min(end - i, MAX_CHARS_PER_WINDOW - 1);
+      if (i + step < end && Character.isHighSurrogate(s.charAt(i + step - 1))) {
+        step++;
       }
+      int upTo = UnicodeUtil.UTF16toUTF8(s, i, step, buf);
+      writeBytes(buf, 0, upTo);
+      i += step;
     }
-
-    bufferFlusher.accept(j);
-    utf8Len += j;
-
-    return utf8Len;
   }
 }