You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2024/01/12 15:25:15 UTC
(lucene) 01/02: Output binary doc values as hex array in SimpleTextCodec (#12987)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 6e1ce4cc6b7a5a06e35a01a2347de9ba765dee61
Author: Michael Froh <ms...@gmail.com>
AuthorDate: Fri Jan 12 15:09:18 2024 +0000
Output binary doc values as hex array in SimpleTextCodec (#12987)
Binary doc values were being written directly in SimpleTextCodec, though
they may not be valid UTF-8 (i.e. they may not be "text"). This change
encodes them as a string representing an array of hexadecimal bytes.
---
.../simpletext/SimpleTextDocValuesReader.java | 8 ++--
.../simpletext/SimpleTextDocValuesWriter.java | 15 ++++---
.../lucene/codecs/simpletext/SimpleTextUtil.java | 3 ++
.../simpletext/TestSimpleTextDocValuesFormat.java | 48 ++++++++++++++++++++++
4 files changed, 63 insertions(+), 11 deletions(-)
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
index 8f1c8d863d2..da7a40e949f 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
@@ -302,6 +302,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
IntFunction<BytesRef> values =
new IntFunction<BytesRef>() {
final BytesRefBuilder term = new BytesRefBuilder();
+ final BytesRefBuilder termByteArray = new BytesRefBuilder();
@Override
public BytesRef apply(int docID) {
@@ -329,9 +330,10 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
} catch (ParseException pe) {
throw new CorruptIndexException("failed to parse int length", in, pe);
}
- term.grow(len);
- term.setLength(len);
- in.readBytes(term.bytes(), 0, len);
+ termByteArray.grow(len);
+ termByteArray.setLength(len);
+ in.readBytes(termByteArray.bytes(), 0, len);
+ term.copyBytes(SimpleTextUtil.fromBytesRefString(termByteArray.get().utf8ToString()));
return term.get();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
index bd0c307acc6..a49041d2730 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
@@ -169,7 +169,7 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
int maxLength = 0;
BinaryDocValues values = valuesProducer.getBinary(field);
for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
- maxLength = Math.max(maxLength, values.binaryValue().length);
+ maxLength = Math.max(maxLength, values.binaryValue().toString().length());
}
writeFieldEntry(field, DocValuesType.BINARY);
@@ -197,17 +197,16 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
values.nextDoc();
assert values.docID() >= i;
}
+ String stringVal = values.docID() == i ? values.binaryValue().toString() : null;
// write length
- final int length = values.docID() != i ? 0 : values.binaryValue().length;
+ final int length = stringVal == null ? 0 : stringVal.length();
SimpleTextUtil.write(data, LENGTH);
SimpleTextUtil.write(data, encoder.format(length), scratch);
SimpleTextUtil.writeNewline(data);
- // write bytes -- don't use SimpleText.write
- // because it escapes:
- if (values.docID() == i) {
- BytesRef value = values.binaryValue();
- data.writeBytes(value.bytes, value.offset, value.length);
+ // write bytes as hex array
+ if (stringVal != null) {
+ SimpleTextUtil.write(data, stringVal, scratch);
}
// pad to fit
@@ -215,7 +214,7 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
data.writeByte((byte) ' ');
}
SimpleTextUtil.writeNewline(data);
- if (values.docID() != i) {
+ if (stringVal == null) {
SimpleTextUtil.write(data, "F", scratch);
} else {
SimpleTextUtil.write(data, "T", scratch);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java
index 011e7560076..a18bb3f3727 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java
@@ -111,6 +111,9 @@ class SimpleTextUtil {
throw new IllegalArgumentException(
"string " + s + " was not created from BytesRef.toString?");
}
+ if (s.length() == 2) {
+ return new BytesRef(BytesRef.EMPTY_BYTES);
+ }
String[] parts = s.substring(1, s.length() - 1).split(" ");
byte[] bytes = new byte[parts.length];
for (int i = 0; i < parts.length; i++) {
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java b/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java
index ed13e2bbad8..12dc2728d5a 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java
@@ -16,8 +16,23 @@
*/
package org.apache.lucene.codecs.simpletext;
+import java.io.IOException;
+import java.util.List;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
/** Tests SimpleTextDocValuesFormat */
public class TestSimpleTextDocValuesFormat extends BaseDocValuesFormatTestCase {
@@ -27,4 +42,37 @@ public class TestSimpleTextDocValuesFormat extends BaseDocValuesFormatTestCase {
protected Codec getCodec() {
return codec;
}
+
+ public void testFileIsUTF8() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf)) {
+ for (int i = 0; i < 100; i++) {
+ writer.addDocument(
+ List.of(
+ new SortedDocValuesField(
+ "sortedVal", newBytesRef(TestUtil.randomSimpleString(random()))),
+ new SortedSetDocValuesField(
+ "sortedSetVal", newBytesRef(TestUtil.randomSimpleString(random()))),
+ new NumericDocValuesField("numberVal", random().nextLong()),
+ new BinaryDocValuesField("binaryVal", TestUtil.randomBinaryTerm(random()))));
+ }
+ }
+ for (String file : dir.listAll()) {
+ if (file.endsWith("dat")) {
+ try (IndexInput input = dir.openChecksumInput(file, IOContext.READONCE)) {
+ long length = input.length();
+ if (length > 20_000) {
+ // Avoid allocating a huge array if the length is wrong
+ fail("Doc values should not be this large");
+ }
+ byte[] bytes = new byte[(int) length];
+ input.readBytes(bytes, 0, bytes.length);
+ BytesRef bytesRef = new BytesRef(bytes);
+ assertNotEquals(bytesRef.toString(), Term.toString(bytesRef));
+ }
+ }
+ }
+ }
+ }
}