You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by at...@apache.org on 2023/07/06 20:21:43 UTC
[pinot] branch pdatabuffer_fst updated: WIP
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch pdatabuffer_fst
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/pdatabuffer_fst by this push:
new 4678be7a84 WIP
4678be7a84 is described below
commit 4678be7a842d38a375aed6f270063c3ab7b36105
Author: Atri Sharma <at...@gmail.com>
AuthorDate: Fri Jul 7 01:51:33 2023 +0530
WIP
---
.../creator/impl/text/NativeTextIndexCreator.java | 6 ++--
.../index/readers/text/NativeTextIndexReader.java | 2 +-
.../pinot/segment/local/utils/nativefst/FST.java | 29 ++++++++++++++----
.../local/utils/nativefst/ImmutableFST.java | 29 +++++++++++++-----
.../utils/nativefst/NativeFSTIndexCreator.java | 34 ++++++++++++++++++++--
.../utils/nativefst/NativeFSTIndexReader.java | 18 ++++++++++--
.../utils/nativefst/FSTRegexpWithWeirdTest.java | 3 +-
.../local/utils/nativefst/FSTSanityTest.java | 3 +-
.../nativefst/ImmutableFSTDeserializedTest.java | 2 +-
.../local/utils/nativefst/ImmutableFSTTest.java | 13 +++++----
.../local/utils/nativefst/SerializerTestBase.java | 6 ++--
11 files changed, 114 insertions(+), 31 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/NativeTextIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/NativeTextIndexCreator.java
index 11af1c14f2..1335d0370c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/NativeTextIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/NativeTextIndexCreator.java
@@ -161,7 +161,7 @@ public class NativeTextIndexCreator extends AbstractTextIndexCreator {
}
private void generateIndexFile()
- throws IOException {
+ throws IOException {
ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_LENGTH);
headerBuffer.putInt(FSTHeader.FST_MAGIC);
headerBuffer.putInt(VERSION);
@@ -172,8 +172,8 @@ public class NativeTextIndexCreator extends AbstractTextIndexCreator {
headerBuffer.position(0);
try (FileChannel indexFileChannel = new RandomAccessFile(_indexFile, "rw").getChannel();
- FileChannel invertedIndexFileChannel = new RandomAccessFile(_invertedIndexFile, "r").getChannel();
- FileChannel fstFileChannel = new RandomAccessFile(_fstIndexFile, "rw").getChannel()) {
+ FileChannel invertedIndexFileChannel = new RandomAccessFile(_invertedIndexFile, "r").getChannel();
+ FileChannel fstFileChannel = new RandomAccessFile(_fstIndexFile, "rw").getChannel()) {
indexFileChannel.write(headerBuffer);
fstFileChannel.transferTo(0, _fstDataSize, indexFileChannel);
invertedIndexFileChannel.transferTo(0, invertedIndexFileLength, indexFileChannel);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
index 3650e3531f..15b276a6b5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
@@ -89,7 +89,7 @@ public class NativeTextIndexReader implements TextIndexReader {
long fstDataEndOffset = fstDataStartOffset + fstDataLength;
ByteBuffer byteBuffer = _buffer.toDirectByteBuffer(fstDataStartOffset, fstDataLength);
try {
- _fst = FST.read(new ByteBufferInputStream(Collections.singletonList(byteBuffer)), ImmutableFST.class, true);
+ _fst = FST.read(new ByteBufferInputStream(Collections.singletonList(byteBuffer)), ImmutableFST.class, true, fstDataLength);
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FST.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FST.java
index 3471b6ed80..a90b88c1d7 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FST.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FST.java
@@ -63,12 +63,24 @@ public abstract class FST implements Iterable<ByteBuffer> {
return buf;
}
+ //HACK: atri
+ public static FST read(InputStream stream)
+ throws IOException {
+ return read(stream, false, new DirectMemoryManager(FST.class.getName()), 0);
+ }
+
/**
* Wrapper for the main read function
*/
- public static FST read(InputStream stream)
+ public static FST read(InputStream stream, final int fstDataSize)
throws IOException {
- return read(stream, false, new DirectMemoryManager(FST.class.getName()));
+ return read(stream, false, new DirectMemoryManager(FST.class.getName()), fstDataSize);
+ }
+
+ //Hack: Atri
+ public static FST read(InputStream stream, boolean hasOutputSymbols, PinotDataBufferMemoryManager memoryManager)
+ throws IOException {
+ return read(stream, hasOutputSymbols, memoryManager, 0);
}
/**
@@ -82,19 +94,24 @@ public abstract class FST implements Iterable<ByteBuffer> {
* If the input stream does not represent an automaton or is
* otherwise invalid.
*/
- public static FST read(InputStream stream, boolean hasOutputSymbols, PinotDataBufferMemoryManager memoryManager)
+ public static FST read(InputStream stream, boolean hasOutputSymbols, PinotDataBufferMemoryManager memoryManager, final int fstDataSize)
throws IOException {
FSTHeader header = FSTHeader.read(stream);
switch (header._version) {
case ImmutableFST.VERSION:
- return new ImmutableFST(stream, hasOutputSymbols, memoryManager);
+ return new ImmutableFST(stream, hasOutputSymbols, memoryManager, fstDataSize);
default:
throw new IOException(
String.format(Locale.ROOT, "Unsupported automaton version: 0x%02x", header._version & 0xFF));
}
}
+ public static <T extends FST> T read(InputStream stream, Class<? extends T> clazz, boolean hasOutputSymbolse)
+ throws IOException {
+ return read(stream, clazz, hasOutputSymbolse, 0);
+ }
+
/**
* A factory for reading a specific FST subclass, including proper casting.
*
@@ -109,9 +126,9 @@ public abstract class FST implements Iterable<ByteBuffer> {
* invalid or the class of the automaton read from the input stream
* is not assignable to <code>clazz</code>.
*/
- public static <T extends FST> T read(InputStream stream, Class<? extends T> clazz, boolean hasOutputSymbols)
+ public static <T extends FST> T read(InputStream stream, Class<? extends T> clazz, boolean hasOutputSymbols, final int fstDataSize)
throws IOException {
- FST fst = read(stream, hasOutputSymbols, new DirectMemoryManager(FST.class.getName()));
+ FST fst = read(stream, hasOutputSymbols, new DirectMemoryManager(FST.class.getName()), fstDataSize);
if (!clazz.isInstance(fst)) {
throw new IOException(
String.format(Locale.ROOT, "Expected FST type %s, but read an incompatible type %s.", clazz.getName(),
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFST.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFST.java
index 7b36d2e3d3..f69ce0c301 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFST.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFST.java
@@ -26,6 +26,7 @@ import java.util.EnumSet;
import java.util.Map;
import java.util.Set;
import org.apache.pinot.segment.local.realtime.impl.dictionary.OffHeapMutableBytesStore;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager;
@@ -139,7 +140,11 @@ public final class ImmutableFST extends FST {
* see the documentation of this class for more information on how this
* structure is organized.
*/
- public final OffHeapMutableBytesStore _mutableBytesStore;
+ //public final OffHeapMutableBytesStore _mutableBytesStore;
+
+ public final PinotDataBuffer _pinotDataBuffer;
+
+ public int _offset;
/**
* The length of the node header structure (if the automaton was compiled with
* <code>NUMBERS</code> option). Otherwise zero.
@@ -162,15 +167,18 @@ public final class ImmutableFST extends FST {
/**
* Read and wrap a binary automaton in FST version 5.
*/
- ImmutableFST(InputStream stream, boolean hasOutputSymbols, PinotDataBufferMemoryManager memoryManager)
+ ImmutableFST(InputStream stream, boolean hasOutputSymbols, PinotDataBufferMemoryManager memoryManager, final int fstDataSize)
throws IOException {
DataInputStream in = new DataInputStream(stream);
+ _offset = 0;
+
_filler = in.readByte();
_annotation = in.readByte();
final byte hgtl = in.readByte();
- _mutableBytesStore = new OffHeapMutableBytesStore(memoryManager, "ImmutableFST");
+ //_mutableBytesStore = new OffHeapMutableBytesStore(memoryManager, "ImmutableFST");
+ _pinotDataBuffer = memoryManager.allocate(fstDataSize, "ImmutableFST");
/*
* Determine if the automaton was compiled with NUMBERS. If so, modify
@@ -204,7 +212,9 @@ public final class ImmutableFST extends FST {
throws IOException {
byte[] buffer = new byte[PER_BUFFER_SIZE];
while ((in.read(buffer)) >= 0) {
- _mutableBytesStore.add(buffer);
+ _pinotDataBuffer.readFrom(_offset, buffer);
+ _offset = _offset + PER_BUFFER_SIZE;
+ //_mutableBytesStore.add(buffer);
}
}
@@ -348,7 +358,9 @@ public final class ImmutableFST extends FST {
int actualArcOffset = seek >= PER_BUFFER_SIZE ? seek / PER_BUFFER_SIZE : 0;
int bufferOffset = seek >= PER_BUFFER_SIZE ? seek - ((actualArcOffset) * PER_BUFFER_SIZE) : seek;
- byte[] inputData = _mutableBytesStore.get(actualArcOffset);
+ //byte[] inputData = _mutableBytesStore.get(actualArcOffset);
+ byte[] inputData = new byte[PER_BUFFER_SIZE];
+ _pinotDataBuffer.copyTo(actualArcOffset, inputData);
r = r << 8 | (inputData[bufferOffset] & 0xff);
}
@@ -382,12 +394,15 @@ public final class ImmutableFST extends FST {
int actualArcOffset = seek >= PER_BUFFER_SIZE ? seek / PER_BUFFER_SIZE : 0;
int bufferOffset = seek >= PER_BUFFER_SIZE ? seek - ((actualArcOffset) * PER_BUFFER_SIZE) : seek;
- byte[] retVal = _mutableBytesStore.get((actualArcOffset));
+ byte[] retVal = new byte[PER_BUFFER_SIZE];
+ //retVal = _mutableBytesStore.get(actualArcOffset);
+ _pinotDataBuffer.copyTo(actualArcOffset, retVal);
int target = bufferOffset + offset;
if (target >= PER_BUFFER_SIZE) {
- retVal = _mutableBytesStore.get(actualArcOffset + 1);
+ //retVal = _mutableBytesStore.get(actualArcOffset + 1);
+ _pinotDataBuffer.copyTo(actualArcOffset + 1, retVal);
target = target - PER_BUFFER_SIZE;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
index 1fb58b56bc..1efe4e576d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
@@ -21,6 +21,10 @@ package org.apache.pinot.segment.local.utils.nativefst;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
import org.apache.pinot.segment.local.utils.nativefst.builder.FSTBuilder;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.IndexCreationContext;
@@ -30,13 +34,24 @@ import org.slf4j.LoggerFactory;
public class NativeFSTIndexCreator implements FSTIndexCreator {
+ /*
+ * MAGIC HEADER (4 bytes)
+ * FST size (4 bytes)
+ */
+ public static final int HEADER_LENGTH = 8;
private static final Logger LOGGER = LoggerFactory.getLogger(NativeFSTIndexCreator.class);
+ private static final String FST_FILE_NAME = "nativefst.fst";
+
private final File _fstIndexFile;
private final FSTBuilder _fstBuilder;
private int _dictId;
+ private int _fstDataSize;
+
+ private final File _indexFile;
+
/**
* This index requires values of the column be added in sorted order. Sorted entries could be passed in through
* constructor or added through addSortedDictIds function. Index of the sorted entry should correspond to the
@@ -47,7 +62,8 @@ public class NativeFSTIndexCreator implements FSTIndexCreator {
* @param sortedEntries Sorted entries of the unique values of the column.
*/
public NativeFSTIndexCreator(File indexDir, String columnName, String[] sortedEntries) {
- _fstIndexFile = new File(indexDir, columnName + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION);
+ _fstIndexFile = new File(indexDir, columnName + FST_FILE_NAME);
+ _indexFile = new File(indexDir, columnName + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION);
_fstBuilder = new FSTBuilder();
_dictId = 0;
@@ -80,7 +96,7 @@ public class NativeFSTIndexCreator implements FSTIndexCreator {
LOGGER.info("Sealing FST index: " + _fstIndexFile.getAbsolutePath());
try (FileOutputStream fileOutputStream = new FileOutputStream(_fstIndexFile)) {
FST fst = _fstBuilder.complete();
- fst.save(fileOutputStream);
+ _fstDataSize = fst.save(fileOutputStream);
}
}
@@ -88,4 +104,18 @@ public class NativeFSTIndexCreator implements FSTIndexCreator {
public void close()
throws IOException {
}
+
+ private void generateIndexFile()
+ throws IOException {
+ ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_LENGTH);
+ headerBuffer.putInt(FSTHeader.FST_MAGIC);
+ headerBuffer.putInt(_fstDataSize);
+ headerBuffer.position(0);
+
+ try (FileChannel indexFileChannel = new RandomAccessFile(_indexFile, "rw").getChannel();
+ FileChannel fstFileChannel = new RandomAccessFile(_fstIndexFile, "rw").getChannel()) {
+ indexFileChannel.write(headerBuffer);
+ fstFileChannel.transferTo(0, _fstDataSize, indexFileChannel);
+ }
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexReader.java
index 0a03389336..468ea9c5d9 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexReader.java
@@ -21,7 +21,10 @@ package org.apache.pinot.segment.local.utils.nativefst;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
+
+import com.google.common.base.Preconditions;
import org.apache.avro.util.ByteBufferInputStream;
+import org.apache.pinot.segment.local.segment.creator.impl.text.NativeTextIndexCreator;
import org.apache.pinot.segment.local.utils.nativefst.utils.RegexpMatcher;
import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
@@ -48,8 +51,19 @@ public class NativeFSTIndexReader implements TextIndexReader {
public NativeFSTIndexReader(PinotDataBuffer dataBuffer)
throws IOException {
// TODO: Implement an InputStream directly on PinotDataBuffer
- ByteBuffer byteBuffer = dataBuffer.toDirectByteBuffer(0, (int) dataBuffer.size());
- _fst = FST.read(new ByteBufferInputStream(Collections.singletonList(byteBuffer)), ImmutableFST.class, true);
+ /*ByteBuffer byteBuffer = dataBuffer.toDirectByteBuffer(0, (int) dataBuffer.size());
+ _fst = FST.read(new ByteBufferInputStream(Collections.singletonList(byteBuffer)), ImmutableFST.class, true);*/
+ int fstMagic = dataBuffer.getInt(0);
+ Preconditions.checkState(fstMagic == FSTHeader.FST_MAGIC, "Invalid native text index magic header: %s", fstMagic);
+ int fstDataLength = dataBuffer.getInt(4);
+
+ long fstDataStartOffset = NativeFSTIndexCreator.HEADER_LENGTH;
+ ByteBuffer byteBuffer = dataBuffer.toDirectByteBuffer(fstDataStartOffset, fstDataLength);
+ try {
+ _fst = FST.read(new ByteBufferInputStream(Collections.singletonList(byteBuffer)), ImmutableFST.class, true, fstDataLength);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
@Override
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTRegexpWithWeirdTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTRegexpWithWeirdTest.java
index c27c8e1050..e471e8419b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTRegexpWithWeirdTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTRegexpWithWeirdTest.java
@@ -66,7 +66,8 @@ public class FSTRegexpWithWeirdTest {
FST fst = fstBuilder.complete();
byte[] fstData = new FSTSerializerImpl().withNumbers().serialize(fst, new ByteArrayOutputStream()).toByteArray();
- _fst = FST.read(new ByteArrayInputStream(fstData), ImmutableFST.class, true);
+ //TODO: atri
+ _fst = FST.read(new ByteArrayInputStream(fstData), ImmutableFST.class, true, 0);
}
@Test
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTSanityTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTSanityTest.java
index 96e7404320..22e7de26a3 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTSanityTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTSanityTest.java
@@ -60,7 +60,8 @@ public class FSTSanityTest {
FST fst = FSTBuilder.buildFST(input);
byte[] fstData = new FSTSerializerImpl().withNumbers().serialize(fst, new ByteArrayOutputStream()).toByteArray();
- _nativeFST = FST.read(new ByteArrayInputStream(fstData), ImmutableFST.class, true);
+ //TODO: Atri
+ _nativeFST = FST.read(new ByteArrayInputStream(fstData), ImmutableFST.class, true, 0);
_fst = org.apache.pinot.segment.local.utils.fst.FSTBuilder.buildFST(input);
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTDeserializedTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTDeserializedTest.java
index a5df1f860e..64758e7b0b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTDeserializedTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTDeserializedTest.java
@@ -37,7 +37,7 @@ public class ImmutableFSTDeserializedTest {
public void setUp()
throws Exception {
try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("data/serfst.txt")) {
- _fst = FST.read(inputStream, true, new DirectMemoryManager(ImmutableFSTDeserializedTest.class.getName()));
+ _fst = FST.read(inputStream, true, new DirectMemoryManager(ImmutableFSTDeserializedTest.class.getName()), 0);
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTTest.java
index 78d4d161e5..cedff25e19 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTTest.java
@@ -78,7 +78,7 @@ public final class ImmutableFSTTest {
public void testVersion5()
throws IOException {
try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("data/abc.native.fst")) {
- FST fst = FST.read(inputStream);
+ FST fst = FST.read(inputStream, 0);
assertFalse(fst.getFlags().contains(FSTFlags.NUMBERS));
verifyContent(fst, _expected);
}
@@ -88,7 +88,7 @@ public final class ImmutableFSTTest {
public void testVersion5WithNumbers()
throws IOException {
try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("data/abc-numbers.native.fst")) {
- FST fst = FST.read(inputStream);
+ FST fst = FST.read(inputStream, 0);
assertTrue(fst.getFlags().contains(FSTFlags.NUMBERS));
verifyContent(fst, _expected);
}
@@ -99,7 +99,8 @@ public final class ImmutableFSTTest {
throws IOException {
for (String resourceName : new String[]{"data/abc.native.fst", "data/abc-numbers.native.fst"}) {
try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream(resourceName)) {
- FST fst = FST.read(inputStream);
+ //TODO: atri
+ FST fst = FST.read(inputStream, 0);
FSTInfo fstInfo = new FSTInfo(fst);
assertEquals(fstInfo._nodeCount, 4);
assertEquals(fstInfo._arcsCount, 7);
@@ -111,7 +112,8 @@ public final class ImmutableFSTTest {
public void testNumbers()
throws IOException {
try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("data/abc-numbers.native.fst")) {
- FST fst = FST.read(inputStream);
+ //TODO: atri
+ FST fst = FST.read(inputStream, 0);
assertTrue(fst.getFlags().contains(FSTFlags.NEXTBIT));
// Get all numbers for nodes.
@@ -139,7 +141,8 @@ public final class ImmutableFSTTest {
fst.save(new FileOutputStream(fstFile));
try (FileInputStream inputStream = new FileInputStream(fstFile)) {
- verifyContent(FST.read(inputStream, ImmutableFST.class, true), inputList);
+ //TODO: Atri
+ verifyContent(FST.read(inputStream, ImmutableFST.class, true, 0), inputList);
}
FileUtils.deleteQuietly(fstFile);
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/SerializerTestBase.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/SerializerTestBase.java
index 82af070da8..7aead6bc3b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/SerializerTestBase.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/SerializerTestBase.java
@@ -135,8 +135,9 @@ public abstract class SerializerTestBase {
private void checkSerialization0(FSTSerializer serializer, final byte[][] in, FST root, boolean hasOutputSymbols)
throws IOException {
byte[] fstData = serializer.serialize(root, new ByteArrayOutputStream()).toByteArray();
+ //TODO: Atri
FST fst = FST.read(new ByteArrayInputStream(fstData), hasOutputSymbols,
- new DirectMemoryManager(SerializerTestBase.class.getName()));
+ new DirectMemoryManager(SerializerTestBase.class.getName()), 0);
checkCorrect(in, fst);
}
@@ -176,8 +177,9 @@ public abstract class SerializerTestBase {
FST fst = FSTBuilder.build(input, new int[]{10, 11, 12, 13});
byte[] fstData = createSerializer().withNumbers().serialize(fst, new ByteArrayOutputStream()).toByteArray();
+ //TODO: Atri
fst =
- FST.read(new ByteArrayInputStream(fstData), true, new DirectMemoryManager(SerializerTestBase.class.getName()));
+ FST.read(new ByteArrayInputStream(fstData), true, new DirectMemoryManager(SerializerTestBase.class.getName()), 0);
// Ensure we have the NUMBERS flag set.
assertTrue(fst.getFlags().contains(NUMBERS));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org