You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by "Jackie-Jiang (via GitHub)" <gi...@apache.org> on 2023/09/25 07:21:16 UTC

[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #11669: Cleanup some reader/writer logic for raw forward index

Jackie-Jiang commented on code in PR #11669:
URL: https://github.com/apache/pinot/pull/11669#discussion_r1335479733


##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexReaderFactory.java:
##########
@@ -69,22 +68,26 @@ public static ForwardIndexReader createIndexReader(PinotDataBuffer dataBuffer, C
             metadata.getBitsPerElement());
       }
     } else {
-      FieldSpec.DataType storedType = metadata.getDataType().getStoredType();
-      if (metadata.isSingleValue()) {
-        int version = dataBuffer.getInt(0);
-        if (storedType.isFixedWidth()) {
-          return version >= FixedBytePower2ChunkSVForwardIndexReader.VERSION
-              ? new FixedBytePower2ChunkSVForwardIndexReader(dataBuffer, storedType)
-              : new FixedByteChunkSVForwardIndexReader(dataBuffer, storedType);
-        }
-        if (version >= VarByteChunkSVForwardIndexWriterV4.VERSION) {
-          return new VarByteChunkSVForwardIndexReaderV4(dataBuffer, storedType);
-        }
-        return new VarByteChunkSVForwardIndexReader(dataBuffer, storedType);
+      return createRawIndexReader(dataBuffer, metadata.getDataType().getStoredType(), metadata.isSingleValue());
+    }
+  }
+
+  public static ForwardIndexReader createRawIndexReader(PinotDataBuffer dataBuffer, DataType storedType,
+      boolean isSingleValue) {
+    int version = dataBuffer.getInt(0);
+    if (isSingleValue) {
+      if (storedType.isFixedWidth()) {
+        return version == FixedBytePower2ChunkSVForwardIndexReader.VERSION
+            ? new FixedBytePower2ChunkSVForwardIndexReader(dataBuffer, storedType)
+            : new FixedByteChunkSVForwardIndexReader(dataBuffer, storedType);
       } else {
-        return storedType.isFixedWidth() ? new FixedByteChunkMVForwardIndexReader(dataBuffer, storedType)
-            : new VarByteChunkMVForwardIndexReader(dataBuffer, storedType);
+        return version == VarByteChunkForwardIndexWriterV4.VERSION ? new VarByteChunkSVForwardIndexReaderV4(dataBuffer,
+            storedType) : new VarByteChunkSVForwardIndexReader(dataBuffer, storedType);
       }
+    } else {
+      // TODO: Support V4 MV reader

Review Comment:
   ^^



##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java:
##########
@@ -77,13 +76,17 @@ public MultiValueVarByteRawIndexCreator(File baseIndexDir, ChunkCompressionType
     //we will prepend the actual content with numElements and length array containing length of each element
     int totalMaxLength = getTotalRowStorageBytes(maxNumberOfElements, maxRowLengthInBytes);
 
-    File file = new File(baseIndexDir,
-        column + Indexes.RAW_MV_FORWARD_INDEX_FILE_EXTENSION);
+    File file = new File(baseIndexDir, column + Indexes.RAW_MV_FORWARD_INDEX_FILE_EXTENSION);
     int numDocsPerChunk = Math.max(
-        TARGET_MAX_CHUNK_SIZE / (totalMaxLength + VarByteChunkSVForwardIndexWriter.CHUNK_HEADER_ENTRY_ROW_OFFSET_SIZE),
+        TARGET_MAX_CHUNK_SIZE / (totalMaxLength + VarByteChunkForwardIndexWriter.CHUNK_HEADER_ENTRY_ROW_OFFSET_SIZE),
         1);
-    _indexWriter = new VarByteChunkSVForwardIndexWriter(file, compressionType, totalDocs, numDocsPerChunk,
-        totalMaxLength, writerVersion);
+    // TODO: Support V4 MV reader

Review Comment:
   ^^



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org