You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2021/10/24 01:04:35 UTC
[pinot] branch master updated: Fix star-tree index map when column
name contains '.' (#7623)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0327932 Fix star-tree index map when column name contains '.' (#7623)
0327932 is described below
commit 032793268997414cb63a82f1b6908a42b1bee910
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Sat Oct 23 18:04:13 2021 -0700
Fix star-tree index map when column name contains '.' (#7623)
---
.../startree/v2/store/StarTreeIndexContainer.java | 11 +++---
.../startree/v2/store/StarTreeIndexMapUtils.java | 42 +++++++++++++---------
2 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java
index b243a9f..d1fe668 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.nio.ByteOrder;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.ConfigurationException;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
@@ -45,14 +44,14 @@ public class StarTreeIndexContainer implements Closeable {
public StarTreeIndexContainer(File segmentDirectory, SegmentMetadataImpl segmentMetadata,
Map<String, ColumnIndexContainer> indexContainerMap, ReadMode readMode)
- throws ConfigurationException, IOException {
+ throws IOException {
File indexFile = new File(segmentDirectory, StarTreeV2Constants.INDEX_FILE_NAME);
if (readMode == ReadMode.heap) {
- _dataBuffer = PinotDataBuffer
- .loadFile(indexFile, 0, indexFile.length(), ByteOrder.LITTLE_ENDIAN, "Star-tree V2 data buffer");
+ _dataBuffer = PinotDataBuffer.loadFile(indexFile, 0, indexFile.length(), ByteOrder.LITTLE_ENDIAN,
+ "Star-tree V2 data buffer");
} else {
- _dataBuffer = PinotDataBuffer
- .mapFile(indexFile, true, 0, indexFile.length(), ByteOrder.LITTLE_ENDIAN, "Star-tree V2 data buffer");
+ _dataBuffer = PinotDataBuffer.mapFile(indexFile, true, 0, indexFile.length(), ByteOrder.LITTLE_ENDIAN,
+ "Star-tree V2 data buffer");
}
File indexMapFile = new File(segmentDirectory, StarTreeV2Constants.INDEX_MAP_FILE_NAME);
List<Map<IndexKey, IndexValue>> indexMapList =
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
index 4d0ef3f..1ed1fd1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
@@ -22,16 +22,16 @@ import com.google.common.base.Preconditions;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
@@ -68,6 +68,8 @@ public class StarTreeIndexMapUtils {
public static final IndexKey STAR_TREE_INDEX_KEY = new IndexKey(IndexType.STAR_TREE, null);
+ private static final char KEY_SEPARATOR = '.';
+ private static final String KEY_TEMPLATE = "%d.%s.%s.%s";
private static final String OFFSET_SUFFIX = "OFFSET";
private static final String SIZE_SUFFIX = "SIZE";
@@ -95,7 +97,7 @@ public class StarTreeIndexMapUtils {
* Returns the property name for the index.
*/
public String getPropertyName(int starTreeId, String suffix) {
- return String.format("%d.%s.%s.%s", starTreeId, _column, _indexType, suffix);
+ return String.format(KEY_TEMPLATE, starTreeId, _column, _indexType, suffix);
}
@Override
@@ -166,37 +168,45 @@ public class StarTreeIndexMapUtils {
/**
* Loads the index maps for multiple star-trees from a file.
*/
- public static List<Map<IndexKey, IndexValue>> loadFromFile(File indexMapFile, int numStarTrees)
- throws ConfigurationException {
+ public static List<Map<IndexKey, IndexValue>> loadFromFile(File indexMapFile, int numStarTrees) {
Preconditions.checkState(indexMapFile.exists(), "Star-tree index map file does not exist");
- PropertiesConfiguration configuration = CommonsConfigurationUtils.fromFile(indexMapFile);
-
- List<Map<IndexKey, IndexValue>> indexMaps =
- IntStream.range(0, numStarTrees).boxed().map(index -> new HashMap<IndexKey, IndexValue>())
- .collect(Collectors.toList());
+ List<Map<IndexKey, IndexValue>> indexMaps = new ArrayList<>(numStarTrees);
+ for (int i = 0; i < numStarTrees; i++) {
+ indexMaps.add(new HashMap<>());
+ }
+ PropertiesConfiguration configuration = CommonsConfigurationUtils.fromFile(indexMapFile);
for (String key : CommonsConfigurationUtils.getKeys(configuration)) {
- String[] split = key.split("\\.");
- Preconditions.checkState(split.length == 4,
- "Invalid key: " + key + " in star-tree index map file: " + indexMapFile.getAbsolutePath());
+ String[] split = StringUtils.split(key, KEY_SEPARATOR);
int starTreeId = Integer.parseInt(split[0]);
Map<IndexKey, IndexValue> indexMap = indexMaps.get(starTreeId);
- IndexType indexType = IndexType.valueOf(split[2]);
+
+ // Handle the case of column name containing '.'
+ String column;
+ int columnSplitEndIndex = split.length - 2;
+ if (columnSplitEndIndex == 2) {
+ column = split[1];
+ } else {
+ column = StringUtils.join(split, KEY_SEPARATOR, 1, columnSplitEndIndex);
+ }
+
+ IndexType indexType = IndexType.valueOf(split[columnSplitEndIndex]);
IndexKey indexKey;
if (indexType == IndexType.STAR_TREE) {
indexKey = STAR_TREE_INDEX_KEY;
} else {
- indexKey = new IndexKey(IndexType.FORWARD_INDEX, split[1]);
+ indexKey = new IndexKey(IndexType.FORWARD_INDEX, column);
}
IndexValue indexValue = indexMap.computeIfAbsent(indexKey, (k) -> new IndexValue());
long value = configuration.getLong(key);
- if (split[3].equals(OFFSET_SUFFIX)) {
+ if (split[columnSplitEndIndex + 1].equals(OFFSET_SUFFIX)) {
indexValue._offset = value;
} else {
indexValue._size = value;
}
}
+
return indexMaps;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org