You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2022/12/21 09:13:20 UTC
[pinot] branch master updated: Handle in-memory segment metadata for index checking (#10017)
This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5dc60be5d5 Handle in-memory segment metadata for index checking (#10017)
5dc60be5d5 is described below
commit 5dc60be5d522a478721ef082105fbfe41bdfbe94
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Wed Dec 21 01:13:13 2022 -0800
Handle in-memory segment metadata for index checking (#10017)
---
.../index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java | 7 +++++--
.../index/loader/defaultcolumn/BaseDefaultColumnHandler.java | 6 ++++--
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
index b426f8ade9..7f3c97c16d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
@@ -42,16 +42,18 @@ import static org.apache.pinot.spi.data.FieldSpec.DataType;
public class ColumnMinMaxValueGenerator {
private final SegmentMetadata _segmentMetadata;
- private final PropertiesConfiguration _segmentProperties;
private final SegmentDirectory.Writer _segmentWriter;
private final ColumnMinMaxValueGeneratorMode _columnMinMaxValueGeneratorMode;
+ // NOTE: _segmentProperties shouldn't be used when checking whether min/max value need to be generated because at that
+ // time _segmentMetadata might not be loaded from a local file
+ private PropertiesConfiguration _segmentProperties;
+
private boolean _minMaxValueAdded;
public ColumnMinMaxValueGenerator(SegmentMetadata segmentMetadata, SegmentDirectory.Writer segmentWriter,
ColumnMinMaxValueGeneratorMode columnMinMaxValueGeneratorMode) {
_segmentMetadata = segmentMetadata;
- _segmentProperties = SegmentMetadataUtils.getPropertiesConfiguration(segmentMetadata);
_segmentWriter = segmentWriter;
_columnMinMaxValueGeneratorMode = columnMinMaxValueGeneratorMode;
}
@@ -68,6 +70,7 @@ public class ColumnMinMaxValueGenerator {
public void addColumnMinMaxValue()
throws Exception {
Preconditions.checkState(_columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE);
+ _segmentProperties = SegmentMetadataUtils.getPropertiesConfiguration(_segmentMetadata);
for (String column : getColumnsToAddMinMaxValue()) {
addColumnMinMaxValueForColumn(column);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 9e3bb38449..5d647cd335 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -120,7 +120,9 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
protected final Schema _schema;
protected final SegmentDirectory.Writer _segmentWriter;
- private final PropertiesConfiguration _segmentProperties;
+ // NOTE: _segmentProperties shouldn't be used when checking whether default column need to be created because at that
+ // time _segmentMetadata might not be loaded from a local file
+ private PropertiesConfiguration _segmentProperties;
protected BaseDefaultColumnHandler(File indexDir, SegmentMetadata segmentMetadata,
IndexLoadingConfig indexLoadingConfig, Schema schema, SegmentDirectory.Writer segmentWriter) {
@@ -129,7 +131,6 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
_indexLoadingConfig = indexLoadingConfig;
_schema = schema;
_segmentWriter = segmentWriter;
- _segmentProperties = SegmentMetadataUtils.getPropertiesConfiguration(segmentMetadata);
}
@Override
@@ -151,6 +152,7 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
}
// Update each default column based on the default column action.
+ _segmentProperties = SegmentMetadataUtils.getPropertiesConfiguration(_segmentMetadata);
Iterator<Map.Entry<String, DefaultColumnAction>> entryIterator = defaultColumnActionMap.entrySet().iterator();
while (entryIterator.hasNext()) {
Map.Entry<String, DefaultColumnAction> entry = entryIterator.next();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org