You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2022/09/21 21:05:47 UTC
[pinot] branch master updated: make IndexHandler methods reentrant (#9440)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 44da34884a make IndexHandler methods reentrant (#9440)
44da34884a is described below
commit 44da34884a69ac0d6f2a8dbd3d974da7216440ef
Author: Xiaobing <61...@users.noreply.github.com>
AuthorDate: Wed Sep 21 14:05:40 2022 -0700
make IndexHandler methods reentrant (#9440)
---
.../index/loader/invertedindex/FSTIndexHandler.java | 12 +++++++-----
.../index/loader/invertedindex/InvertedIndexHandler.java | 14 ++++++++------
.../index/loader/invertedindex/JsonIndexHandler.java | 14 ++++++++------
.../index/loader/invertedindex/RangeIndexHandler.java | 12 +++++++-----
.../index/loader/invertedindex/TextIndexHandler.java | 12 +++++++-----
5 files changed, 37 insertions(+), 27 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
index 7b7b9ff485..3b74a5cbbb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
@@ -74,22 +74,23 @@ public class FSTIndexHandler implements IndexHandler {
public FSTIndexHandler(SegmentMetadata segmentMetadata, IndexLoadingConfig indexLoadingConfig) {
_segmentMetadata = segmentMetadata;
_fstType = indexLoadingConfig.getFSTIndexType();
- _columnsToAddIdx = new HashSet<>(indexLoadingConfig.getFSTIndexColumns());
+ _columnsToAddIdx = indexLoadingConfig.getFSTIndexColumns();
}
@Override
public boolean needUpdateIndices(SegmentDirectory.Reader segmentReader) {
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentReader.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.FST_INDEX);
// Check if any existing index need to be removed.
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Need to remove existing FST index from segment: {}, column: {}", segmentName, column);
return true;
}
}
// Check if any new index need to be added.
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateFSTIndex(columnMetadata)) {
LOGGER.info("Need to create new FST index for segment: {}, column: {}", segmentName, column);
@@ -104,15 +105,16 @@ public class FSTIndexHandler implements IndexHandler {
throws Exception {
// Remove indices not set in table config any more
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentWriter.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.FST_INDEX);
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Removing existing FST index from segment: {}, column: {}", segmentName, column);
segmentWriter.removeIndex(column, ColumnIndexType.FST_INDEX);
LOGGER.info("Removed existing FST index from segment: {}, column: {}", segmentName, column);
}
}
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateFSTIndex(columnMetadata)) {
createFSTIndexForColumn(segmentWriter, columnMetadata, indexCreatorProvider);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
index cb0b076635..d9508cca06 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
@@ -47,27 +47,28 @@ public class InvertedIndexHandler implements IndexHandler {
private static final Logger LOGGER = LoggerFactory.getLogger(InvertedIndexHandler.class);
private final SegmentMetadata _segmentMetadata;
- private final HashSet<String> _columnsToAddIdx;
+ private final Set<String> _columnsToAddIdx;
public InvertedIndexHandler(SegmentMetadata segmentMetadata, IndexLoadingConfig indexLoadingConfig) {
_segmentMetadata = segmentMetadata;
- _columnsToAddIdx = new HashSet<>(indexLoadingConfig.getInvertedIndexColumns());
+ _columnsToAddIdx = indexLoadingConfig.getInvertedIndexColumns();
}
@Override
public boolean needUpdateIndices(SegmentDirectory.Reader segmentReader) {
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns =
segmentReader.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.INVERTED_INDEX);
// Check if any existing index need to be removed.
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Need to remove existing inverted index from segment: {}, column: {}", segmentName, column);
return true;
}
}
// Check if any new index need to be added.
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateInvertedIndex(columnMetadata)) {
LOGGER.info("Need to create new inverted index for segment: {}, column: {}", segmentName, column);
@@ -82,16 +83,17 @@ public class InvertedIndexHandler implements IndexHandler {
throws IOException {
// Remove indices not set in table config any more.
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns =
segmentWriter.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.INVERTED_INDEX);
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Removing existing inverted index from segment: {}, column: {}", segmentName, column);
segmentWriter.removeIndex(column, ColumnIndexType.INVERTED_INDEX);
LOGGER.info("Removed existing inverted index from segment: {}, column: {}", segmentName, column);
}
}
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateInvertedIndex(columnMetadata)) {
createInvertedIndexForColumn(segmentWriter, columnMetadata, indexCreatorProvider);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
index 2b582c6dcf..e064448673 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
@@ -50,26 +50,27 @@ public class JsonIndexHandler implements IndexHandler {
private static final Logger LOGGER = LoggerFactory.getLogger(JsonIndexHandler.class);
private final SegmentMetadata _segmentMetadata;
- private final HashSet<String> _columnsToAddIdx;
+ private final Set<String> _columnsToAddIdx;
public JsonIndexHandler(SegmentMetadata segmentMetadata, IndexLoadingConfig indexLoadingConfig) {
_segmentMetadata = segmentMetadata;
- _columnsToAddIdx = new HashSet<>(indexLoadingConfig.getJsonIndexColumns());
+ _columnsToAddIdx = indexLoadingConfig.getJsonIndexColumns();
}
@Override
public boolean needUpdateIndices(SegmentDirectory.Reader segmentReader) {
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentReader.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.JSON_INDEX);
// Check if any existing index need to be removed.
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Need to remove existing json index from segment: {}, column: {}", segmentName, column);
return true;
}
}
// Check if any new index need to be added.
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateJsonIndex(columnMetadata)) {
LOGGER.info("Need to create new json index for segment: {}, column: {}", segmentName, column);
@@ -84,15 +85,16 @@ public class JsonIndexHandler implements IndexHandler {
throws Exception {
// Remove indices not set in table config any more
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentWriter.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.JSON_INDEX);
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Removing existing json index from segment: {}, column: {}", segmentName, column);
segmentWriter.removeIndex(column, ColumnIndexType.JSON_INDEX);
LOGGER.info("Removed existing json index from segment: {}, column: {}", segmentName, column);
}
}
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateJsonIndex(columnMetadata)) {
createJsonIndexForColumn(segmentWriter, columnMetadata, indexCreatorProvider);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
index c16ae4629b..4f19a2b6d4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
@@ -52,23 +52,24 @@ public class RangeIndexHandler implements IndexHandler {
public RangeIndexHandler(SegmentMetadata segmentMetadata, IndexLoadingConfig indexLoadingConfig) {
_segmentMetadata = segmentMetadata;
- _columnsToAddIdx = new HashSet<>(indexLoadingConfig.getRangeIndexColumns());
+ _columnsToAddIdx = indexLoadingConfig.getRangeIndexColumns();
_rangeIndexVersion = indexLoadingConfig.getRangeIndexVersion();
}
@Override
public boolean needUpdateIndices(SegmentDirectory.Reader segmentReader) {
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentReader.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.RANGE_INDEX);
// Check if any existing index need to be removed.
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Need to remove existing range index from segment: {}, column: {}", segmentName, column);
return true;
}
}
// Check if any new index need to be added.
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateRangeIndex(columnMetadata)) {
LOGGER.info("Need to create new range index for segment: {}, column: {}", segmentName, column);
@@ -83,15 +84,16 @@ public class RangeIndexHandler implements IndexHandler {
throws IOException {
// Remove indices not set in table config any more
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentWriter.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.RANGE_INDEX);
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Removing existing range index from segment: {}, column: {}", segmentName, column);
segmentWriter.removeIndex(column, ColumnIndexType.RANGE_INDEX);
LOGGER.info("Removed existing range index from segment: {}, column: {}", segmentName, column);
}
}
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateRangeIndex(columnMetadata)) {
createRangeIndexForColumn(segmentWriter, columnMetadata, indexCreatorProvider);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
index d18407cc29..4f47c298bb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
@@ -91,22 +91,23 @@ public class TextIndexHandler implements IndexHandler {
public TextIndexHandler(SegmentMetadata segmentMetadata, IndexLoadingConfig indexLoadingConfig) {
_segmentMetadata = segmentMetadata;
_fstType = indexLoadingConfig.getFSTIndexType();
- _columnsToAddIdx = new HashSet<>(indexLoadingConfig.getTextIndexColumns());
+ _columnsToAddIdx = indexLoadingConfig.getTextIndexColumns();
}
@Override
public boolean needUpdateIndices(SegmentDirectory.Reader segmentReader) {
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentReader.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.TEXT_INDEX);
// Check if any existing index need to be removed.
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Need to remove existing text index from segment: {}, column: {}", segmentName, column);
return true;
}
}
// Check if any new index need to be added.
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateTextIndex(columnMetadata)) {
LOGGER.info("Need to create new text index for segment: {}, column: {}", segmentName, column);
@@ -121,15 +122,16 @@ public class TextIndexHandler implements IndexHandler {
throws Exception {
// Remove indices not set in table config any more
String segmentName = _segmentMetadata.getName();
+ Set<String> columnsToAddIdx = new HashSet<>(_columnsToAddIdx);
Set<String> existingColumns = segmentWriter.toSegmentDirectory().getColumnsWithIndex(ColumnIndexType.TEXT_INDEX);
for (String column : existingColumns) {
- if (!_columnsToAddIdx.remove(column)) {
+ if (!columnsToAddIdx.remove(column)) {
LOGGER.info("Removing existing text index from segment: {}, column: {}", segmentName, column);
segmentWriter.removeIndex(column, ColumnIndexType.TEXT_INDEX);
LOGGER.info("Removed existing text index from segment: {}, column: {}", segmentName, column);
}
}
- for (String column : _columnsToAddIdx) {
+ for (String column : columnsToAddIdx) {
ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
if (shouldCreateTextIndex(columnMetadata)) {
createTextIndexForColumn(segmentWriter, columnMetadata, indexCreatorProvider);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org