You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2021/12/02 21:14:33 UTC
[pinot] branch master updated: Clean up deprecated fields from segment metadata (#7853)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5006bbc Clean up deprecated fields from segment metadata (#7853)
5006bbc is described below
commit 5006bbc609b34dac0d9b86f5415beb35a94dbdd9
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Thu Dec 2 13:14:06 2021 -0800
Clean up deprecated fields from segment metadata (#7853)
---
.../tests/OfflineClusterIntegrationTest.java | 2 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 35 ++--------------------
.../defaultcolumn/BaseDefaultColumnHandler.java | 17 +++++------
.../loader/invertedindex/JsonIndexHandler.java | 6 ----
.../loader/invertedindex/TextIndexHandler.java | 9 ------
.../org/apache/pinot/segment/spi/V1Constants.java | 15 ----------
.../segment/spi/index/creator/TextIndexType.java | 23 --------------
7 files changed, 10 insertions(+), 97 deletions(-)
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index df79063..8e1e287 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -135,7 +135,7 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
private static final String COLUMN_CARDINALITY_MAP_KEY = "columnCardinalityMap";
// TODO: This might lead to flaky test, as this disk size is not deterministic
// as it depends on the iteration order of a HashSet.
- private static final int DISK_SIZE_IN_BYTES = 20989776;
+ private static final int DISK_SIZE_IN_BYTES = 20796000;
private static final int NUM_ROWS = 115545;
private final List<ServiceStatus.ServiceStatusCallback> _serviceStatusCallbacks =
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
index d34c1a2..764d36f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -64,7 +64,6 @@ import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
import org.apache.pinot.segment.spi.index.creator.JsonIndexCreator;
import org.apache.pinot.segment.spi.index.creator.SegmentIndexCreationInfo;
import org.apache.pinot.segment.spi.index.creator.TextIndexCreator;
-import org.apache.pinot.segment.spi.index.creator.TextIndexType;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.config.table.FSTType;
@@ -700,32 +699,8 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
ColumnIndexCreationInfo columnIndexCreationInfo = entry.getValue();
SegmentDictionaryCreator dictionaryCreator = _dictionaryCreatorMap.get(column);
int dictionaryElementSize = (dictionaryCreator != null) ? dictionaryCreator.getNumBytesPerEntry() : 0;
-
- // TODO: after fixing the server-side dependency on HAS_INVERTED_INDEX and deployed, set HAS_INVERTED_INDEX
- // properly
- // The hasInvertedIndex flag in segment metadata is picked up in ColumnMetadata, and will be used during the query
- // plan phase. If it is set to false, then inverted indexes are not used in queries even if they are created
- // via table
- // configs on segment load. So, we set it to true here for now, until we fix the server to update the value inside
- // ColumnMetadata, export information to the query planner that the inverted index available is current and can
- // be used.
- //
- // boolean hasInvertedIndex = invertedIndexCreatorMap.containsKey();
- boolean hasInvertedIndex = true;
-
- // for new generated segment we write as NONE if text index does not exist
- // for reading existing segments that don't have this property, non-existence
- // of this property will be treated as NONE. See the builder in ColumnMetadata
- TextIndexType textIndexType =
- _textIndexCreatorMap.containsKey(column) ? TextIndexType.LUCENE : TextIndexType.NONE;
-
- boolean hasFSTIndex = _fstIndexCreatorMap.containsKey(column);
-
- boolean hasJsonIndex = _jsonIndexCreatorMap.containsKey(column);
-
addColumnMetadataInfo(properties, column, columnIndexCreationInfo, _totalDocs, _schema.getFieldSpecFor(column),
- _dictionaryCreatorMap.containsKey(column), dictionaryElementSize, hasInvertedIndex, textIndexType,
- hasFSTIndex, hasJsonIndex);
+ _dictionaryCreatorMap.containsKey(column), dictionaryElementSize);
}
properties.save();
@@ -733,8 +708,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
public static void addColumnMetadataInfo(PropertiesConfiguration properties, String column,
ColumnIndexCreationInfo columnIndexCreationInfo, int totalDocs, FieldSpec fieldSpec, boolean hasDictionary,
- int dictionaryElementSize, boolean hasInvertedIndex, TextIndexType textIndexType, boolean hasFSTIndex,
- boolean hasJsonIndex) {
+ int dictionaryElementSize) {
int cardinality = columnIndexCreationInfo.getDistinctValueCount();
properties.setProperty(getKeyFor(column, CARDINALITY), String.valueOf(cardinality));
properties.setProperty(getKeyFor(column, TOTAL_DOCS), String.valueOf(totalDocs));
@@ -745,12 +719,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
properties.setProperty(getKeyFor(column, DICTIONARY_ELEMENT_SIZE), String.valueOf(dictionaryElementSize));
properties.setProperty(getKeyFor(column, COLUMN_TYPE), String.valueOf(fieldSpec.getFieldType()));
properties.setProperty(getKeyFor(column, IS_SORTED), String.valueOf(columnIndexCreationInfo.isSorted()));
- properties.setProperty(getKeyFor(column, HAS_NULL_VALUE), String.valueOf(columnIndexCreationInfo.hasNulls()));
properties.setProperty(getKeyFor(column, HAS_DICTIONARY), String.valueOf(hasDictionary));
- properties.setProperty(getKeyFor(column, TEXT_INDEX_TYPE), textIndexType.name());
- properties.setProperty(getKeyFor(column, HAS_INVERTED_INDEX), String.valueOf(hasInvertedIndex));
- properties.setProperty(getKeyFor(column, HAS_FST_INDEX), String.valueOf(hasFSTIndex));
- properties.setProperty(getKeyFor(column, HAS_JSON_INDEX), String.valueOf(hasJsonIndex));
properties.setProperty(getKeyFor(column, IS_SINGLE_VALUED), String.valueOf(fieldSpec.isSingleValueField()));
properties.setProperty(getKeyFor(column, MAX_MULTI_VALUE_ELEMENTS),
String.valueOf(columnIndexCreationInfo.getMaxNumberOfMultiValueElements()));
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 72413e4..6e791cc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -53,7 +53,6 @@ import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.ColumnIndexCreationInfo;
import org.apache.pinot.segment.spi.creator.StatsCollectorConfig;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
-import org.apache.pinot.segment.spi.index.creator.TextIndexType;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
@@ -154,8 +153,9 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
LoaderUtils.getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.DIMENSIONS, _segmentProperties);
List<String> metricColumns =
LoaderUtils.getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.METRICS, _segmentProperties);
- List<String> dateTimeColumns = LoaderUtils
- .getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.DATETIME_COLUMNS, _segmentProperties);
+ List<String> dateTimeColumns =
+ LoaderUtils.getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.DATETIME_COLUMNS,
+ _segmentProperties);
for (Map.Entry<String, DefaultColumnAction> entry : defaultColumnActionMap.entrySet()) {
String column = entry.getKey();
DefaultColumnAction action = entry.getValue();
@@ -468,10 +468,8 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
}
// Add the column metadata information to the metadata properties.
- SegmentColumnarIndexCreator
- .addColumnMetadataInfo(_segmentProperties, column, columnIndexCreationInfo, totalDocs, fieldSpec,
- true/*hasDictionary*/, dictionaryElementSize, true/*hasInvertedIndex*/, TextIndexType.NONE,
- false/*hasFSTIndex*/, false/*hasJsonIndex*/);
+ SegmentColumnarIndexCreator.addColumnMetadataInfo(_segmentProperties, column, columnIndexCreationInfo, totalDocs,
+ fieldSpec, true/*hasDictionary*/, dictionaryElementSize);
}
/**
@@ -661,9 +659,8 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
}
// Add the column metadata
- SegmentColumnarIndexCreator
- .addColumnMetadataInfo(_segmentProperties, column, indexCreationInfo, numDocs, fieldSpec, true,
- dictionaryCreator.getNumBytesPerEntry(), true, TextIndexType.NONE, false, false);
+ SegmentColumnarIndexCreator.addColumnMetadataInfo(_segmentProperties, column, indexCreationInfo, numDocs,
+ fieldSpec, true, dictionaryCreator.getNumBytesPerEntry());
}
} finally {
for (ValueReader valueReader : valueReaders) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
index d0eb678..93d52e8 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OffHeapJsonIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexHandler;
@@ -33,7 +32,6 @@ import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
-import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
@@ -120,10 +118,6 @@ public class JsonIndexHandler implements IndexHandler {
FileUtils.deleteQuietly(inProgress);
LOGGER.info("Created json index for segment: {}, column: {}", segmentName, columnName);
- PropertiesConfiguration properties = SegmentMetadataImpl.getPropertiesConfiguration(_indexDir);
- properties.setProperty(
- V1Constants.MetadataKeys.Column.getKeyFor(columnName, V1Constants.MetadataKeys.Column.HAS_JSON_INDEX), true);
- properties.save();
}
private void handleDictionaryBasedColumn(ColumnMetadata columnMetadata)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
index c12308e..19e3a91 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
@@ -40,7 +40,6 @@ import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexHandler;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
@@ -49,8 +48,6 @@ import org.apache.pinot.segment.local.segment.index.loader.SegmentPreProcessor;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.index.creator.TextIndexCreator;
-import org.apache.pinot.segment.spi.index.creator.TextIndexType;
-import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
@@ -61,9 +58,6 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.TEXT_INDEX_TYPE;
-import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.getKeyFor;
-
/**
* Helper class for text indexes used by {@link SegmentPreProcessor}.
@@ -160,9 +154,6 @@ public class TextIndexHandler implements IndexHandler {
}
LOGGER.info("Created text index for column: {} in segment: {}", column, segmentName);
- PropertiesConfiguration properties = SegmentMetadataImpl.getPropertiesConfiguration(_indexDir);
- properties.setProperty(getKeyFor(column, TEXT_INDEX_TYPE), TextIndexType.LUCENE.name());
- properties.save();
}
private void processSVField(boolean hasDictionary, ForwardIndexReader forwardIndexReader,
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index e037544..5f45aa5 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -100,21 +100,6 @@ public class V1Constants {
public static final String DATETIME_FORMAT = "datetimeFormat";
public static final String DATETIME_GRANULARITY = "datetimeGranularity";
- // TODO: Remove these 2 fields after releasing 0.8.0 because they are always set to true and never used
- @Deprecated
- public static final String HAS_NULL_VALUE = "hasNullValue";
- @Deprecated
- public static final String HAS_INVERTED_INDEX = "hasInvertedIndex";
-
- // TODO: Remove these 3 fields after releasing 0.8.0 because the index info is maintained within the DataSource
- // based on the actual indexes loaded
- @Deprecated
- public static final String HAS_FST_INDEX = "hasFSTIndex";
- @Deprecated
- public static final String TEXT_INDEX_TYPE = "textIndexType";
- @Deprecated
- public static final String HAS_JSON_INDEX = "hasJsonIndex";
-
public static final String COLUMN_PROPS_KEY_PREFIX = "column.";
public static String getKeyFor(String column, String key) {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/TextIndexType.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/TextIndexType.java
deleted file mode 100644
index 991d5e8..0000000
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/TextIndexType.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.segment.spi.index.creator;
-
-public enum TextIndexType {
- NONE, LUCENE, LUCENE_FST
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org