You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2020/08/07 03:52:24 UTC
[incubator-pinot] branch add-virtual-key-to-segment-metadata
created (now ac6e647)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch add-virtual-key-to-segment-metadata
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at ac6e647 Identify whether the values are converted from map and emit server metric
This branch includes the following new commits:
new ac6e647 Identify whether the values are converted from map and emit server metric
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Identify whether the values are converted
from map and emit server metric
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch add-virtual-key-to-segment-metadata
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit ac6e64709416f0422d90502c691a3d9e7b6c248f
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Thu Aug 6 20:51:51 2020 -0700
Identify whether the values are converted from map and emit server metric
---
.../apache/pinot/common/metrics/ServerGauge.java | 3 +-
.../recordtransformer/DataTypeTransformer.java | 22 ++++---
.../creator/impl/SegmentColumnarIndexCreator.java | 4 ++
.../core/segment/creator/impl/V1Constants.java | 1 +
.../index/metadata/SegmentMetadataImpl.java | 28 ++++-----
.../recordtransformer/DataTypeTransformerTest.java | 70 +++++++++++-----------
.../starter/helix/SegmentFetcherAndLoader.java | 9 ++-
.../apache/pinot/spi/data/readers/GenericRow.java | 6 ++
8 files changed, 83 insertions(+), 60 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
index d739c0f..de43278 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
@@ -40,7 +40,8 @@ public enum ServerGauge implements AbstractMetrics.Gauge {
LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS("seconds", false),
REALTIME_OFFHEAP_MEMORY_USED("bytes", false),
REALTIME_SEGMENT_NUM_PARTITIONS("realtimeSegmentNumPartitions", false),
- LLC_SIMULTANEOUS_SEGMENT_BUILDS("llcSimultaneousSegmentBuilds", true);
+ LLC_SIMULTANEOUS_SEGMENT_BUILDS("llcSimultaneousSegmentBuilds", true),
+ SEGMENT_VALUES_CONVERTED_FROM_MAP_TO_ARRAY("segments", false);
private final String gaugeName;
private final String unit;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
index ccdda33..bc2d777 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
@@ -84,7 +84,7 @@ public class DataTypeTransformer implements RecordTransformer {
continue;
}
PinotDataType dest = entry.getValue();
- value = standardize(column, value, dest.isSingleValue());
+ value = standardize(record, column, value, dest.isSingleValue());
// NOTE: The standardized value could be null for empty Collection/Map/Object[].
if (value == null) {
record.putValue(column, null);
@@ -126,15 +126,19 @@ public class DataTypeTransformer implements RecordTransformer {
*/
@VisibleForTesting
@Nullable
- static Object standardize(String column, @Nullable Object value, boolean isSingleValue) {
+ static Object standardize(GenericRow record, String column, @Nullable Object value, boolean isSingleValue) {
if (value == null) {
return null;
}
if (value instanceof Collection) {
- return standardizeCollection(column, (Collection) value, isSingleValue);
+ return standardizeCollection(record, column, (Collection) value, isSingleValue);
}
if (value instanceof Map) {
- return standardizeCollection(column, ((Map) value).values(), isSingleValue);
+ Collection values = ((Map) value).values();
+ if (values.size() == 1) {
+ record.putValue(GenericRow.CONVERT_MAP_VALUE_TO_ARRAY_VALUE_KEY, Boolean.TRUE.toString());
+ }
+ return standardizeCollection(record, column, values, isSingleValue);
}
if (value instanceof Object[]) {
Object[] values = (Object[]) value;
@@ -143,11 +147,11 @@ public class DataTypeTransformer implements RecordTransformer {
return null;
}
if (numValues == 1) {
- return standardize(column, values[0], isSingleValue);
+ return standardize(record, column, values[0], isSingleValue);
}
List<Object> standardizedValues = new ArrayList<>(numValues);
for (Object singleValue : values) {
- Object standardizedValue = standardize(column, singleValue, true);
+ Object standardizedValue = standardize(record, column, singleValue, true);
if (standardizedValue != null) {
standardizedValues.add(standardizedValue);
}
@@ -166,17 +170,17 @@ public class DataTypeTransformer implements RecordTransformer {
return value;
}
- private static Object standardizeCollection(String column, Collection collection, boolean isSingleValue) {
+ private static Object standardizeCollection(GenericRow record, String column, Collection collection, boolean isSingleValue) {
int numValues = collection.size();
if (numValues == 0) {
return null;
}
if (numValues == 1) {
- return standardize(column, collection.iterator().next(), isSingleValue);
+ return standardize(record, column, collection.iterator().next(), isSingleValue);
}
List<Object> standardizedValues = new ArrayList<>(numValues);
for (Object singleValue : collection) {
- Object standardizedValue = standardize(column, singleValue, true);
+ Object standardizedValue = standardize(record, column, singleValue, true);
if (standardizedValue != null) {
standardizedValues.add(standardizedValue);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentColumnarIndexCreator.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentColumnarIndexCreator.java
index 34131ab..21a37a6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentColumnarIndexCreator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -91,6 +91,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
private int totalDocs;
private int docIdCounter;
private boolean _nullHandlingEnabled;
+ private boolean _convertMapToArray;
private final Set<String> _textIndexColumns = new HashSet<>();
@@ -217,6 +218,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
// Initialize Null value vector map
_nullValueVectorCreatorMap.put(columnName, new NullValueVectorCreator(_indexDir, columnName));
}
+ _convertMapToArray = false;
}
}
@@ -304,6 +306,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
@Override
public void indexRow(GenericRow row) {
+ _convertMapToArray |= Boolean.parseBoolean((String) row.getValue(GenericRow.CONVERT_MAP_VALUE_TO_ARRAY_VALUE_KEY));
for (Map.Entry<String, ForwardIndexCreator> entry : _forwardIndexCreatorMap.entrySet()) {
String columnName = entry.getKey();
ForwardIndexCreator forwardIndexCreator = entry.getValue();
@@ -414,6 +417,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
String timeColumnName = config.getTimeColumnName();
properties.setProperty(TIME_COLUMN_NAME, timeColumnName);
properties.setProperty(SEGMENT_TOTAL_DOCS, String.valueOf(totalDocs));
+ properties.setProperty(SEGMENT_CONVERT_MAP_VALUE_TO_ARRAY_VALUE, _convertMapToArray);
// Write time related metadata (start time, end time, time unit)
if (timeColumnName != null) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/V1Constants.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/V1Constants.java
index 0498f8c..56cf320 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/V1Constants.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/V1Constants.java
@@ -62,6 +62,7 @@ public class V1Constants {
public static final String DATETIME_COLUMNS = "segment.datetime.column.names";
public static final String SEGMENT_TOTAL_DOCS = "segment.total.docs";
public static final String SEGMENT_PADDING_CHARACTER = "segment.padding.character";
+ public static final String SEGMENT_CONVERT_MAP_VALUE_TO_ARRAY_VALUE = "segment.convert.map.value.to.array.value";
}
public static class Column {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/metadata/SegmentMetadataImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/metadata/SegmentMetadataImpl.java
index 1245034..a127652 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/metadata/SegmentMetadataImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/metadata/SegmentMetadataImpl.java
@@ -18,20 +18,6 @@
*/
package org.apache.pinot.core.segment.index.metadata;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.DATETIME_COLUMNS;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.DIMENSIONS;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.METRICS;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_CREATOR_VERSION;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_END_TIME;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_NAME;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_PADDING_CHARACTER;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_START_TIME;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.SEGMENT_VERSION;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.TABLE_NAME;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.TIME_COLUMN_NAME;
-import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.TIME_UNIT;
-
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -75,6 +61,8 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
+import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.*;
+
public class SegmentMetadataImpl implements SegmentMetadata {
private static final Logger LOGGER = LoggerFactory.getLogger(SegmentMetadataImpl.class);
@@ -104,6 +92,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
private int _totalDocs;
private long _segmentStartTime;
private long _segmentEndTime;
+ private boolean _valuesConvertedFromMapToArray;
/**
* For segments on disk.
@@ -165,7 +154,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
public static PropertiesConfiguration getPropertiesConfiguration(File indexDir) {
File metadataFile = SegmentDirectoryPaths.findMetadataFile(indexDir);
Preconditions.checkNotNull(metadataFile, "Cannot find segment metadata file under directory: %s", indexDir);
-
+
return CommonsConfigurationUtils.fromFile(metadataFile);
}
@@ -263,6 +252,11 @@ public class SegmentMetadataImpl implements SegmentMetadata {
segmentMetadataPropertiesConfiguration.subset(StarTreeV2Constants.MetadataKey.getStarTreePrefix(i))));
}
}
+
+ if (segmentMetadataPropertiesConfiguration.containsKey(SEGMENT_CONVERT_MAP_VALUE_TO_ARRAY_VALUE)) {
+ _valuesConvertedFromMapToArray =
+ segmentMetadataPropertiesConfiguration.getBoolean(SEGMENT_CONVERT_MAP_VALUE_TO_ARRAY_VALUE);
+ }
}
/**
@@ -489,6 +483,10 @@ public class SegmentMetadataImpl implements SegmentMetadata {
return _paddingCharacter;
}
+ public boolean areValuesConvertedFromMapToArray() {
+ return _valuesConvertedFromMapToArray;
+ }
+
/**
* Converts segment metadata to json
* @param columnFilter list only the columns in the set. Lists all the columns if
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformerTest.java
index a60c460..be4ad8d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformerTest.java
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.pinot.spi.data.readers.GenericRow;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -39,17 +40,18 @@ public class DataTypeTransformerTest {
/**
* Tests for Map
*/
+ GenericRow record = new GenericRow();
// Empty Map
Map<String, Object> map = Collections.emptyMap();
- assertNull(DataTypeTransformer.standardize(COLUMN, map, true));
- assertNull(DataTypeTransformer.standardize(COLUMN, map, false));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, map, true));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, map, false));
// Map with single entry
String expectedValue = "testValue";
map = Collections.singletonMap("testKey", expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, map, true), expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, map, false), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, map, true), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, map, false), expectedValue);
// Map with multiple entries
Object[] expectedValues = new Object[]{"testValue1", "testValue2"};
@@ -58,12 +60,12 @@ public class DataTypeTransformerTest {
map.put("testKey2", "testValue2");
try {
// Should fail because Map with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, map, true);
+ DataTypeTransformer.standardize(record, COLUMN, map, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(COLUMN, map, false), expectedValues);
+ assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(record, COLUMN, map, false), expectedValues);
/**
* Tests for List
@@ -71,24 +73,24 @@ public class DataTypeTransformerTest {
// Empty List
List<Object> list = Collections.emptyList();
- assertNull(DataTypeTransformer.standardize(COLUMN, list, true));
- assertNull(DataTypeTransformer.standardize(COLUMN, list, false));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, list, true));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, list, false));
// List with single entry
list = Collections.singletonList(expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, list, true), expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, list, false), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, list, true), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, list, false), expectedValue);
// List with multiple entries
list = Arrays.asList(expectedValues);
try {
// Should fail because List with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, list, true);
+ DataTypeTransformer.standardize(record, COLUMN, list, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEquals((Object[]) DataTypeTransformer.standardize(COLUMN, list, false), expectedValues);
+ assertEquals((Object[]) DataTypeTransformer.standardize(record, COLUMN, list, false), expectedValues);
/**
* Tests for Object[]
@@ -96,24 +98,24 @@ public class DataTypeTransformerTest {
// Empty Object[]
Object[] values = new Object[0];
- assertNull(DataTypeTransformer.standardize(COLUMN, values, true));
- assertNull(DataTypeTransformer.standardize(COLUMN, values, false));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, values, true));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, values, false));
// Object[] with single entry
values = new Object[]{expectedValue};
- assertEquals(DataTypeTransformer.standardize(COLUMN, values, true), expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, values, false), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, values, true), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, values, false), expectedValue);
// Object[] with multiple entries
values = new Object[]{"testValue1", "testValue2"};
try {
// Should fail because Object[] with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, values, true);
+ DataTypeTransformer.standardize(record, COLUMN, values, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEquals((Object[]) DataTypeTransformer.standardize(COLUMN, values, false), expectedValues);
+ assertEquals((Object[]) DataTypeTransformer.standardize(record, COLUMN, values, false), expectedValues);
/**
* Tests for nested Map/List/Object[]
@@ -121,32 +123,32 @@ public class DataTypeTransformerTest {
// Map with empty List
map = Collections.singletonMap("testKey", Collections.emptyList());
- assertNull(DataTypeTransformer.standardize(COLUMN, map, true));
- assertNull(DataTypeTransformer.standardize(COLUMN, map, false));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, map, true));
+ assertNull(DataTypeTransformer.standardize(record, COLUMN, map, false));
// Map with single-entry List
map = Collections.singletonMap("testKey", Collections.singletonList(expectedValue));
- assertEquals(DataTypeTransformer.standardize(COLUMN, map, true), expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, map, false), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, map, true), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, map, false), expectedValue);
// Map with one empty Map and one single-entry Map
map = new HashMap<>();
map.put("testKey1", Collections.emptyMap());
map.put("testKey2", Collections.singletonMap("testKey", expectedValue));
// Can be standardized into single value because empty Map should be ignored
- assertEquals(DataTypeTransformer.standardize(COLUMN, map, true), expectedValue);
- assertEquals(DataTypeTransformer.standardize(COLUMN, map, false), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, map, true), expectedValue);
+ assertEquals(DataTypeTransformer.standardize(record, COLUMN, map, false), expectedValue);
// Map with multi-entries List
map = Collections.singletonMap("testKey", Arrays.asList(expectedValues));
try {
// Should fail because Map with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, map, true);
+ DataTypeTransformer.standardize(record, COLUMN, map, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(COLUMN, map, false), expectedValues);
+ assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(record, COLUMN, map, false), expectedValues);
// Map with one empty Map, one single-entry List and one single-entry Object[]
map = new HashMap<>();
@@ -155,12 +157,12 @@ public class DataTypeTransformerTest {
map.put("testKey3", new Object[]{"testValue2"});
try {
// Should fail because Map with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, map, true);
+ DataTypeTransformer.standardize(record, COLUMN, map, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(COLUMN, map, false), expectedValues);
+ assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(record, COLUMN, map, false), expectedValues);
// List with two single-entry Maps and one empty Map
list = Arrays
@@ -168,35 +170,35 @@ public class DataTypeTransformerTest {
Collections.emptyMap());
try {
// Should fail because List with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, list, true);
+ DataTypeTransformer.standardize(record, COLUMN, list, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEquals((Object[]) DataTypeTransformer.standardize(COLUMN, list, false), expectedValues);
+ assertEquals((Object[]) DataTypeTransformer.standardize(record, COLUMN, list, false), expectedValues);
// Object[] with two single-entry Maps
values = new Object[]{Collections.singletonMap("testKey", "testValue1"), Collections.singletonMap("testKey",
"testValue2")};
try {
// Should fail because Object[] with multiple entries cannot be standardized as single value
- DataTypeTransformer.standardize(COLUMN, values, true);
+ DataTypeTransformer.standardize(record, COLUMN, values, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(COLUMN, values, false), expectedValues);
+ assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(record, COLUMN, values, false), expectedValues);
// Object[] with one empty Object[], one multi-entries List of nested Map/List/Object[]
values = new Object[]{new Object[0], Collections.singletonList(
Collections.singletonMap("testKey", "testValue1")), Collections.singletonMap("testKey",
Arrays.asList(new Object[]{"testValue2"}, Collections.emptyMap()))};
try {
- DataTypeTransformer.standardize(COLUMN, values, true);
+ DataTypeTransformer.standardize(record, COLUMN, values, true);
fail();
} catch (Exception e) {
// Expected
}
- assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(COLUMN, values, false), expectedValues);
+ assertEqualsNoOrder((Object[]) DataTypeTransformer.standardize(record, COLUMN, values, false), expectedValues);
}
}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
index 65b6003..3710fd0 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
@@ -28,6 +28,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.Utils;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerGauge;
import org.apache.pinot.common.metrics.ServerMeter;
import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.common.utils.CommonConstants;
@@ -154,10 +155,16 @@ public class SegmentFetcherAndLoader {
// Retry will be done here.
String localSegmentDir = downloadSegmentToLocal(uri, crypter, tableNameWithType, segmentName);
- SegmentMetadata segmentMetadata = new SegmentMetadataImpl(new File(localSegmentDir));
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(new File(localSegmentDir));
_instanceDataManager.addOfflineSegment(tableNameWithType, segmentName, new File(localSegmentDir));
LOGGER.info("Downloaded segment {} of table {} crc {} from controller", segmentName, tableNameWithType,
segmentMetadata.getCrc());
+
+ // Emit server metric if the generated values of the segment are converted from map to array.
+ if (segmentMetadata.areValuesConvertedFromMapToArray()) {
+ _serverMetrics
+ .setValueOfTableGauge(tableNameWithType, ServerGauge.SEGMENT_VALUES_CONVERTED_FROM_MAP_TO_ARRAY, 1L);
+ }
} else {
LOGGER.info("Got already loaded segment {} of table {} crc {} again, will do nothing.", segmentName,
tableNameWithType, localSegmentMetadata.getCrc());
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/GenericRow.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/GenericRow.java
index 5c45d6b..8bd0e7c 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/GenericRow.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/GenericRow.java
@@ -61,6 +61,12 @@ public class GenericRow {
*/
public static final String SKIP_RECORD_KEY = "$SKIP_RECORD_KEY$";
+ /**
+ * This key is used to identify whether a map with only 1 entry is converted to a value of a array for multi-value column.
+ * This is necessary for us to identify whether there is any existing use case that is leveraging this way to fetch values.
+ */
+ public static final String CONVERT_MAP_VALUE_TO_ARRAY_VALUE_KEY = "$CONVERT_MAP_VALUE_TO_ARRAY_VALUE_KEY$";
+
private final Map<String, Object> _fieldToValueMap = new HashMap<>();
private final Set<String> _nullValueFields = new HashSet<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org