You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2019/02/23 01:47:41 UTC
[incubator-pinot] branch master updated: Disable realtime aggregate
metrics if multi value columns present (#3868)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f70935e Disable realtime aggregate metrics if multi value columns present (#3868)
f70935e is described below
commit f70935e4c2562b075c98035107a7bc00a2b5498d
Author: Neha Pawar <np...@linkedin.com>
AuthorDate: Fri Feb 22 17:47:37 2019 -0800
Disable realtime aggregate metrics if multi value columns present (#3868)
---
.../realtime/LLRealtimeSegmentDataManager.java | 2 +-
.../core/indexsegment/mutable/MutableSegmentImpl.java | 19 +++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index 8e29e21..18b1409 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -447,7 +447,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
canTakeMore = _realtimeSegment.index(transformedRow);
} catch (Exception e) {
- segmentLogger.debug("Caught exception while transforming the record: {}", decodedRow, e);
+ segmentLogger.error("Caught exception while transforming the record: {}", decodedRow, e);
_numRowsErrored++;
}
} else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
index c975702..1b554e8 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
@@ -333,6 +333,8 @@ public class MutableSegmentImpl implements MutableSegment {
(FixedByteSingleColumnSingleValueReaderWriter) _indexReaderWriterMap.get(column);
Preconditions.checkState(_dictionaryMap.get(column) == null, "Updating metrics not supported with dictionary.");
FieldSpec.DataType dataType = metricSpec.getDataType();
+
+ // FIXME: this breaks for multi value metrics. https://github.com/apache/incubator-pinot/issues/3867
switch (dataType) {
case INT:
indexReaderWriter.setInt(docId, (Integer) value + indexReaderWriter.getInt(docId));
@@ -668,6 +670,7 @@ public class MutableSegmentImpl implements MutableSegment {
int i = 0;
int[] dictIds = new int[_numKeyColumns]; // dimensions + time column.
+ // FIXME: this for loop breaks for multi value dimensions. https://github.com/apache/incubator-pinot/issues/3867
for (String column : _schema.getDimensionNames()) {
dictIds[i++] = (Integer) dictIdMap.get(column);
}
@@ -705,6 +708,7 @@ public class MutableSegmentImpl implements MutableSegment {
}
// All metric columns should have no-dictionary index.
+ // All metric columns must be single value
for (String metric : schema.getMetricNames()) {
if (!noDictionaryColumns.contains(metric)) {
_logger
@@ -712,9 +716,17 @@ public class MutableSegmentImpl implements MutableSegment {
_aggregateMetrics = false;
break;
}
+ // https://github.com/apache/incubator-pinot/issues/3867
+ if (!schema.getMetricSpec(metric).isSingleValueField()) {
+ _logger
+ .warn("Metrics aggregation cannot be turned ON in presence of multi-value metric columns, eg: {}", metric);
+ _aggregateMetrics = false;
+ break;
+ }
}
// All dimension columns should be dictionary encoded.
+ // All dimension columns must be single value
for (String dimension : schema.getDimensionNames()) {
if (noDictionaryColumns.contains(dimension)) {
_logger
@@ -722,6 +734,13 @@ public class MutableSegmentImpl implements MutableSegment {
_aggregateMetrics = false;
break;
}
+ // https://github.com/apache/incubator-pinot/issues/3867
+ if (!schema.getDimensionSpec(dimension).isSingleValueField()) {
+ _logger
+ .warn("Metrics aggregation cannot be turned ON in presence of multi-value dimension columns, eg: {}", dimension);
+ _aggregateMetrics = false;
+ break;
+ }
}
// Time column should be dictionary encoded.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org