You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2023/12/11 06:59:07 UTC
(pinot) branch master updated: Reduce logging volume to log the conversions per record instead of per conversion for SpecialValueTransformer (#12121)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 27cc8a796f Reduce logging volume to log the conversions per record instead of per conversion for SpecialValueTransformer (#12121)
27cc8a796f is described below
commit 27cc8a796f0053dc2b2bd8f18579c60ebbc16c15
Author: aishikbh <ai...@startree.ai>
AuthorDate: Mon Dec 11 12:29:00 2023 +0530
Reduce logging volume to log the conversions per record instead of per conversion for SpecialValueTransformer (#12121)
* reduce logging volume to log the conversions per record.
* fix checkstyle
* changed to print the log only if conversions are made.
---
.../recordtransformer/SpecialValueTransformer.java | 25 +++++++++++++++++-----
.../recordtransformer/RecordTransformerTest.java | 10 ++++-----
2 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SpecialValueTransformer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SpecialValueTransformer.java
index 29596e0935..484cae888a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SpecialValueTransformer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SpecialValueTransformer.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.segment.local.recordtransformer;
+import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -43,6 +44,8 @@ public class SpecialValueTransformer implements RecordTransformer {
private static final Logger LOGGER = LoggerFactory.getLogger(NullValueTransformer.class);
private final HashSet<String> _specialValuesKeySet = new HashSet<>();
+ private int _negativeZeroConversionCount = 0;
+ private int _nanConversionCount = 0;
public SpecialValueTransformer(Schema schema) {
for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
@@ -55,22 +58,22 @@ public class SpecialValueTransformer implements RecordTransformer {
private Object transformNegativeZero(Object value) {
if ((value instanceof Float) && (Float.floatToRawIntBits((float) value) == Float.floatToRawIntBits(-0.0f))) {
- LOGGER.info("-0.0f value detected, converting to 0.0.");
value = 0.0f;
+ _negativeZeroConversionCount++;
} else if ((value instanceof Double) && (Double.doubleToLongBits((double) value) == Double.doubleToLongBits(
-0.0d))) {
- LOGGER.info("-0.0d value detected, converting to 0.0.");
value = 0.0d;
+ _negativeZeroConversionCount++;
}
return value;
}
private Object transformNaN(Object value) {
if ((value instanceof Float) && ((Float) value).isNaN()) {
- LOGGER.info("Float.NaN detected, converting to default null.");
value = null;
+ _nanConversionCount++;
} else if ((value instanceof Double) && ((Double) value).isNaN()) {
- LOGGER.info("Double.NaN detected, converting to default null.");
+ _nanConversionCount++;
value = null;
}
return value;
@@ -90,7 +93,6 @@ public class SpecialValueTransformer implements RecordTransformer {
Object[] values = (Object[]) value;
int numValues = values.length;
List<Object> negativeZeroNanSanitizedValues = new ArrayList<>(numValues);
- int numberOfElements = values.length;
for (Object o : values) {
Object zeroTransformedValue = transformNegativeZero(o);
Object nanTransformedValue = transformNaN(zeroTransformedValue);
@@ -108,6 +110,19 @@ public class SpecialValueTransformer implements RecordTransformer {
}
}
}
+ if (_negativeZeroConversionCount > 0 || _nanConversionCount > 0) {
+ LOGGER.info("Converted {} -0.0s to 0.0 and {} NaNs to null", _negativeZeroConversionCount, _nanConversionCount);
+ }
return record;
}
+
+ @VisibleForTesting
+ int getNegativeZeroConversionCount() {
+ return _negativeZeroConversionCount;
+ }
+
+ @VisibleForTesting
+ int getNanConversionCount() {
+ return _nanConversionCount;
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
index 410fa19564..46a4181a2b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/RecordTransformerTest.java
@@ -277,7 +277,7 @@ public class RecordTransformerTest {
@Test
public void testSpecialValueTransformer() {
- RecordTransformer transformer = new SpecialValueTransformer(SCHEMA);
+ SpecialValueTransformer transformer = new SpecialValueTransformer(SCHEMA);
GenericRow record = getRecord();
for (int i = 0; i < NUM_ROUNDS; i++) {
record = transformer.transform(record);
@@ -290,10 +290,10 @@ public class RecordTransformerTest {
assertEquals(record.getValue("mvDoubleNegativeZero"), new Double[]{0.0d, 1.0d, 0.0d, 3.0d});
assertNull(record.getValue("svFloatNaN"));
assertNull(record.getValue("svDoubleNaN"));
- assertEquals(record.getValue("mvFloatNaN"),
- new Float[]{0.0f, 2.0f});
- assertEquals(record.getValue("mvDoubleNaN"),
- new Double[]{0.0d, 2.0d});
+ assertEquals(record.getValue("mvFloatNaN"), new Float[]{0.0f, 2.0f});
+ assertEquals(record.getValue("mvDoubleNaN"), new Double[]{0.0d, 2.0d});
+ assertEquals(transformer.getNegativeZeroConversionCount(), 6);
+ assertEquals(transformer.getNanConversionCount(), 4);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org