You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/02/26 06:01:21 UTC
(incubator-paimon) branch master updated: [core] Remove legacy stats FieldStatsArraySerializer.fromBinary (#2900)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 98ce2972e [core] Remove legacy stats FieldStatsArraySerializer.fromBinary (#2900)
98ce2972e is described below
commit 98ce2972e583fb99e0361e4df1510d9ae4fbee5c
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Feb 26 14:01:16 2024 +0800
[core] Remove legacy stats FieldStatsArraySerializer.fromBinary (#2900)
---
.../paimon/stats/FieldStatsArraySerializer.java | 54 +--------------------
.../apache/paimon/stats/FieldStatsConverters.java | 7 ---
.../org/apache/paimon/table/system/FilesTable.java | 26 ++++++----
.../paimon/io/KeyValueFileReadWriteTest.java | 16 +++----
.../apache/paimon/manifest/ManifestFileTest.java | 10 ++--
.../stats/FieldStatsArraySerializerTest.java | 55 ++++++++++++----------
.../org/apache/paimon/stats/StatsTestUtils.java | 15 ++++++
7 files changed, 75 insertions(+), 108 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
index 0d830f551..4881fa881 100644
--- a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
@@ -18,7 +18,6 @@
package org.apache.paimon.stats;
-import org.apache.paimon.casting.CastExecutor;
import org.apache.paimon.casting.CastFieldGetter;
import org.apache.paimon.casting.CastedRow;
import org.apache.paimon.data.BinaryArray;
@@ -37,14 +36,12 @@ import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.InternalRowUtils;
import org.apache.paimon.utils.ProjectedRow;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
-import java.util.stream.IntStream;
import static org.apache.paimon.utils.SerializationUtils.newBytesType;
@@ -53,32 +50,20 @@ public class FieldStatsArraySerializer {
private final InternalRowSerializer serializer;
- private final InternalRow.FieldGetter[] fieldGetters;
-
@Nullable private final int[] indexMapping;
- @Nullable private final CastExecutor<Object, Object>[] converterMapping;
@Nullable private final CastFieldGetter[] castFieldGetters;
public FieldStatsArraySerializer(RowType type) {
- this(type, null, null, null);
+ this(type, null, null);
}
public FieldStatsArraySerializer(
RowType type,
@Nullable int[] indexMapping,
- @Nullable CastExecutor<Object, Object>[] converterMapping,
@Nullable CastFieldGetter[] castFieldGetters) {
RowType safeType = toAllFieldsNullableRowType(type);
this.serializer = new InternalRowSerializer(safeType);
- this.fieldGetters =
- IntStream.range(0, safeType.getFieldCount())
- .mapToObj(
- i ->
- InternalRowUtils.createNullCheckingFieldGetter(
- safeType.getTypeAt(i), i))
- .toArray(InternalRow.FieldGetter[]::new);
this.indexMapping = indexMapping;
- this.converterMapping = converterMapping;
this.castFieldGetters = castFieldGetters;
}
@@ -98,43 +83,6 @@ public class FieldStatsArraySerializer {
BinaryArray.fromLongArray(nullCounts));
}
- public FieldStats[] fromBinary(BinaryTableStats array) {
- return fromBinary(array, null);
- }
-
- public FieldStats[] fromBinary(BinaryTableStats array, @Nullable Long rowCount) {
- int fieldCount = indexMapping == null ? fieldGetters.length : indexMapping.length;
- FieldStats[] stats = new FieldStats[fieldCount];
- BinaryArray nullCounts = array.nullCounts();
- for (int i = 0; i < fieldCount; i++) {
- int fieldIndex = indexMapping == null ? i : indexMapping[i];
- if (fieldIndex < 0 || fieldIndex >= array.minValues().getFieldCount()) {
- // simple evolution for add column
- if (rowCount == null) {
- throw new RuntimeException("Schema Evolution for stats needs row count.");
- }
- stats[i] = new FieldStats(null, null, rowCount);
- } else {
- CastExecutor<Object, Object> converter =
- converterMapping == null ? null : converterMapping[i];
- Object min = fieldGetters[fieldIndex].getFieldOrNull(array.minValues());
- min = converter == null || min == null ? min : converter.cast(min);
-
- Object max = fieldGetters[fieldIndex].getFieldOrNull(array.maxValues());
- max = converter == null || max == null ? max : converter.cast(max);
-
- stats[i] =
- new FieldStats(
- min,
- max,
- nullCounts.isNullAt(fieldIndex)
- ? null
- : nullCounts.getLong(fieldIndex));
- }
- }
- return stats;
- }
-
public InternalRow evolution(BinaryRow values) {
InternalRow row = values;
if (indexMapping != null) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
index 9c5f421ac..6733e1fea 100644
--- a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
+++ b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
@@ -18,9 +18,7 @@
package org.apache.paimon.stats;
-import org.apache.paimon.casting.CastExecutor;
import org.apache.paimon.schema.IndexCastMapping;
-import org.apache.paimon.schema.SchemaEvolutionUtil;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
@@ -65,15 +63,10 @@ public class FieldStatsConverters {
IndexCastMapping indexCastMapping =
createIndexCastMapping(schemaTableFields, dataFields);
@Nullable int[] indexMapping = indexCastMapping.getIndexMapping();
- CastExecutor<Object, Object>[] castExecutors =
- (CastExecutor<Object, Object>[])
- SchemaEvolutionUtil.createConvertMapping(
- schemaTableFields, dataFields, indexMapping);
// Create field stats array serializer with schema evolution
return new FieldStatsArraySerializer(
new RowType(dataFields),
indexMapping,
- castExecutors,
indexCastMapping.getCastMapping());
});
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
index 899e7e4e4..a10a8e771 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
@@ -20,10 +20,10 @@ package org.apache.paimon.table.system;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.LazyGenericRow;
import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.predicate.Equal;
@@ -48,9 +48,11 @@ import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.InternalRowUtils;
import org.apache.paimon.utils.IteratorRecordReader;
import org.apache.paimon.utils.ProjectedRow;
import org.apache.paimon.utils.RowDataToObjectArrayConverter;
@@ -449,20 +451,24 @@ public class FilesTable implements ReadonlyTable {
}
private void initialize() {
- FieldStatsArraySerializer fieldStatsArraySerializer =
+ FieldStatsArraySerializer serializer =
fieldStatsConverters.getOrCreate(file.schemaId());
// Create value stats
- FieldStats[] fieldStatsArray =
- fieldStatsArraySerializer.fromBinary(tableStats, file.rowCount());
+ InternalRow min = serializer.evolution(tableStats.minValues());
+ InternalRow max = serializer.evolution(tableStats.maxValues());
+ InternalArray nullCounts =
+ serializer.evolution(tableStats.nullCounts(), file.rowCount());
lazyNullValueCounts = new TreeMap<>();
lazyLowerValueBounds = new TreeMap<>();
lazyUpperValueBounds = new TreeMap<>();
- for (int i = 0; i < fieldStatsArray.length; i++) {
- String fieldName = fieldStatsConverters.tableDataFields().get(i).name();
- FieldStats fieldStats = fieldStatsArray[i];
- lazyNullValueCounts.put(fieldName, fieldStats.nullCount());
- lazyLowerValueBounds.put(fieldName, fieldStats.minValue());
- lazyUpperValueBounds.put(fieldName, fieldStats.maxValue());
+ for (int i = 0; i < min.getFieldCount(); i++) {
+ DataField field = fieldStatsConverters.tableDataFields().get(i);
+ String name = field.name();
+ DataType type = field.type();
+ lazyNullValueCounts.put(
+ name, nullCounts.isNullAt(i) ? null : nullCounts.getLong(i));
+ lazyLowerValueBounds.put(name, InternalRowUtils.get(min, i, type));
+ lazyUpperValueBounds.put(name, InternalRowUtils.get(max, i, type));
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
index aab4a5f98..df6110ce6 100644
--- a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
@@ -25,6 +25,7 @@ import org.apache.paimon.TestKeyValueGenerator;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.format.FieldStats;
import org.apache.paimon.format.FlushingFileFormat;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileIOFinder;
@@ -33,7 +34,6 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.reader.RecordReaderIterator;
-import org.apache.paimon.stats.FieldStatsArraySerializer;
import org.apache.paimon.stats.StatsTestUtils;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataType;
@@ -60,6 +60,7 @@ import java.util.function.Function;
import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_ROW_TYPE;
import static org.apache.paimon.TestKeyValueGenerator.KEY_TYPE;
import static org.apache.paimon.TestKeyValueGenerator.createTestSchemaManager;
+import static org.apache.paimon.stats.StatsTestUtils.convertWithoutSchemaEvolution;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -338,10 +339,6 @@ public class KeyValueFileReadWriteTest {
private void checkRollingFiles(
DataFileMeta expected, List<DataFileMeta> actual, long suggestedFileSize) {
- FieldStatsArraySerializer keyStatsConverter = new FieldStatsArraySerializer(KEY_TYPE);
- FieldStatsArraySerializer valueStatsConverter =
- new FieldStatsArraySerializer(DEFAULT_ROW_TYPE);
-
// all but last file should be no smaller than suggestedFileSize
for (int i = 0; i + 1 < actual.size(); i++) {
assertThat(actual.get(i).fileSize() >= suggestedFileSize).isTrue();
@@ -358,19 +355,20 @@ public class KeyValueFileReadWriteTest {
assertThat(actual.get(actual.size() - 1).maxKey()).isEqualTo(expected.maxKey());
// check stats
+ FieldStats[] keyStats = convertWithoutSchemaEvolution(expected.keyStats(), KEY_TYPE);
for (int i = 0; i < KEY_TYPE.getFieldCount(); i++) {
int idx = i;
StatsTestUtils.checkRollingFileStats(
- keyStatsConverter.fromBinary(expected.keyStats())[i],
+ keyStats[i],
actual,
- m -> keyStatsConverter.fromBinary(m.keyStats())[idx]);
+ m -> convertWithoutSchemaEvolution(m.keyStats(), KEY_TYPE)[idx]);
}
for (int i = 0; i < DEFAULT_ROW_TYPE.getFieldCount(); i++) {
int idx = i;
StatsTestUtils.checkRollingFileStats(
- valueStatsConverter.fromBinary(expected.valueStats())[i],
+ convertWithoutSchemaEvolution(expected.valueStats(), DEFAULT_ROW_TYPE)[i],
actual,
- m -> valueStatsConverter.fromBinary(m.valueStats())[idx]);
+ m -> convertWithoutSchemaEvolution(m.valueStats(), DEFAULT_ROW_TYPE)[idx]);
}
// expected.minSequenceNumber == min(minSequenceNumber)
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 0c0a0ce55..daa3b71a1 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -27,7 +27,6 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.SchemaManager;
-import org.apache.paimon.stats.FieldStatsArraySerializer;
import org.apache.paimon.stats.StatsTestUtils;
import org.apache.paimon.utils.FailingFileIO;
import org.apache.paimon.utils.FileStorePathFactory;
@@ -43,6 +42,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_PART_TYPE;
+import static org.apache.paimon.stats.StatsTestUtils.convertWithoutSchemaEvolution;
import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link ManifestFile}. */
@@ -130,14 +130,16 @@ public class ManifestFileTest {
.isEqualTo(expected.numDeletedFiles());
// check stats
- FieldStatsArraySerializer statsConverter = new FieldStatsArraySerializer(DEFAULT_PART_TYPE);
- FieldStats[] fieldStats = statsConverter.fromBinary(expected.partitionStats());
+ FieldStats[] fieldStats =
+ convertWithoutSchemaEvolution(expected.partitionStats(), DEFAULT_PART_TYPE);
for (int i = 0; i < fieldStats.length; i++) {
int idx = i;
StatsTestUtils.checkRollingFileStats(
fieldStats[i],
actual,
- meta -> statsConverter.fromBinary(meta.partitionStats())[idx]);
+ meta ->
+ convertWithoutSchemaEvolution(meta.partitionStats(), DEFAULT_PART_TYPE)[
+ idx]);
}
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
index 2da96070f..5a587a441 100644
--- a/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
@@ -18,10 +18,10 @@
package org.apache.paimon.stats;
-import org.apache.paimon.casting.CastExecutor;
import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.format.FieldStats;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.schema.IndexCastMapping;
import org.apache.paimon.schema.SchemaEvolutionUtil;
import org.apache.paimon.schema.TableSchema;
@@ -38,6 +38,7 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link FieldStatsArraySerializer}. */
public class FieldStatsArraySerializerTest {
+
@Test
public void testFromBinary() {
TableSchema dataSchema =
@@ -49,9 +50,9 @@ public class FieldStatsArraySerializerTest {
new DataField(2, "c", new IntType()),
new DataField(3, "d", new IntType())),
3,
- Collections.EMPTY_LIST,
- Collections.EMPTY_LIST,
- Collections.EMPTY_MAP,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyMap(),
"");
TableSchema tableSchema =
new TableSchema(
@@ -63,24 +64,19 @@ public class FieldStatsArraySerializerTest {
new DataField(6, "e", new IntType()),
new DataField(7, "b", new IntType())),
7,
- Collections.EMPTY_LIST,
- Collections.EMPTY_LIST,
- Collections.EMPTY_MAP,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyMap(),
"");
IndexCastMapping indexCastMapping =
SchemaEvolutionUtil.createIndexCastMapping(
tableSchema.fields(), dataSchema.fields());
int[] indexMapping = indexCastMapping.getIndexMapping();
- CastExecutor<Object, Object>[] converterMapping =
- (CastExecutor<Object, Object>[])
- SchemaEvolutionUtil.createConvertMapping(
- tableSchema.fields(), dataSchema.fields(), indexMapping);
- FieldStatsArraySerializer fieldStatsArraySerializer =
+ FieldStatsArraySerializer serializer =
new FieldStatsArraySerializer(
tableSchema.logicalRowType(),
indexMapping,
- converterMapping,
indexCastMapping.getCastMapping());
BinaryRow minRowData = row(1, 2, 3, 4);
BinaryRow maxRowData = row(100, 99, 98, 97);
@@ -88,18 +84,27 @@ public class FieldStatsArraySerializerTest {
BinaryTableStats dataTableStats =
new BinaryTableStats(minRowData, maxRowData, BinaryArray.fromLongArray(nullCounts));
- FieldStats[] fieldStatsArray = fieldStatsArraySerializer.fromBinary(dataTableStats, 1000L);
- assertThat(fieldStatsArray.length).isEqualTo(tableSchema.fields().size()).isEqualTo(5);
- checkFieldStats(fieldStatsArray[0], 2, 99, 0L);
- checkFieldStats(fieldStatsArray[1], 4, 97, 100L);
- checkFieldStats(fieldStatsArray[2], null, null, 1000L);
- checkFieldStats(fieldStatsArray[3], null, null, 1000L);
- checkFieldStats(fieldStatsArray[4], null, null, 1000L);
+ InternalRow min = serializer.evolution(dataTableStats.minValues());
+ InternalRow max = serializer.evolution(dataTableStats.maxValues());
+ InternalArray nulls = serializer.evolution(dataTableStats.nullCounts(), 1000L);
+
+ checkFieldStats(min, max, nulls, 0, 2, 99, 0L);
+ checkFieldStats(min, max, nulls, 1, 4, 97, 100L);
+ checkFieldStats(min, max, nulls, 2, null, null, 1000L);
+ checkFieldStats(min, max, nulls, 3, null, null, 1000L);
+ checkFieldStats(min, max, nulls, 4, null, null, 1000L);
}
- private void checkFieldStats(FieldStats fieldStats, Integer min, Integer max, Long nullCount) {
- assertThat(fieldStats.minValue()).isEqualTo(min);
- assertThat(fieldStats.maxValue()).isEqualTo(max);
- assertThat(fieldStats.nullCount()).isEqualTo(nullCount);
+ private void checkFieldStats(
+ InternalRow minRow,
+ InternalRow maxRow,
+ InternalArray nulls,
+ int i,
+ Integer min,
+ Integer max,
+ Long nullCount) {
+ assertThat(minRow.isNullAt(i) ? null : minRow.getInt(i)).isEqualTo(min);
+ assertThat(maxRow.isNullAt(i) ? null : maxRow.getInt(i)).isEqualTo(max);
+ assertThat(nulls.getLong(i)).isEqualTo(nullCount);
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java
index d3ccceb56..cda5aedab 100644
--- a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java
+++ b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java
@@ -18,9 +18,11 @@
package org.apache.paimon.stats;
+import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.RowDataToObjectArrayConverter;
import java.util.ArrayList;
import java.util.List;
@@ -31,6 +33,19 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Utils for stats related tests. */
public class StatsTestUtils {
+ public static FieldStats[] convertWithoutSchemaEvolution(
+ BinaryTableStats stats, RowType rowType) {
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ Object[] mins = converter.convert(stats.minValues());
+ Object[] maxs = converter.convert(stats.maxValues());
+ FieldStats[] ret = new FieldStats[rowType.getFieldCount()];
+ BinaryArray nulls = stats.nullCounts();
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = new FieldStats(mins[i], maxs[i], nulls.isNullAt(i) ? null : nulls.getLong(i));
+ }
+ return ret;
+ }
+
@SuppressWarnings("unchecked")
public static <T> void checkRollingFileStats(
FieldStats expected, List<T> actualObjects, Function<T, FieldStats> mapToStats) {