You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/12/28 11:02:21 UTC
(incubator-paimon) branch master updated: [core] Primary key table only max level can convert to raw files (#2586)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 43eeef5ec [core] Primary key table only max level can convert to raw files (#2586)
43eeef5ec is described below
commit 43eeef5ec9e5db1c1571216805a8bb0a4cb2fe21
Author: shidayang <53...@qq.com>
AuthorDate: Thu Dec 28 19:02:17 2023 +0800
[core] Primary key table only max level can convert to raw files (#2586)
---
.../paimon/table/source/snapshot/SnapshotReaderImpl.java | 11 ++---------
.../paimon/table/source/snapshot/SnapshotReaderTest.java | 13 +------------
2 files changed, 3 insertions(+), 21 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
index 86e6f5d3e..82b68b6e6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
@@ -417,20 +417,13 @@ public class SnapshotReaderImpl implements SnapshotReader {
BinaryRow partition, int bucket, List<DataFileMeta> dataFiles) {
String bucketPath = pathFactory.bucketPath(partition, bucket).toString();
- // bucket with only one file can be returned
- if (dataFiles.size() == 1) {
- return Collections.singletonList(makeRawTableFile(bucketPath, dataFiles.get(0)));
- }
-
// append only files can be returned
if (tableSchema.primaryKeys().isEmpty()) {
return makeRawTableFiles(bucketPath, dataFiles);
}
- // bucket containing only one level (except level 0) can be returned
- Set<Integer> levels =
- dataFiles.stream().map(DataFileMeta::level).collect(Collectors.toSet());
- if (levels.size() == 1 && !levels.contains(0)) {
+ int maxLevel = options.numLevels() - 1;
+ if (dataFiles.stream().map(DataFileMeta::level).allMatch(l -> l == maxLevel)) {
return makeRawTableFiles(bucketPath, dataFiles);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
index f38ce9064..e448d8684 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
@@ -100,18 +100,7 @@ public class SnapshotReaderTest {
assertThat(dataSplit.dataFiles()).hasSize(1);
DataFileMeta meta = dataSplit.dataFiles().get(0);
String partition = dataSplit.partition().getString(0).toString();
- assertThat(dataSplit.convertToRawFiles())
- .hasValue(
- Collections.singletonList(
- new RawFile(
- String.format(
- "%s/pt=%s/bucket-0/%s",
- tablePath, partition, meta.fileName()),
- 0,
- meta.fileSize(),
- "avro",
- meta.schemaId(),
- meta.rowCount())));
+ assertThat(dataSplit.convertToRawFiles()).isNotPresent();
}
// write another file on level 0