You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2025/10/16 13:16:27 UTC
(paimon) branch release-1.3 updated (ef998c2b6c -> 05ad2a058a)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
from ef998c2b6c [hotfix][flink] fix the partition error for IncrementalClusterSplitSource (#6416)
new b1cc1e5632 [core] Fix that FilesTable cannot output level0 files when dv enabled (#6414)
new 018d61be14 [oss] add fs.oss.sld.enabled to support oss private link (#6413)
new 05ad2a058a [core] Fix that cannot read binlog table with projection (#6417)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/paimon/table/system/BinlogTable.java | 2 +-
.../org/apache/paimon/table/system/FilesTable.java | 3 +++
.../main/java/org/apache/paimon/oss/OSSFileIO.java | 23 ++++++++++++++++++++++
.../apache/paimon/flink/BatchFileStoreITCase.java | 9 +++++++++
.../apache/paimon/flink/CatalogTableITCase.java | 20 +++++++++++++++++++
5 files changed, 56 insertions(+), 1 deletion(-)
(paimon) 02/03: [oss] add fs.oss.sld.enabled to support oss private link (#6413)
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 018d61be142e8eab0b81156d2c837af2c51c2290
Author: shyjsarah <44...@users.noreply.github.com>
AuthorDate: Thu Oct 16 19:36:17 2025 +0800
[oss] add fs.oss.sld.enabled to support oss private link (#6413)
---
.../main/java/org/apache/paimon/oss/OSSFileIO.java | 23 ++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
index 4c457598e7..d18ddcc94b 100644
--- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
+++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
@@ -22,10 +22,14 @@ import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.ReflectionUtils;
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.common.comm.ServiceClient;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +60,7 @@ public class OSSFileIO extends HadoopCompliantFileIO {
private static final String OSS_ACCESS_KEY_ID = "fs.oss.accessKeyId";
private static final String OSS_ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken";
+ private static final String OSS_SECOND_LEVEL_DOMAIN_ENABLED = "fs.oss.sld.enabled";
private static final Map<String, String> CASE_SENSITIVE_KEYS =
new HashMap<String, String>() {
@@ -137,6 +142,11 @@ public class OSSFileIO extends HadoopCompliantFileIO {
} catch (IOException e) {
throw new UncheckedIOException(e);
}
+
+ if (hadoopOptions.getBoolean(OSS_SECOND_LEVEL_DOMAIN_ENABLED, false)) {
+ enableSecondLevelDomain(fs);
+ }
+
return fs;
};
@@ -156,6 +166,19 @@ public class OSSFileIO extends HadoopCompliantFileIO {
}
}
+ public void enableSecondLevelDomain(AliyunOSSFileSystem fs) {
+ AliyunOSSFileSystemStore store = fs.getStore();
+ try {
+ OSSClient ossClient = ReflectionUtils.getPrivateFieldValue(store, "ossClient");
+ ServiceClient serviceClient =
+ ReflectionUtils.getPrivateFieldValue(ossClient, "serviceClient");
+ serviceClient.getClientConfiguration().setSLDEnabled(true);
+ } catch (Exception e) {
+ LOG.error("Failed to enable second level domain.", e);
+ throw new RuntimeException("Failed to enable second level domain.", e);
+ }
+ }
+
private static class CacheKey {
private final Options options;
(paimon) 03/03: [core] Fix that cannot read binlog table with projection (#6417)
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 05ad2a058aa12364069aa420c865ab9a9943ebd1
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Thu Oct 16 21:13:17 2025 +0800
[core] Fix that cannot read binlog table with projection (#6417)
---
.../org/apache/paimon/table/system/BinlogTable.java | 2 +-
.../org/apache/paimon/flink/CatalogTableITCase.java | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java
index 8488e3ff9b..cc8d1621a3 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java
@@ -131,7 +131,7 @@ public class BinlogTable extends AuditLogTable {
(row1, row2) ->
new AuditLogRow(
readProjection, convertToArray(row1, row2, fieldGetters)),
- wrapped.rowType());
+ this.wrappedReadType);
} else {
return dataRead.createReader(split)
.transform(
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
index bcdf1e2ef6..3d7429177a 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
@@ -1172,6 +1172,26 @@ public class CatalogTableITCase extends CatalogITCaseBase {
iterator.close();
}
+ @Test
+ public void testBinlogTableStreamReadWithProjection() throws Exception {
+ sql(
+ "CREATE TABLE T (a INT, b INT, primary key (a) NOT ENFORCED) with ('changelog-producer' = 'lookup', "
+ + "'bucket' = '2')");
+ BlockingIterator<Row, Row> iterator =
+ streamSqlBlockIter(
+ "SELECT rowkind, a FROM T$binlog /*+ OPTIONS('scan.mode' = 'latest') */");
+ sql("INSERT INTO T VALUES (1, 2)");
+ sql("INSERT INTO T VALUES (1, 3)");
+ sql("INSERT INTO T VALUES (2, 2)");
+ List<Row> rows = iterator.collect(3);
+ assertThat(rows)
+ .containsExactly(
+ Row.of("+I", new Integer[] {1}),
+ Row.of("+U", new Integer[] {1, 1}),
+ Row.of("+I", new Integer[] {2}));
+ iterator.close();
+ }
+
@Test
public void testBinlogTableBatchRead() throws Exception {
sql(
(paimon) 01/03: [core] Fix that FilesTable cannot output level0 files when dv enabled (#6414)
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit b1cc1e5632bbdadec23bc455a3a2209bac840c62
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Thu Oct 16 19:20:41 2025 +0800
[core] Fix that FilesTable cannot output level0 files when dv enabled (#6414)
---
.../src/main/java/org/apache/paimon/table/system/FilesTable.java | 3 +++
.../test/java/org/apache/paimon/flink/BatchFileStoreITCase.java | 9 +++++++++
2 files changed, 12 insertions(+)
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
index eb18e679f5..caa8315eb9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
@@ -274,6 +274,9 @@ public class FilesTable implements ReadonlyTable {
return levelPredicate.test(
GenericRow.of(null, null, null, null, null, level));
});
+ } else {
+ // avoid that batchScanSkipLevel0 is true
+ scan.withLevelFilter(level -> true);
}
return scan.plan();
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
index 13425014f5..30b2e6d0fd 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
@@ -1016,4 +1016,13 @@ public class BatchFileStoreITCase extends CatalogITCaseBase {
"SELECT * FROM T /*+ OPTIONS('scan.dedicated-split-generation'='true') */"))
.hasSize(0);
}
+
+ @Test
+ public void testLevel0FileCanBeReadForFilesTable() {
+ sql(
+ "CREATE TABLE test_table (a int PRIMARY KEY NOT ENFORCED, b string) "
+ + "WITH ('deletion-vectors.enabled' = 'true', 'write-only' = 'true');");
+ sql("INSERT INTO test_table VALUES (1, 'A')");
+ assertThat(sql("SELECT * FROM `test_table$files`")).isNotEmpty();
+ }
}