You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/10/17 05:32:41 UTC
(paimon) branch master updated: [core] delete created_from_snapshot and created_from_snapshot from branchTable (#4159)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e1419ef3b [core] delete created_from_snapshot and created_from_snapshot from branchTable (#4159)
e1419ef3b is described below
commit e1419ef3b05f913827b1db4f161a7c540b9185c5
Author: herefree <84...@qq.com>
AuthorDate: Thu Oct 17 13:32:35 2024 +0800
[core] delete created_from_snapshot and created_from_snapshot from branchTable (#4159)
---
docs/content/maintenance/system-tables.md | 12 +++---
.../apache/paimon/table/system/BranchesTable.java | 48 +---------------------
.../ProcedurePositionalArgumentsITCase.java | 8 ++--
.../org/apache/paimon/flink/BranchSqlITCase.java | 39 ++----------------
4 files changed, 16 insertions(+), 91 deletions(-)
diff --git a/docs/content/maintenance/system-tables.md b/docs/content/maintenance/system-tables.md
index a94031e48..78c2b878a 100644
--- a/docs/content/maintenance/system-tables.md
+++ b/docs/content/maintenance/system-tables.md
@@ -216,12 +216,12 @@ You can query the branches of the table.
SELECT * FROM my_table$branches;
/*
-+----------------------+---------------------------+--------------------------+-------------------------+
-| branch_name | created_from_tag | created_from_snapshot | create_time |
-+----------------------+---------------------------+--------------------------+-------------------------+
-| branch1 | tag1 | 2 | 2024-07-18 20:31:39.084 |
-| branch2 | tag2 | 5 | 2024-07-18 21:11:14.373 |
-+----------------------+---------------------------+--------------------------+-------------------------+
++----------------------+-------------------------+
+| branch_name | create_time |
++----------------------+-------------------------+
+| branch1 | 2024-07-18 20:31:39.084 |
+| branch2 | 2024-07-18 21:11:14.373 |
++----------------------+-------------------------+
2 rows in set
*/
```
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java
index a055db6d5..f523f20e9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java
@@ -18,7 +18,6 @@
package org.apache.paimon.table.system;
-import org.apache.paimon.Snapshot;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
@@ -28,8 +27,6 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
-import org.apache.paimon.schema.SchemaManager;
-import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.FileStoreTableFactory;
import org.apache.paimon.table.ReadonlyTable;
@@ -40,7 +37,6 @@ import org.apache.paimon.table.source.ReadOnceTableScan;
import org.apache.paimon.table.source.SingletonSplit;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
-import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.TimestampType;
@@ -62,15 +58,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Optional;
-import java.util.SortedMap;
import java.util.stream.Collectors;
import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER;
import static org.apache.paimon.utils.BranchManager.BRANCH_PREFIX;
-import static org.apache.paimon.utils.BranchManager.branchPath;
import static org.apache.paimon.utils.FileUtils.listVersionedDirectories;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
/** A {@link Table} for showing branches of table. */
public class BranchesTable implements ReadonlyTable {
@@ -84,10 +76,7 @@ public class BranchesTable implements ReadonlyTable {
Arrays.asList(
new DataField(
0, "branch_name", SerializationUtils.newStringType(false)),
- new DataField(
- 1, "created_from_tag", SerializationUtils.newStringType(true)),
- new DataField(2, "created_from_snapshot", new BigIntType(true)),
- new DataField(3, "create_time", new TimestampType(false, 3))));
+ new DataField(1, "create_time", new TimestampType(false, 3))));
private final FileIO fileIO;
private final Path location;
@@ -113,7 +102,7 @@ public class BranchesTable implements ReadonlyTable {
@Override
public List<String> primaryKeys() {
- return Arrays.asList("branch_name", "tag_name");
+ return Collections.singletonList("branch_name");
}
@Override
@@ -227,7 +216,6 @@ public class BranchesTable implements ReadonlyTable {
private List<InternalRow> branches(FileStoreTable table) throws IOException {
BranchManager branchManager = table.branchManager();
- SchemaManager schemaManager = new SchemaManager(fileIO, table.location());
List<Pair<Path, Long>> paths =
listVersionedDirectories(fileIO, branchManager.branchDirectory(), BRANCH_PREFIX)
@@ -237,42 +225,10 @@ public class BranchesTable implements ReadonlyTable {
for (Pair<Path, Long> path : paths) {
String branchName = path.getLeft().getName().substring(BRANCH_PREFIX.length());
- String basedTag = null;
- Long basedSnapshotId = null;
long creationTime = path.getRight();
-
- Optional<TableSchema> tableSchema =
- schemaManager.copyWithBranch(branchName).latest();
- if (tableSchema.isPresent()) {
- FileStoreTable branchTable =
- FileStoreTableFactory.create(
- fileIO, new Path(branchPath(table.location(), branchName)));
- SortedMap<Snapshot, List<String>> snapshotTags =
- branchTable.tagManager().tags();
- Long earliestSnapshotId = branchTable.snapshotManager().earliestSnapshotId();
- if (snapshotTags.isEmpty()) {
- // create based on snapshotId
- basedSnapshotId = earliestSnapshotId;
- } else {
- Snapshot snapshot = snapshotTags.firstKey();
- if (Objects.equals(earliestSnapshotId, snapshot.id())) {
- // create based on tag
- List<String> tags = snapshotTags.get(snapshot);
- checkArgument(tags.size() == 1);
- basedTag = tags.get(0);
- basedSnapshotId = snapshot.id();
- } else {
- // create based on snapshotId
- basedSnapshotId = earliestSnapshotId;
- }
- }
- }
-
result.add(
GenericRow.of(
BinaryString.fromString(branchName),
- BinaryString.fromString(basedTag),
- basedSnapshotId,
Timestamp.fromLocalDateTime(
DateTimeUtils.toLocalDateTime(creationTime))));
}
diff --git a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java
index 70f2a2f7e..bb461dab6 100644
--- a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java
+++ b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java
@@ -320,13 +320,13 @@ public class ProcedurePositionalArgumentsITCase extends CatalogITCaseBase {
sql("CALL sys.create_branch('default.T', 'test', 'tag1')");
sql("CALL sys.create_branch('default.T', 'test2', 'tag2')");
- assertThat(collectToString("SELECT branch_name, created_from_snapshot FROM `T$branches`"))
- .containsExactlyInAnyOrder("+I[test, 1]", "+I[test2, 2]");
+ assertThat(collectToString("SELECT branch_name FROM `T$branches`"))
+ .containsExactlyInAnyOrder("+I[test]", "+I[test2]");
sql("CALL sys.delete_branch('default.T', 'test')");
- assertThat(collectToString("SELECT branch_name, created_from_snapshot FROM `T$branches`"))
- .containsExactlyInAnyOrder("+I[test2, 2]");
+ assertThat(collectToString("SELECT branch_name FROM `T$branches`"))
+ .containsExactlyInAnyOrder("+I[test2]");
sql("CALL sys.fast_forward('default.T', 'test2')");
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
index 6970eb043..6cf82131f 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
@@ -194,44 +194,13 @@ public class BranchSqlITCase extends CatalogITCaseBase {
sql("CALL sys.create_branch('default.T', 'test', 'tag1')");
sql("CALL sys.create_branch('default.T', 'test2', 'tag2')");
- assertThat(collectResult("SELECT branch_name, created_from_snapshot FROM `T$branches`"))
- .containsExactlyInAnyOrder("+I[test, 1]", "+I[test2, 2]");
+ assertThat(collectResult("SELECT branch_name FROM `T$branches`"))
+ .containsExactlyInAnyOrder("+I[test]", "+I[test2]");
sql("CALL sys.delete_branch('default.T', 'test')");
- assertThat(collectResult("SELECT branch_name, created_from_snapshot FROM `T$branches`"))
- .containsExactlyInAnyOrder("+I[test2, 2]");
- }
-
- @Test
- public void testBranchManagerGetBranchSnapshotsList() throws Exception {
- sql(
- "CREATE TABLE T ("
- + " pt INT"
- + ", k INT"
- + ", v STRING"
- + ", PRIMARY KEY (pt, k) NOT ENFORCED"
- + " ) PARTITIONED BY (pt) WITH ("
- + " 'bucket' = '2'"
- + " )");
-
- sql("INSERT INTO T VALUES (1, 10, 'hxh')");
- sql("INSERT INTO T VALUES (1, 20, 'hxh')");
- sql("INSERT INTO T VALUES (1, 30, 'hxh')");
-
- FileStoreTable table = paimonTable("T");
- checkSnapshots(table.snapshotManager(), 1, 3);
-
- sql("CALL sys.create_tag('default.T', 'tag1', 1)");
- sql("CALL sys.create_tag('default.T', 'tag2', 2)");
- sql("CALL sys.create_tag('default.T', 'tag3', 3)");
-
- sql("CALL sys.create_branch('default.T', 'test1', 'tag1')");
- sql("CALL sys.create_branch('default.T', 'test2', 'tag2')");
- sql("CALL sys.create_branch('default.T', 'test3', 'tag3')");
-
- assertThat(collectResult("SELECT created_from_snapshot FROM `T$branches`"))
- .containsExactlyInAnyOrder("+I[1]", "+I[2]", "+I[3]");
+ assertThat(collectResult("SELECT branch_name FROM `T$branches`"))
+ .containsExactlyInAnyOrder("+I[test2]");
}
@Test