You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/09/07 03:54:26 UTC
[incubator-doris] branch master updated: [Bug] [Truncate] Fix
tablet number is different between the new partition and base partition
after truncate operation (#6552)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 80cb22c [Bug] [Truncate] Fix tablet number is different between the new partition and base partition after truncate operation (#6552)
80cb22c is described below
commit 80cb22c880394a4c7d5b3ada4251c6b2e49a297b
Author: weizuo93 <we...@apache.org>
AuthorDate: Tue Sep 7 11:54:15 2021 +0800
[Bug] [Truncate] Fix tablet number is different between the new partition and base partition after truncate operation (#6552)
Fix #6551
`Truncate` operation will create new partition to replace base partition.
Tablets in new partition would be created based on table default bucket number.
If the number of tablet for a partition is different from table default bucket number,
the number of tablet will be different between the new created partition and base partition after truncate operation.
In any case, the new partition should have the same number of tablet as the base partition after truncate operation.
Tablets in new partition should be created based on tablet number of base partition rather than table default bucket number for truncate operation.
---
.../java/org/apache/doris/catalog/Catalog.java | 5 +-
.../apache/doris/catalog/TruncateTableTest.java | 123 +++++++++++++++++++++
2 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index c6ecb27..2e83e05 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -6577,6 +6577,7 @@ public class Catalog {
// check, and save some info which need to be checked again later
Map<String, Long> origPartitions = Maps.newHashMap();
+ Map<Long, DistributionInfo> partitionsDistributionInfo = Maps.newHashMap();
OlapTable copiedTbl;
boolean truncateEntireTable = tblRef.getPartitionNames() == null;
@@ -6597,10 +6598,12 @@ public class Catalog {
throw new DdlException("Partition " + partName + " does not exist");
}
origPartitions.put(partName, partition.getId());
+ partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo());
}
} else {
for (Partition partition : olapTable.getPartitions()) {
origPartitions.put(partition.getName(), partition.getId());
+ partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo());
}
}
copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), true, IndexExtState.VISIBLE);
@@ -6625,7 +6628,7 @@ public class Catalog {
db.getId(), copiedTbl.getId(), copiedTbl.getBaseIndexId(),
newPartitionId, entry.getKey(),
copiedTbl.getIndexIdToMeta(),
- copiedTbl.getDefaultDistributionInfo(),
+ partitionsDistributionInfo.get(oldPartitionId),
copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).getStorageMedium(),
copiedTbl.getPartitionInfo().getReplicaAllocation(oldPartitionId),
null /* version info */,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
new file mode 100644
index 0000000..bb1c09d
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
@@ -0,0 +1,123 @@
+package org.apache.doris.catalog;
+
+import org.apache.doris.analysis.*;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowExecutor;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.utframe.UtFrameUtils;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.List;
+import java.util.UUID;
+
+public class TruncateTableTest {
+ private static String runningDir = "fe/mocked/TruncateTableTest/" + UUID.randomUUID().toString() + "/";
+
+ private static ConnectContext connectContext;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ UtFrameUtils.createMinDorisCluster(runningDir);
+ connectContext = UtFrameUtils.createDefaultCtx();
+ // create database
+ String createDbStmtStr = "create database test;";
+ String createTableStr = "create table test.tbl(d1 date, k1 int, k2 bigint)" +
+ "duplicate key(d1, k1) " +
+ "PARTITION BY RANGE(d1)" +
+ "(PARTITION p20210901 VALUES [('2021-09-01'), ('2021-09-02')))" +
+ "distributed by hash(k1) buckets 2 " +
+ "properties('replication_num' = '1');";
+ createDb(createDbStmtStr);
+ createTable(createTableStr);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ File file = new File(runningDir);
+ file.delete();
+ }
+
+ @Test
+ public void testTruncateTable() throws Exception {
+ String stmtStr = "ALTER TABLE test.tbl ADD PARTITION p20210902 VALUES [('2021-09-02'), ('2021-09-03')) DISTRIBUTED BY HASH(`k1`) BUCKETS 3;";
+ alterTable(stmtStr);
+ stmtStr = "ALTER TABLE test.tbl ADD PARTITION p20210903 VALUES [('2021-09-03'), ('2021-09-04')) DISTRIBUTED BY HASH(`k1`) BUCKETS 4;";
+ alterTable(stmtStr);
+ stmtStr = "ALTER TABLE test.tbl ADD PARTITION p20210904 VALUES [('2021-09-04'), ('2021-09-05')) DISTRIBUTED BY HASH(`k1`) BUCKETS 5;";
+ alterTable(stmtStr);
+ checkShowTabletResultNum("test.tbl", "p20210901", 2);
+ checkShowTabletResultNum("test.tbl", "p20210902", 3);
+ checkShowTabletResultNum("test.tbl", "p20210903", 4);
+ checkShowTabletResultNum("test.tbl", "p20210904", 5);
+
+ String truncateStr = "truncate table test.tbl;";
+ TruncateTableStmt truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
+ Catalog.getCurrentCatalog().truncateTable(truncateTableStmt);
+ checkShowTabletResultNum("test.tbl", "p20210901", 2);
+ checkShowTabletResultNum("test.tbl", "p20210902", 3);
+ checkShowTabletResultNum("test.tbl", "p20210903", 4);
+ checkShowTabletResultNum("test.tbl", "p20210904", 5);
+
+ truncateStr = "truncate table test.tbl partition(p20210901, p20210902, p20210903, p20210904);";
+ truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
+ Catalog.getCurrentCatalog().truncateTable(truncateTableStmt);
+ checkShowTabletResultNum("test.tbl", "p20210901", 2);
+ checkShowTabletResultNum("test.tbl", "p20210902", 3);
+ checkShowTabletResultNum("test.tbl", "p20210903", 4);
+ checkShowTabletResultNum("test.tbl", "p20210904", 5);
+
+ truncateStr = "truncate table test.tbl partition (p20210901);";
+ truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
+ Catalog.getCurrentCatalog().truncateTable(truncateTableStmt);
+ checkShowTabletResultNum("test.tbl", "p20210901", 2);
+
+ truncateStr = "truncate table test.tbl partition (p20210902);";
+ truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
+ Catalog.getCurrentCatalog().truncateTable(truncateTableStmt);
+ checkShowTabletResultNum("test.tbl", "p20210902", 3);
+
+ truncateStr = "truncate table test.tbl partition (p20210903);";
+ truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
+ Catalog.getCurrentCatalog().truncateTable(truncateTableStmt);
+ checkShowTabletResultNum("test.tbl", "p20210903", 4);
+
+ truncateStr = "truncate table test.tbl partition (p20210904);";
+ truncateTableStmt = (TruncateTableStmt)UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
+ Catalog.getCurrentCatalog().truncateTable(truncateTableStmt);
+ checkShowTabletResultNum("test.tbl", "p20210904", 5);
+ }
+
+ private static void createDb(String sql) throws Exception {
+ CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
+ }
+
+ private static void createTable(String sql) throws Exception {
+ CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
+ }
+
+ private List<List<String>> checkShowTabletResultNum(String tbl, String partition, int expected) throws Exception {
+ String showStr = "show tablet from " + tbl + " partition(" + partition + ")";
+ ShowTabletStmt showStmt = (ShowTabletStmt) UtFrameUtils.parseAndAnalyzeStmt(showStr, connectContext);
+ ShowExecutor executor = new ShowExecutor(connectContext, (ShowStmt) showStmt);
+ ShowResultSet showResultSet = executor.execute();
+ List<List<String>> rows = showResultSet.getResultRows();
+ Assert.assertEquals(expected, rows.size());
+ return rows;
+ }
+
+ private void alterTable(String sql) throws Exception {
+ try {
+ AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(alterTableStmt);
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org