You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/11 16:55:27 UTC
[doris] 10/47: [bugfix](DDL) Fix the bug of incorrect partition policy setting (#25021)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 74e5ecd3193eb1faff7bcf150f244930a4d94a6e
Author: AlexYue <yj...@gmail.com>
AuthorDate: Mon Oct 9 11:28:48 2023 +0800
[bugfix](DDL) Fix the bug of incorrect partition policy setting (#25021)
---
.../apache/doris/datasource/InternalCatalog.java | 15 ++++++---
.../java/org/apache/doris/alter/AlterTest.java | 37 ++++++++++++++++++++++
2 files changed, 47 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 308fe182997..65c8f885604 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2498,10 +2498,13 @@ public class InternalCatalog implements CatalogIf<Database> {
"Can not create UNIQUE KEY table that enables Merge-On-write"
+ " with storage policy(" + partionStoragePolicy + ")");
}
- if (!partionStoragePolicy.equals("")) {
- storagePolicy = partionStoragePolicy;
+ // The table's storage policy has higher priority than partition's policy,
+ // so we'll directly use table's policy when it's set. Otherwise we use the
+ // partition's policy
+ if (!storagePolicy.isEmpty()) {
+ partionStoragePolicy = storagePolicy;
}
- Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy);
+ Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(partionStoragePolicy);
Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(),
olapTable.getId(), olapTable.getName(), olapTable.getBaseIndexId(), entry.getValue(),
@@ -2509,8 +2512,8 @@ public class InternalCatalog implements CatalogIf<Database> {
dataProperty.getStorageMedium(), partitionInfo.getReplicaAllocation(entry.getValue()),
versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory,
storageFormat, partitionInfo.getTabletType(entry.getValue()), compressionType,
- olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy,
- idGeneratorBuffer, olapTable.disableAutoCompaction(),
+ olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(),
+ partionStoragePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(),
olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad,
olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
olapTable.getTimeSeriesCompactionFileCountThreshold(),
@@ -2518,6 +2521,8 @@ public class InternalCatalog implements CatalogIf<Database> {
storeRowColumn, isDynamicSchema, binlogConfigForTask,
dataProperty.isStorageMediumSpecified());
olapTable.addPartition(partition);
+ olapTable.getPartitionInfo().getDataProperty(partition.getId())
+ .setStoragePolicy(partionStoragePolicy);
}
} else {
throw new DdlException("Unsupported partition method: " + partitionInfo.getType().name());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
index a3ddd9991a8..924fcd53b7d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
@@ -195,6 +195,14 @@ public class AlterTest {
"CREATE STORAGE POLICY testPolicyAnotherResource\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3_1\",\n"
+ " \"cooldown_ttl\" = \"1\"\n" + ");");
+ createTable("CREATE TABLE test.tbl_remote1\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ + " PARTITION p2 values less than('2020-03-01') ('storage_policy' = 'testPolicy'),\n"
+ + " PARTITION p3 values less than('2020-04-01'),\n"
+ + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n"
+ + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00'\n" + ");");
+
createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01'),\n"
@@ -685,6 +693,35 @@ public class AlterTest {
}
+ @Test
+ public void testAlterRemoteStorageTableDataPropertiesPolicy() throws Exception {
+ Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test");
+ OlapTable tblRemote = (OlapTable) db.getTableOrMetaException("tbl_remote1");
+ Partition p1 = tblRemote.getPartition("p1");
+ Partition p2 = tblRemote.getPartition("p2");
+ Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(p2.getId()), "testPolicy");
+ Partition p3 = tblRemote.getPartition("p3");
+ Partition p4 = tblRemote.getPartition("p4");
+
+ DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME);
+ long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
+ DataProperty oldDataPropertyWithPolicy = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy");
+ List<Partition> partitionList = Lists.newArrayList(p1, p3, p4);
+ for (Partition partition : partitionList) {
+ Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), "");
+ }
+ Assert.assertEquals(oldDataPropertyWithPolicy, tblRemote.getPartitionInfo().getDataProperty(p2.getId()));
+
+ // alter recover to old state
+ String stmt = "alter table test.tbl_remote1 modify partition (p1, p3, p4) set ("
+ + "'storage_policy' = 'testPolicy')";
+ alterTable(stmt, false);
+ for (Partition partition : partitionList) {
+ Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), "testPolicy");
+ }
+
+ }
+
@Test
public void testDynamicPartitionDropAndAdd() throws Exception {
// test day range
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org