You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2022/10/13 03:16:23 UTC
[doris] branch master updated: [feature](restore) Add new property 'reserve_dynamic_partition_enable' to restore statement (#12498)
This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e08ba8d573 [feature](restore) Add new property 'reserve_dynamic_partition_enable' to restore statement (#12498)
e08ba8d573 is described below
commit e08ba8d5732bba73aa821d8c7fa1c44e072ab253
Author: xueweizhang <zx...@163.com>
AuthorDate: Thu Oct 13 11:16:15 2022 +0800
[feature](restore) Add new property 'reserve_dynamic_partition_enable' to restore statement (#12498)
Add restore new property 'reserve_dynamic_partition_enable', which means you can
get a table with dynamic_partition_enable property which has the same value
as before the backup. before this commit, you always get a table with property
'dynamic_partition_enable=false' when restore.
---
.../Backup-and-Restore/RESTORE.md | 1 +
.../Backup-and-Restore/RESTORE.md | 1 +
.../org/apache/doris/analysis/RestoreStmt.java | 19 +++++++++++++++
.../org/apache/doris/analysis/ShowRestoreStmt.java | 6 ++---
.../org/apache/doris/backup/BackupHandler.java | 3 ++-
.../java/org/apache/doris/backup/RestoreJob.java | 27 +++++++++++++++++-----
.../java/org/apache/doris/catalog/OlapTable.java | 4 ++--
.../org/apache/doris/catalog/TableProperty.java | 6 +++--
.../org/apache/doris/backup/RestoreJobTest.java | 2 +-
.../org/apache/doris/catalog/OlapTableTest.java | 4 ++--
10 files changed, 56 insertions(+), 17 deletions(-)
diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
index 5ed19eb5d7..d176816a3c 100644
--- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
@@ -58,6 +58,7 @@ illustrate:
- "backup_timestamp" = "2018-05-04-16-45-08": Specifies which time version of the corresponding backup to restore, required. This information can be obtained with the `SHOW SNAPSHOT ON repo;` statement.
- "replication_num" = "3": Specifies the number of replicas for the restored table or partition. Default is 3. If restoring an existing table or partition, the number of replicas must be the same as the number of replicas of the existing table or partition. At the same time, there must be enough hosts to accommodate multiple replicas.
- "reserve_replica" = "true": Default is false. When this property is true, the replication_num property is ignored and the restored table or partition will have the same number of replication as before the backup. Supports multiple tables or multiple partitions within a table with different replication number.
+ - "reserve_dynamic_partition_enable" = "true": Default is false. When this property is true, the restored table will have the same value of 'dynamic_partition_enable' as before the backup. if this property is not true, the restored table will set 'dynamic_partition_enable=false'.
- "timeout" = "3600": The task timeout period, the default is one day. in seconds.
- "meta_version" = 40: Use the specified meta_version to read the previously backed up metadata. Note that this parameter is used as a temporary solution and is only used to restore the data backed up by the old version of Doris. The latest version of the backup data already contains the meta version, no need to specify it.
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
index 2f912c6b92..4461c47505 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
@@ -57,6 +57,7 @@ PROPERTIES ("key"="value", ...);
- "backup_timestamp" = "2018-05-04-16-45-08"：指定了恢复对应备份的哪个时间版本，必填。该信息可以通过 `SHOW SNAPSHOT ON repo;` 语句获得。
- "replication_num" = "3"：指定恢复的表或分区的副本数。默认为3。若恢复已存在的表或分区，则副本数必须和已存在表或分区的副本数相同。同时，必须有足够的 host 容纳多个副本。
- "reserve_replica" = "true"：默认为 false。当该属性为 true 时，会忽略 replication_num 属性，恢复的表或分区的副本数将与备份之前一样。支持多个表或表内多个分区有不同的副本数。
+ - "reserve_dynamic_partition_enable" = "true"：默认为 false。当该属性为 true 时，恢复的表会保留该表备份之前的'dynamic_partition_enable'属性值。该值不为true时，则恢复出来的表的'dynamic_partition_enable'属性值会设置为false。
- "timeout" = "3600"：任务超时时间，默认为一天。单位秒。
- "meta_version" = 40：使用指定的 meta_version 来读取之前备份的元数据。注意，该参数作为临时方案，仅用于恢复老版本 Doris 备份的数据。最新版本的备份数据中已经包含 meta version，无需再指定。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
index a98e6f2891..679ebd8cb8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
@@ -37,12 +37,14 @@ public class RestoreStmt extends AbstractBackupStmt {
private static final String PROP_BACKUP_TIMESTAMP = "backup_timestamp";
private static final String PROP_META_VERSION = "meta_version";
private static final String PROP_RESERVE_REPLICA = "reserve_replica";
+ private static final String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE = "reserve_dynamic_partition_enable";
private boolean allowLoad = false;
private ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
private String backupTimestamp = null;
private int metaVersion = -1;
private boolean reserveReplica = false;
+ private boolean reserveDynamicPartitionEnable = false;
public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause restoreTableRefClause,
Map<String, String> properties) {
@@ -69,6 +71,10 @@ public class RestoreStmt extends AbstractBackupStmt {
return reserveReplica;
}
+ public boolean reserveDynamicPartitionEnable() {
+ return reserveDynamicPartitionEnable;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
@@ -124,6 +130,19 @@ public class RestoreStmt extends AbstractBackupStmt {
}
copiedProperties.remove(PROP_RESERVE_REPLICA);
}
+ // reserve dynamic partition enable
+ if (copiedProperties.containsKey(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE)) {
+ if (copiedProperties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE).equalsIgnoreCase("true")) {
+ reserveDynamicPartitionEnable = true;
+ } else if (copiedProperties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE).equalsIgnoreCase("false")) {
+ reserveDynamicPartitionEnable = false;
+ } else {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid reserve dynamic partition enable value: "
+ + copiedProperties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE));
+ }
+ copiedProperties.remove(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE);
+ }
// backup timestamp
if (copiedProperties.containsKey(PROP_BACKUP_TIMESTAMP)) {
backupTimestamp = copiedProperties.get(PROP_BACKUP_TIMESTAMP);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
index 43f703fb75..d8f7c86713 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
@@ -40,9 +40,9 @@ public class ShowRestoreStmt extends ShowStmt {
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("JobId").add("Label").add("Timestamp").add("DbName").add("State")
.add("AllowLoad").add("ReplicationNum").add("ReplicaAllocation").add("ReserveReplica")
- .add("RestoreObjs").add("CreateTime").add("MetaPreparedTime").add("SnapshotFinishedTime")
- .add("DownloadFinishedTime").add("FinishedTime").add("UnfinishedTasks").add("Progress")
- .add("TaskErrMsg").add("Status").add("Timeout")
+ .add("ReserveDynamicPartitionEnable").add("RestoreObjs").add("CreateTime").add("MetaPreparedTime")
+ .add("SnapshotFinishedTime").add("DownloadFinishedTime").add("FinishedTime").add("UnfinishedTasks")
+ .add("Progress").add("TaskErrMsg").add("Status").add("Timeout")
.build();
private String dbName;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index fd1e8efdd7..72eb59392b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -401,7 +401,8 @@ public class BackupHandler extends MasterDaemon implements Writable {
// Create a restore job
RestoreJob restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(),
db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(),
- stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), env, repository.getId());
+ stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveDynamicPartitionEnable(),
+ env, repository.getId());
env.getEditLog().logRestoreJob(restoreJob);
// must put to dbIdToBackupOrRestoreJob after edit log, otherwise the state of job may be changed.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index ceeaeebfce..2b0f828053 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -50,6 +50,7 @@ import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
+import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
@@ -57,6 +58,7 @@ import org.apache.doris.common.MarkedCountDownLatch;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
+import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.resource.Tag;
import org.apache.doris.task.AgentBatchTask;
@@ -98,6 +100,7 @@ import java.util.stream.Collectors;
public class RestoreJob extends AbstractJob {
private static final String PROP_RESERVE_REPLICA = "reserve_replica";
+ private static final String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE = "reserve_dynamic_partition_enable";
private static final Logger LOG = LogManager.getLogger(RestoreJob.class);
@@ -135,6 +138,7 @@ public class RestoreJob extends AbstractJob {
private ReplicaAllocation replicaAlloc;
private boolean reserveReplica = false;
+ private boolean reserveDynamicPartitionEnable = false;
// this 2 members is to save all newly restored objs
// tbl name -> part
@@ -166,8 +170,8 @@ public class RestoreJob extends AbstractJob {
}
public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad,
- ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica, Env env,
- long repoId) {
+ ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica,
+ boolean reserveDynamicPartitionEnable, Env env, long repoId) {
super(JobType.RESTORE, label, dbId, dbName, timeoutMs, env, repoId);
this.backupTimestamp = backupTs;
this.jobInfo = jobInfo;
@@ -176,7 +180,9 @@ public class RestoreJob extends AbstractJob {
this.state = RestoreJobState.PENDING;
this.metaVersion = metaVersion;
this.reserveReplica = reserveReplica;
+ this.reserveDynamicPartitionEnable = reserveDynamicPartitionEnable;
properties.put(PROP_RESERVE_REPLICA, String.valueOf(reserveReplica));
+ properties.put(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE, String.valueOf(reserveDynamicPartitionEnable));
}
public RestoreJobState getState() {
@@ -666,7 +672,7 @@ public class RestoreJob extends AbstractJob {
}
// Reset properties to correct values.
- remoteOlapTbl.resetPropertiesForRestore();
+ remoteOlapTbl.resetPropertiesForRestore(reserveDynamicPartitionEnable);
// DO NOT set remote table's new name here, cause we will still need the origin name later
// remoteOlapTbl.setName(jobInfo.getAliasByOriginNameIfSet(tblInfo.name));
@@ -1453,7 +1459,7 @@ public class RestoreJob extends AbstractJob {
// set all restored partition version and version hash
// set all tables' state to NORMAL
- setTableStateToNormal(db);
+ setTableStateToNormal(db, true);
for (long tblId : restoredVersionInfo.rowKeySet()) {
Table tbl = db.getTableNullable(tblId);
if (tbl == null) {
@@ -1546,6 +1552,7 @@ public class RestoreJob extends AbstractJob {
info.add(String.valueOf(replicaAlloc.getTotalReplicaNum()));
info.add(replicaAlloc.toCreateStmt());
info.add(String.valueOf(reserveReplica));
+ info.add(String.valueOf(reserveDynamicPartitionEnable));
info.add(getRestoreObjs());
info.add(TimeUtils.longToTimeString(createTime));
info.add(TimeUtils.longToTimeString(metaPreparedTime));
@@ -1620,7 +1627,7 @@ public class RestoreJob extends AbstractJob {
Database db = env.getInternalCatalog().getDbNullable(dbId);
if (db != null) {
// rollback table's state to NORMAL
- setTableStateToNormal(db);
+ setTableStateToNormal(db, false);
// remove restored tbls
for (Table restoreTbl : restoredTbls) {
@@ -1697,7 +1704,7 @@ public class RestoreJob extends AbstractJob {
LOG.info("finished to cancel restore job. is replay: {}. {}", isReplay, this);
}
- private void setTableStateToNormal(Database db) {
+ private void setTableStateToNormal(Database db, boolean committed) {
for (String tableName : jobInfo.backupOlapTableObjects.keySet()) {
Table tbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName));
if (tbl == null) {
@@ -1729,6 +1736,13 @@ public class RestoreJob extends AbstractJob {
partition.setState(PartitionState.NORMAL);
}
}
+ if (committed && reserveDynamicPartitionEnable) {
+ if (DynamicPartitionUtil.isDynamicPartitionTable(tbl)) {
+ DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTbl, false);
+ Env.getCurrentEnv().getDynamicPartitionScheduler().createOrUpdateRuntimeInfo(tbl.getId(),
+ DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime());
+ }
+ }
} finally {
tbl.writeUnlock();
}
@@ -1891,6 +1905,7 @@ public class RestoreJob extends AbstractJob {
properties.put(key, value);
}
reserveReplica = Boolean.parseBoolean(properties.get(PROP_RESERVE_REPLICA));
+ reserveDynamicPartitionEnable = Boolean.parseBoolean(properties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 61a835ee41..faf009a1dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -425,9 +425,9 @@ public class OlapTable extends Table {
/**
* Reset properties to correct values.
*/
- public void resetPropertiesForRestore() {
+ public void resetPropertiesForRestore(boolean reserveDynamicPartitionEnable) {
if (tableProperty != null) {
- tableProperty.resetPropertiesForRestore();
+ tableProperty.resetPropertiesForRestore(reserveDynamicPartitionEnable);
}
// remove colocate property.
setColocateGroup(null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index c57d85a90b..2883cf6917 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -118,10 +118,12 @@ public class TableProperty implements Writable {
*
* @return this for chained
*/
- public TableProperty resetPropertiesForRestore() {
+ public TableProperty resetPropertiesForRestore(boolean reserveDynamicPartitionEnable) {
// disable dynamic partition
if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
- properties.put(DynamicPartitionProperty.ENABLE, "false");
+ if (!reserveDynamicPartitionEnable) {
+ properties.put(DynamicPartitionProperty.ENABLE, "false");
+ }
executeBuildDynamicProperty();
}
return this;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index 81930d5c73..0667076da0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -242,7 +242,7 @@ public class RestoreJobTest {
db.dropTable(expectedRestoreTbl.getName());
job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), jobInfo, false,
- new ReplicaAllocation((short) 3), 100000, -1, false, env, repo.getId());
+ new ReplicaAllocation((short) 3), 100000, -1, false, false, env, repo.getId());
List<Table> tbls = Lists.newArrayList();
List<Resource> resources = Lists.newArrayList();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index ce1c3dae36..c5de47f6e1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -90,7 +90,7 @@ public class OlapTableTest {
olapTable.setColocateGroup("test_group");
Assert.assertTrue(olapTable.isColocateTable());
- olapTable.resetPropertiesForRestore();
+ olapTable.resetPropertiesForRestore(false);
Assert.assertEquals(tableProperty.getProperties(), olapTable.getTableProperty().getProperties());
Assert.assertFalse(tableProperty.getDynamicPartitionProperty().isExist());
Assert.assertFalse(olapTable.isColocateTable());
@@ -109,7 +109,7 @@ public class OlapTableTest {
tableProperty = new TableProperty(properties);
olapTable.setTableProperty(tableProperty);
- olapTable.resetPropertiesForRestore();
+ olapTable.resetPropertiesForRestore(false);
Map<String, String> expectedProperties = Maps.newHashMap(properties);
expectedProperties.put(DynamicPartitionProperty.ENABLE, "false");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org