You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/10/21 13:08:24 UTC
[doris] branch branch-1.1-lts updated: [branch-1.1-lts](restore) Add new property 'reserve_dynamic_partition_enable' to restore statement (#12498) (#13559)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 3d8e75039b [branch-1.1-lts](restore) Add new property 'reserve_dynamic_partition_enable' to restore statement (#12498) (#13559)
3d8e75039b is described below
commit 3d8e75039b8d4c2896b0b0bf15aa1ea6f6a7bb3c
Author: xueweizhang <zx...@163.com>
AuthorDate: Fri Oct 21 21:08:17 2022 +0800
[branch-1.1-lts](restore) Add new property 'reserve_dynamic_partition_enable' to restore statement (#12498) (#13559)
Add restore new property 'reserve_dynamic_partition_enable', which means you can
get a table with dynamic_partition_enable property which has the same value
as before the backup. before this commit, you always get a table with property
'dynamic_partition_enable=false' when restore.
---
.../org/apache/doris/analysis/RestoreStmt.java | 19 +++++++++++++++
.../org/apache/doris/analysis/ShowRestoreStmt.java | 6 ++---
.../org/apache/doris/backup/BackupHandler.java | 3 ++-
.../java/org/apache/doris/backup/RestoreJob.java | 27 +++++++++++++++++-----
.../java/org/apache/doris/catalog/OlapTable.java | 4 ++--
.../org/apache/doris/catalog/TableProperty.java | 6 +++--
.../org/apache/doris/backup/RestoreJobTest.java | 2 +-
.../org/apache/doris/catalog/OlapTableTest.java | 4 ++--
8 files changed, 54 insertions(+), 17 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
index 199a212420..bfffa54ca3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
@@ -37,12 +37,14 @@ public class RestoreStmt extends AbstractBackupStmt {
private final static String PROP_BACKUP_TIMESTAMP = "backup_timestamp";
private final static String PROP_META_VERSION = "meta_version";
private final static String PROP_RESERVE_REPLICA = "reserve_replica";
+ private final static String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE = "reserve_dynamic_partition_enable";
private boolean allowLoad = false;
private ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
private String backupTimestamp = null;
private int metaVersion = -1;
private boolean reserveReplica = false;
+ private boolean reserveDynamicPartitionEnable = false;
public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause restoreTableRefClause,
Map<String, String> properties) {
@@ -69,6 +71,10 @@ public class RestoreStmt extends AbstractBackupStmt {
return reserveReplica;
}
+ public boolean reserveDynamicPartitionEnable() {
+ return reserveDynamicPartitionEnable;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
@@ -124,6 +130,19 @@ public class RestoreStmt extends AbstractBackupStmt {
}
copiedProperties.remove(PROP_RESERVE_REPLICA);
}
+ // reserve dynamic partition enable
+ if (copiedProperties.containsKey(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE)) {
+ if (copiedProperties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE).equalsIgnoreCase("true")) {
+ reserveDynamicPartitionEnable = true;
+ } else if (copiedProperties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE).equalsIgnoreCase("false")) {
+ reserveDynamicPartitionEnable = false;
+ } else {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid reserve dynamic partition enable value: "
+ + copiedProperties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE));
+ }
+ copiedProperties.remove(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE);
+ }
// backup timestamp
if (copiedProperties.containsKey(PROP_BACKUP_TIMESTAMP)) {
backupTimestamp = copiedProperties.get(PROP_BACKUP_TIMESTAMP);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
index 2b05aa34c0..8e47266f26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
@@ -40,9 +40,9 @@ public class ShowRestoreStmt extends ShowStmt {
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("JobId").add("Label").add("Timestamp").add("DbName").add("State")
.add("AllowLoad").add("ReplicationNum").add("ReplicaAllocation").add("ReserveReplica")
- .add("RestoreObjs").add("CreateTime").add("MetaPreparedTime").add("SnapshotFinishedTime")
- .add("DownloadFinishedTime").add("FinishedTime").add("UnfinishedTasks").add("Progress")
- .add("TaskErrMsg").add("Status").add("Timeout")
+ .add("ReserveDynamicPartitionEnable").add("RestoreObjs").add("CreateTime").add("MetaPreparedTime")
+ .add("SnapshotFinishedTime").add("DownloadFinishedTime").add("FinishedTime").add("UnfinishedTasks")
+ .add("Progress").add("TaskErrMsg").add("Status").add("Timeout")
.build();
private String dbName;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 0887c4f287..099535af94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -400,7 +400,8 @@ public class BackupHandler extends MasterDaemon implements Writable {
// Create a restore job
RestoreJob restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(),
db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(),
- stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), catalog, repository.getId());
+ stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveDynamicPartitionEnable(),
+ catalog, repository.getId());
catalog.getEditLog().logRestoreJob(restoreJob);
// must put to dbIdToBackupOrRestoreJob after edit log, otherwise the state of job may be changed.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 9fd97a23ff..fd3194c08f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -50,6 +50,7 @@ import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
+import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
@@ -57,6 +58,7 @@ import org.apache.doris.common.MarkedCountDownLatch;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
+import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.resource.Tag;
import org.apache.doris.task.AgentBatchTask;
@@ -99,6 +101,7 @@ import java.util.stream.Collectors;
public class RestoreJob extends AbstractJob {
private static final String PROP_RESERVE_REPLICA = "reserve_replica";
+ private static final String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE = "reserve_dynamic_partition_enable";
private static final Logger LOG = LogManager.getLogger(RestoreJob.class);
@@ -134,6 +137,7 @@ public class RestoreJob extends AbstractJob {
private ReplicaAllocation replicaAlloc;
private boolean reserveReplica = false;
+ private boolean reserveDynamicPartitionEnable = false;
// this 2 members is to save all newly restored objs
// tbl name -> part
@@ -165,8 +169,8 @@ public class RestoreJob extends AbstractJob {
}
public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad,
- ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica, Catalog catalog,
- long repoId) {
+ ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica,
+ boolean reserveDynamicPartitionEnable, Catalog catalog, long repoId) {
super(JobType.RESTORE, label, dbId, dbName, timeoutMs, catalog, repoId);
this.backupTimestamp = backupTs;
this.jobInfo = jobInfo;
@@ -175,7 +179,9 @@ public class RestoreJob extends AbstractJob {
this.state = RestoreJobState.PENDING;
this.metaVersion = metaVersion;
this.reserveReplica = reserveReplica;
+ this.reserveDynamicPartitionEnable = reserveDynamicPartitionEnable;
properties.put(PROP_RESERVE_REPLICA, String.valueOf(reserveReplica));
+ properties.put(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE, String.valueOf(reserveDynamicPartitionEnable));
}
public RestoreJobState getState() {
@@ -660,7 +666,7 @@ public class RestoreJob extends AbstractJob {
}
// Reset properties to correct values.
- remoteOlapTbl.resetPropertiesForRestore();
+ remoteOlapTbl.resetPropertiesForRestore(reserveDynamicPartitionEnable);
// DO NOT set remote table's new name here, cause we will still need the origin name later
// remoteOlapTbl.setName(jobInfo.getAliasByOriginNameIfSet(tblInfo.name));
@@ -1446,7 +1452,7 @@ public class RestoreJob extends AbstractJob {
// set all restored partition version and version hash
// set all tables' state to NORMAL
- setTableStateToNormal(db);
+ setTableStateToNormal(db, true);
for (long tblId : restoredVersionInfo.rowKeySet()) {
Table tbl = db.getTableNullable(tblId);
if (tbl == null) {
@@ -1539,6 +1545,7 @@ public class RestoreJob extends AbstractJob {
info.add(String.valueOf(replicaAlloc.getTotalReplicaNum()));
info.add(replicaAlloc.toCreateStmt());
info.add(String.valueOf(reserveReplica));
+ info.add(String.valueOf(reserveDynamicPartitionEnable));
info.add(getRestoreObjs());
info.add(TimeUtils.longToTimeString(createTime));
info.add(TimeUtils.longToTimeString(metaPreparedTime));
@@ -1613,7 +1620,7 @@ public class RestoreJob extends AbstractJob {
Database db = catalog.getDbNullable(dbId);
if (db != null) {
// rollback table's state to NORMAL
- setTableStateToNormal(db);
+ setTableStateToNormal(db, false);
// remove restored tbls
for (Table restoreTbl : restoredTbls) {
@@ -1690,7 +1697,7 @@ public class RestoreJob extends AbstractJob {
LOG.info("finished to cancel restore job. is replay: {}. {}", isReplay, this);
}
- private void setTableStateToNormal(Database db) {
+ private void setTableStateToNormal(Database db, boolean committed) {
for (String tableName : jobInfo.backupOlapTableObjects.keySet()) {
Table tbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName));
if (tbl == null) {
@@ -1722,6 +1729,13 @@ public class RestoreJob extends AbstractJob {
partition.setState(PartitionState.NORMAL);
}
}
+ if (committed && reserveDynamicPartitionEnable) {
+ if (DynamicPartitionUtil.isDynamicPartitionTable(tbl)) {
+ DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTbl, false);
+ catalog.getDynamicPartitionScheduler().createOrUpdateRuntimeInfo(tbl.getId(),
+ DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime());
+ }
+ }
} finally {
tbl.writeUnlock();
}
@@ -1883,6 +1897,7 @@ public class RestoreJob extends AbstractJob {
properties.put(key, value);
}
reserveReplica = Boolean.parseBoolean(properties.get(PROP_RESERVE_REPLICA));
+ reserveDynamicPartitionEnable = Boolean.parseBoolean(properties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 455af83688..60d8c33373 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -424,9 +424,9 @@ public class OlapTable extends Table {
/**
* Reset properties to correct values.
*/
- public void resetPropertiesForRestore() {
+ public void resetPropertiesForRestore(boolean reserveDynamicPartitionEnable) {
if (tableProperty != null) {
- tableProperty.resetPropertiesForRestore();
+ tableProperty.resetPropertiesForRestore(reserveDynamicPartitionEnable);
}
// remove colocate property.
setColocateGroup(null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 91ef9b0c32..8672ccb868 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -107,10 +107,12 @@ public class TableProperty implements Writable {
* Reset properties to correct values.
* @return this for chained
*/
- public TableProperty resetPropertiesForRestore() {
+ public TableProperty resetPropertiesForRestore(boolean reserveDynamicPartitionEnable) {
// disable dynamic partition
if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
- properties.put(DynamicPartitionProperty.ENABLE, "false");
+ if (!reserveDynamicPartitionEnable) {
+ properties.put(DynamicPartitionProperty.ENABLE, "false");
+ }
executeBuildDynamicProperty();
}
return this;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index 31a92178fd..4444297e64 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -234,7 +234,7 @@ public class RestoreJobTest {
db.dropTable(expectedRestoreTbl.getName());
job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), jobInfo, false,
- new ReplicaAllocation((short) 3), 100000, -1, false, catalog, repo.getId());
+ new ReplicaAllocation((short) 3), 100000, -1, false, false, catalog, repo.getId());
List<Table> tbls = Lists.newArrayList();
List<Resource> resources = Lists.newArrayList();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 2c57fe054d..4a7b355f4e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -91,7 +91,7 @@ public class OlapTableTest {
olapTable.setColocateGroup("test_group");
Assert.assertTrue(olapTable.isColocateTable());
- olapTable.resetPropertiesForRestore();
+ olapTable.resetPropertiesForRestore(false);
Assert.assertEquals(tableProperty.getProperties(), olapTable.getTableProperty().getProperties());
Assert.assertFalse(tableProperty.getDynamicPartitionProperty().isExist());
Assert.assertFalse(olapTable.isColocateTable());
@@ -110,7 +110,7 @@ public class OlapTableTest {
tableProperty = new TableProperty(properties);
olapTable.setTableProperty(tableProperty);
- olapTable.resetPropertiesForRestore();
+ olapTable.resetPropertiesForRestore(false);
Map<String, String> expectedProperties = Maps.newHashMap(properties);
expectedProperties.put(DynamicPartitionProperty.ENABLE, "false");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org