You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2023/06/20 07:37:48 UTC
[doris] branch master updated: [enhancement](backup) teach fe to acquire a consistent backup between be and fe (#21014)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0d80456869 [enhancement](backup) teach fe to acquire a consistent backup between be and fe (#21014)
0d80456869 is described below
commit 0d80456869b3833a1234da292deaa8e92afea6c7
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Tue Jun 20 15:37:41 2023 +0800
[enhancement](backup) teach fe to acquire a consistent backup between be and fe (#21014)
---
.../java/org/apache/doris/backup/BackupJob.java | 249 ++++++++++-----------
1 file changed, 122 insertions(+), 127 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 60d486c310..61eda11a79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -32,7 +32,6 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.io.Text;
@@ -358,6 +357,9 @@ public class BackupJob extends AbstractJob {
unfinishedTaskIds.clear();
taskProgress.clear();
taskErrMsg.clear();
+ // copy all related schema at this moment
+ List<Table> copiedTables = Lists.newArrayList();
+ List<Resource> copiedResources = Lists.newArrayList();
AgentBatchTask batchTask = new AgentBatchTask();
for (TableRef tableRef : tableRefs) {
String tblName = tableRef.getName().getTbl();
@@ -366,37 +368,34 @@ public class BackupJob extends AbstractJob {
status = new Status(ErrCode.NOT_FOUND, "table " + tblName + " does not exist");
return;
}
- switch (tbl.getType()) {
- case OLAP:
- checkOlapTable((OlapTable) tbl, tableRef);
- if (getContent() == BackupContent.ALL) {
- prepareSnapshotTaskForOlapTable((OlapTable) tbl, tableRef, batchTask);
- }
- break;
- case VIEW:
- break;
- case ODBC:
- OdbcTable odbcTable = (OdbcTable) tbl;
- if (odbcTable.getOdbcCatalogResourceName() != null) {
- String odbcResourceName = odbcTable.getOdbcCatalogResourceName();
- Resource resource = Env.getCurrentEnv().getResourceMgr()
- .getResource(odbcResourceName);
- if (resource == null) {
- status = new Status(ErrCode.NOT_FOUND, "resource " + odbcResourceName
- + " related to " + tblName + "does not exist.");
- return;
+ tbl.readLock();
+ try {
+ switch (tbl.getType()) {
+ case OLAP:
+ OlapTable olapTable = (OlapTable) tbl;
+ checkOlapTable(olapTable, tableRef);
+ if (getContent() == BackupContent.ALL) {
+ prepareSnapshotTaskForOlapTableWithoutLock((OlapTable) tbl, tableRef, batchTask);
}
- }
- break;
- default:
- status = new Status(ErrCode.COMMON_ERROR,
- "backup job does not support this type of table " + tblName);
- return;
+ prepareBackupMetaForOlapTableWithoutLock(tableRef, olapTable, copiedTables);
+ break;
+ case VIEW:
+ prepareBackupMetaForViewWithoutLock((View) tbl, copiedTables);
+ break;
+ case ODBC:
+ prepareBackupMetaForOdbcTableWithoutLock((OdbcTable) tbl, copiedTables, copiedResources);
+ break;
+ default:
+ status = new Status(ErrCode.COMMON_ERROR,
+ "backup job does not support this type of table " + tblName);
+ return;
+ }
+ } finally {
+ tbl.readUnlock();
}
}
- // copy all related schema at this moment
- prepareBackupMeta(db);
+ backupMeta = new BackupMeta(copiedTables, copiedResources);
// send tasks
for (AgentTask task : batchTask.getAllTasks()) {
@@ -430,121 +429,117 @@ public class BackupJob extends AbstractJob {
}
}
- private void prepareSnapshotTaskForOlapTable(OlapTable olapTable,
+ private void prepareSnapshotTaskForOlapTableWithoutLock(OlapTable olapTable,
TableRef backupTableRef, AgentBatchTask batchTask) {
- olapTable.readLock();
- try {
- // check backup table again
- if (backupTableRef.getPartitionNames() != null) {
- for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
- if (partition == null) {
- status = new Status(ErrCode.NOT_FOUND, "partition " + partName
- + " does not exist in table" + backupTableRef.getName().getTbl());
- return;
- }
+ // check backup table again
+ if (backupTableRef.getPartitionNames() != null) {
+ for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) {
+ Partition partition = olapTable.getPartition(partName);
+ if (partition == null) {
+ status = new Status(ErrCode.NOT_FOUND, "partition " + partName
+ + " does not exist in table" + backupTableRef.getName().getTbl());
+ return;
}
}
+ }
- // create snapshot tasks
- List<Partition> partitions = Lists.newArrayList();
- if (backupTableRef.getPartitionNames() == null) {
- partitions.addAll(olapTable.getPartitions());
- } else {
- for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
- partitions.add(partition);
- }
+ // create snapshot tasks
+ List<Partition> partitions = Lists.newArrayList();
+ if (backupTableRef.getPartitionNames() == null) {
+ partitions.addAll(olapTable.getPartitions());
+ } else {
+ for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) {
+ Partition partition = olapTable.getPartition(partName);
+ partitions.add(partition);
}
+ }
- // snapshot partitions
- for (Partition partition : partitions) {
- long visibleVersion = partition.getVisibleVersion();
- List<MaterializedIndex> indexes = partition.getMaterializedIndices(IndexExtState.VISIBLE);
- for (MaterializedIndex index : indexes) {
- int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
- List<Tablet> tablets = index.getTablets();
- for (Tablet tablet : tablets) {
- Replica replica = chooseReplica(tablet, visibleVersion);
- if (replica == null) {
- status = new Status(ErrCode.COMMON_ERROR,
- "failed to choose replica to make snapshot for tablet " + tablet.getId()
- + ". visible version: " + visibleVersion);
- return;
- }
- SnapshotTask task = new SnapshotTask(null, replica.getBackendId(), tablet.getId(),
- jobId, dbId, olapTable.getId(), partition.getId(),
- index.getId(), tablet.getId(),
- visibleVersion,
- schemaHash, timeoutMs, false /* not restore task */);
- batchTask.addTask(task);
- unfinishedTaskIds.put(tablet.getId(), replica.getBackendId());
+ // snapshot partitions
+ for (Partition partition : partitions) {
+ long visibleVersion = partition.getVisibleVersion();
+ List<MaterializedIndex> indexes = partition.getMaterializedIndices(IndexExtState.VISIBLE);
+ for (MaterializedIndex index : indexes) {
+ int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
+ List<Tablet> tablets = index.getTablets();
+ for (Tablet tablet : tablets) {
+ Replica replica = chooseReplica(tablet, visibleVersion);
+ if (replica == null) {
+ status = new Status(ErrCode.COMMON_ERROR,
+ "failed to choose replica to make snapshot for tablet " + tablet.getId()
+ + ". visible version: " + visibleVersion);
+ return;
}
+ SnapshotTask task = new SnapshotTask(null, replica.getBackendId(), tablet.getId(),
+ jobId, dbId, olapTable.getId(), partition.getId(),
+ index.getId(), tablet.getId(),
+ visibleVersion,
+ schemaHash, timeoutMs, false /* not restore task */);
+ batchTask.addTask(task);
+ unfinishedTaskIds.put(tablet.getId(), replica.getBackendId());
}
+ }
- LOG.info("snapshot for partition {}, version: {}",
- partition.getId(), visibleVersion);
+ LOG.info("snapshot for partition {}, version: {}",
+ partition.getId(), visibleVersion);
+ }
+ }
+
+ private void checkResourceForOdbcTable(OdbcTable odbcTable) {
+ if (odbcTable.getOdbcCatalogResourceName() != null) {
+ String odbcResourceName = odbcTable.getOdbcCatalogResourceName();
+ Resource resource = Env.getCurrentEnv().getResourceMgr()
+ .getResource(odbcResourceName);
+ if (resource == null) {
+ status = new Status(ErrCode.NOT_FOUND, "resource " + odbcResourceName
+ + " related to " + odbcTable.getName() + "does not exist.");
+ return;
}
- } finally {
- olapTable.readUnlock();
}
}
- private void prepareBackupMeta(Database db) {
- // copy all related schema at this moment
- List<Table> copiedTables = Lists.newArrayList();
- List<Resource> copiedResources = Lists.newArrayList();
- for (TableRef tableRef : tableRefs) {
- String tblName = tableRef.getName().getTbl();
- Table table = db.getTableNullable(tblName);
- table.readLock();
- try {
- if (table.getType() == TableType.OLAP) {
- OlapTable olapTable = (OlapTable) table;
- // only copy visible indexes
- List<String> reservedPartitions = tableRef.getPartitionNames() == null ? null
- : tableRef.getPartitionNames().getPartitionNames();
- OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions, IndexExtState.VISIBLE, true);
- if (copiedTbl == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to copy table: " + tblName);
- return;
- }
+ private void prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapTable olapTable,
+ List<Table> copiedTables) {
+ // only copy visible indexes
+ List<String> reservedPartitions = tableRef.getPartitionNames() == null ? null
+ : tableRef.getPartitionNames().getPartitionNames();
+ OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions, IndexExtState.VISIBLE, true);
+ if (copiedTbl == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy table: " + olapTable.getName());
+ return;
+ }
- removeUnsupportProperties(copiedTbl);
- copiedTables.add(copiedTbl);
- } else if (table.getType() == TableType.VIEW) {
- View view = (View) table;
- View copiedView = view.clone();
- if (copiedView == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to copy view: " + tblName);
- return;
- }
- copiedTables.add(copiedView);
- } else if (table.getType() == TableType.ODBC) {
- OdbcTable odbcTable = (OdbcTable) table;
- OdbcTable copiedOdbcTable = odbcTable.clone();
- if (copiedOdbcTable == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to copy odbc table: " + tblName);
- return;
- }
- copiedTables.add(copiedOdbcTable);
- if (copiedOdbcTable.getOdbcCatalogResourceName() != null) {
- Resource resource = Env.getCurrentEnv().getResourceMgr()
- .getResource(copiedOdbcTable.getOdbcCatalogResourceName());
- Resource copiedResource = resource.clone();
- if (copiedResource == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to copy odbc resource: "
- + resource.getName());
- return;
- }
- copiedResources.add(copiedResource);
- }
- }
- } finally {
- table.readUnlock();
+ removeUnsupportProperties(copiedTbl);
+ copiedTables.add(copiedTbl);
+ }
+
+ private void prepareBackupMetaForViewWithoutLock(View view, List<Table> copiedTables) {
+ View copiedView = view.clone();
+ if (copiedView == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy view: " + view.getName());
+ return;
+ }
+ copiedTables.add(copiedView);
+ }
+
+ private void prepareBackupMetaForOdbcTableWithoutLock(OdbcTable odbcTable, List<Table> copiedTables,
+ List<Resource> copiedResources) {
+ OdbcTable copiedOdbcTable = odbcTable.clone();
+ if (copiedOdbcTable == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy odbc table: " + odbcTable.getName());
+ return;
+ }
+ copiedTables.add(copiedOdbcTable);
+ if (copiedOdbcTable.getOdbcCatalogResourceName() != null) {
+ Resource resource = Env.getCurrentEnv().getResourceMgr()
+ .getResource(copiedOdbcTable.getOdbcCatalogResourceName());
+ Resource copiedResource = resource.clone();
+ if (copiedResource == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy odbc resource: "
+ + resource.getName());
+ return;
}
+ copiedResources.add(copiedResource);
}
- backupMeta = new BackupMeta(copiedTables, copiedResources);
}
private void removeUnsupportProperties(OlapTable tbl) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org