You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2022/04/25 11:32:49 UTC
[incubator-doris] branch master updated: keep at least one validated image file (#9192)
This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 687421b43f keep at least one validated image file (#9192)
687421b43f is described below
commit 687421b43f0c22588b56929df375ef77af43349f
Author: dataroaring <98...@users.noreply.github.com>
AuthorDate: Mon Apr 25 19:32:43 2022 +0800
keep at least one validated image file (#9192)
* rename ImageSeq to LatestImageSeq in Storage
* keep at least one validated image file
---
.../java/org/apache/doris/catalog/Catalog.java | 4 +-
.../doris/httpv2/controller/HaController.java | 2 +-
.../org/apache/doris/httpv2/meta/MetaService.java | 2 +-
.../org/apache/doris/httpv2/rest/ShowAction.java | 2 +-
.../java/org/apache/doris/master/Checkpoint.java | 8 ++--
.../java/org/apache/doris/persist/MetaCleaner.java | 2 +-
.../java/org/apache/doris/persist/Storage.java | 55 ++++++++++++----------
.../java/org/apache/doris/persist/StorageTest.java | 8 ++--
8 files changed, 46 insertions(+), 37 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 86d6763b24..09a06a1596 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -1531,7 +1531,7 @@ public class Catalog {
private void getNewImage(Pair<String, Integer> helperNode) throws IOException {
long localImageVersion = 0;
Storage storage = new Storage(this.imageDir);
- localImageVersion = storage.getImageSeq();
+ localImageVersion = storage.getLatestImageSeq();
try {
URL infoUrl = new URL("http://" + helperNode.first + ":" + Config.http_port + "/info");
@@ -1617,7 +1617,7 @@ public class Catalog {
LOG.info("image does not exist: {}", curFile.getAbsolutePath());
return;
}
- replayedJournalId.set(storage.getImageSeq());
+ replayedJournalId.set(storage.getLatestImageSeq());
MetaReader.read(curFile, this);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java
index cc983ad6e8..515c933e86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java
@@ -134,7 +134,7 @@ public class HaController {
Map<String, Object> checkPoint = new HashMap<>();
Storage storage = new Storage(Config.meta_dir + "/image");
checkPoint.put("Name", "Version");
- checkPoint.put("Value", storage.getImageSeq());
+ checkPoint.put("Value", storage.getLatestImageSeq());
list.add(checkPoint);
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
Date date = new Date(lastCheckpointTime);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
index 22796efb42..48d0e4720f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
@@ -111,7 +111,7 @@ public class MetaService extends RestBaseController {
try {
Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath());
StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(),
- currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq());
+ currentStorageInfo.getLatestImageSeq(), currentStorageInfo.getEditsSeq());
return ResponseEntityBuilder.ok(storageInfo);
} catch (IOException e) {
return ResponseEntityBuilder.internalError(e.getMessage());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
index e08ebe5fd7..e3543db693 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
@@ -242,7 +242,7 @@ public class ShowAction extends RestBaseController {
feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady()));
Storage storage = new Storage(Config.meta_dir + "/image");
- feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq()));
+ feInfo.put("last_checkpoint_version", String.valueOf(storage.getLatestImageSeq()));
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
index 69efd618f8..7865817a82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
@@ -83,7 +83,7 @@ public class Checkpoint extends MasterDaemon {
try {
storage = new Storage(imageDir);
// get max image version
- imageVersion = storage.getImageSeq();
+ imageVersion = storage.getLatestImageSeq();
// get max finalized journal id
checkPointVersion = editLog.getFinalizedJournalId();
LOG.info("last checkpoint journal id: {}, current finalized journal id: {}", imageVersion, checkPointVersion);
@@ -184,7 +184,9 @@ public class Checkpoint extends MasterDaemon {
if (successPushed == otherNodesCount) {
try {
long minOtherNodesJournalId = Long.MAX_VALUE;
- long deleteVersion = checkPointVersion;
+ // Actually, storage.getLatestValidatedImageSeq returns number before this
+ // checkpoint.
+ long deleteVersion = storage.getLatestValidatedImageSeq();
if (successPushed > 0) {
for (Frontend fe : allFrontends) {
String host = fe.getHost();
@@ -220,7 +222,7 @@ public class Checkpoint extends MasterDaemon {
}
}
}
- deleteVersion = Math.min(minOtherNodesJournalId, checkPointVersion);
+ deleteVersion = Math.min(minOtherNodesJournalId, deleteVersion);
}
editLog.deleteJournals(deleteVersion + 1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java
index 2c2b17a941..2a7acd1f09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java
@@ -34,7 +34,7 @@ public class MetaCleaner {
public void clean() throws IOException {
Storage storage = new Storage(imageDir);
- long currentVersion = storage.getImageSeq();
+ long currentVersion = storage.getLatestValidatedImageSeq();
long imageDeleteVersion = currentVersion - 1;
File currentImage = storage.getImageFile(currentVersion);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/Storage.java b/fe/fe-core/src/main/java/org/apache/doris/persist/Storage.java
index 999687cdcf..8eb0336d71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/Storage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/Storage.java
@@ -60,7 +60,8 @@ public class Storage {
private FrontendNodeType role = FrontendNodeType.UNKNOWN;
private String nodeName;
private long editsSeq;
- private long imageSeq;
+ private long latestImageSeq;
+ private long latestValidatedImageSeq;
private String metaDir;
private List<Long> editsFileSequenceNumbers;
@@ -70,11 +71,11 @@ public class Storage {
this.metaDir = metaDir;
}
- public Storage(int clusterID, String token, long imageSeq, long editsSeq, String metaDir) {
+ public Storage(int clusterID, String token, long latestImageSeq, long editsSeq, String metaDir) {
this.clusterID = clusterID;
this.token = token;
this.editsSeq = editsSeq;
- this.imageSeq = imageSeq;
+ this.latestImageSeq = latestImageSeq;
this.metaDir = metaDir;
}
@@ -114,31 +115,33 @@ public class Storage {
nodeName = prop.getProperty(NODE_NAME, null);
}
- // Find the latest image
+ // Find the latest two images
File dir = new File(metaDir);
File[] children = dir.listFiles();
if (children == null) {
return;
- } else {
- for (File child : children) {
- String name = child.getName();
- try {
- if (!name.equals(EDITS) && !name.equals(IMAGE_NEW)
- && !name.endsWith(".part") && name.contains(".")) {
- if (name.startsWith(IMAGE)) {
- imageSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), imageSeq);
- } else if (name.startsWith(EDITS)) {
- // Just record the sequence part of the file name
- editsFileSequenceNumbers.add(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)));
- editsSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), editsSeq);
+ }
+ for (File child : children) {
+ String name = child.getName();
+ try {
+ if (!name.equals(EDITS) && !name.equals(IMAGE_NEW)
+ && !name.endsWith(".part") && name.contains(".")) {
+ if (name.startsWith(IMAGE)) {
+ long fileSeq = Long.parseLong(name.substring(name.lastIndexOf('.') + 1));
+ if (latestImageSeq < fileSeq) {
+ latestValidatedImageSeq = latestImageSeq;
+ latestImageSeq = fileSeq;
}
+ } else if (name.startsWith(EDITS)) {
+ // Just record the sequence part of the file name
+ editsFileSequenceNumbers.add(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)));
+ editsSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), editsSeq);
}
- } catch (Exception e) {
- LOG.warn(name + " is not a validate meta file, ignore it");
}
+ } catch (Exception e) {
+ LOG.warn(name + " is not a validate meta file, ignore it");
}
}
-
}
public int getClusterID() {
@@ -173,12 +176,16 @@ public class Storage {
this.metaDir = metaDir;
}
- public long getImageSeq() {
- return imageSeq;
+ public long getLatestImageSeq() {
+ return latestImageSeq;
+ }
+
+ public long getLatestValidatedImageSeq() {
+ return latestValidatedImageSeq;
}
- public void setImageSeq(long imageSeq) {
- this.imageSeq = imageSeq;
+ public void setLatestImageSeq(long latestImageSeq) {
+ this.latestImageSeq = latestImageSeq;
}
public void setEditsSeq(long editsSeq) {
@@ -272,7 +279,7 @@ public class Storage {
}
public File getCurrentImageFile() {
- return getImageFile(imageSeq);
+ return getImageFile(latestImageSeq);
}
public File getImageFile(long version) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/StorageTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/StorageTest.java
index 1c2237a794..ca17a0006c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/StorageTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/StorageTest.java
@@ -102,7 +102,7 @@ public class StorageTest {
Storage storage2 = new Storage(1, "token", 2, 3, "test");
Assert.assertEquals(1, storage2.getClusterID());
- Assert.assertEquals(2, storage2.getImageSeq());
+ Assert.assertEquals(2, storage2.getLatestImageSeq());
Assert.assertEquals(3, storage2.getEditsSeq());
Assert.assertEquals("test", storage2.getMetaDir());
}
@@ -116,7 +116,7 @@ public class StorageTest {
Assert.assertEquals(966271669, storage.getClusterID());
storage.setClusterID(1234);
Assert.assertEquals(1234, storage.getClusterID());
- Assert.assertEquals(0, storage.getImageSeq());
+ Assert.assertEquals(0, storage.getLatestImageSeq());
Assert.assertEquals(10, Storage.getMetaSeq(new File("storageTestDir/edits.10")));
Assert.assertTrue(Storage.getCurrentEditsFile(new File("storageTestDir"))
.equals(new File("storageTestDir/edits")));
@@ -133,8 +133,8 @@ public class StorageTest {
Assert.assertTrue(storage.getVersionFile().equals(new File("storageTestDir/VERSION")));
- storage.setImageSeq(100);
- Assert.assertEquals(100, storage.getImageSeq());
+ storage.setLatestImageSeq(100);
+ Assert.assertEquals(100, storage.getLatestImageSeq());
storage.setEditsSeq(100);
Assert.assertEquals(100, storage.getEditsSeq());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org