You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2020/06/23 09:42:22 UTC
[incubator-iotdb] branch iotdb-726 created (now a324944)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a change to branch iotdb-726
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at a324944 fix error that the line number error after restart the server while mlog.txt exists but snapshot file does not exist
This branch includes the following new commits:
new a324944 fix error that the line number error after restart the server while mlog.txt exists but snapshot file does not exist
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: fix error that the line number error after
restart the server while mlog.txt exists but snapshot file does not exist
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch iotdb-726
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit a32494404225a864bb47944776278aac4876afe0
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Tue Jun 23 17:41:58 2020 +0800
fix error that the line number error after restart the server while mlog.txt exists but snapshot file does not exist
---
.../org/apache/iotdb/db/metadata/MLogWriter.java | 8 ++
.../org/apache/iotdb/db/metadata/MManager.java | 88 +++++++++++++------
.../iotdb/db/metadata/id/ID2NodeManager.java | 42 +++++++++
.../org/apache/iotdb/db/metadata/id/IDManager.java | 99 ++++++++++++++++++++++
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +-
5 files changed, 215 insertions(+), 27 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
index 0b9ab3e..8f22ffc 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
@@ -169,4 +169,12 @@ public class MLogWriter {
int getLineNumber() {
return lineNumber;
}
+
+ /**
+ * only used for initialize a mlog file writer.
+ * @param number
+ */
+ void settLineNumber(int number) {
+ lineNumber = number;
+ }
}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 16847ca..3cd2ac4 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -205,7 +205,7 @@ public class MManager {
tagLogFile = new TagLogFile(config.getSchemaDir(), MetadataConstant.TAG_LOG);
isRecovering = true;
- initFromLog(logFile);
+ int lineNumber = initFromLog(logFile);
if (config.isEnableParameterAdapter()) {
List<String> storageGroups = mtree.getAllStorageGroupNames();
@@ -218,6 +218,7 @@ public class MManager {
}
logWriter = new MLogWriter(config.getSchemaDir(), MetadataConstant.METADATA_LOG);
+ logWriter.settLineNumber(lineNumber);
isRecovering = false;
} catch (IOException | MetadataException e) {
mtree = new MTree();
@@ -226,7 +227,13 @@ public class MManager {
initialized = true;
}
- private void initFromLog(File logFile) throws IOException {
+ /**
+ *
+ * @param logFile
+ * @return line number of the logFile
+ * @throws IOException
+ */
+ private int initFromLog(File logFile) throws IOException {
File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
if (tmpFile.exists()) {
logger.warn("Creating MTree snapshot not successful before crashing...");
@@ -238,25 +245,37 @@ public class MManager {
mtree = new MTree();
} else {
mtree = MTree.deserializeFrom(mtreeSnapshot);
+ lastSnapshotLogLineNumber = mtree.getSnapshotLineNumber();
}
// init the metadata from the operation log
if (logFile.exists()) {
+ int idx = 0;
try (FileReader fr = new FileReader(logFile);
BufferedReader br = new BufferedReader(fr)) {
String cmd;
- int idx = 0;
while (idx < mtree.getSnapshotLineNumber()) {
cmd = br.readLine();
+ if (cmd == null) {
+ throw new IOException(String
+ .format("mtree snapshot file has %d lines but the mlog.txt has only %d lines.",
+ mtree.getSnapshotLineNumber(), idx));
+ }
idx++;
}
while ((cmd = br.readLine()) != null) {
try {
operation(cmd);
+ idx++;
} catch (Exception e) {
logger.error("Can not operate cmd {}", cmd, e);
}
}
}
+ return idx;
+ } else if (mtreeSnapshot.exists()) {
+ throw new IOException("mtree snapshot file exists but mlog.txt does not exist.");
+ } else {
+ return 0;
}
}
@@ -1773,31 +1792,48 @@ public class MManager {
}
private void checkMTreeModified() {
- if (System.currentTimeMillis() - logFile.lastModified() < mtreeSnapshotThresholdTime) {
- logger.info("MTree snapshot is not created because of active modification");
- } else if (logWriter.getLineNumber() - lastSnapshotLogLineNumber < mtreeSnapshotInterval) {
- logger.info(
- "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}",
- logWriter.getLineNumber(), lastSnapshotLogLineNumber);
+ if (logWriter == null || logFile == null) {
+ //the logWriter is not initialized now, we skip the check once.
+ return;
+ }
+ if (System.currentTimeMillis() - logFile.lastModified() >= mtreeSnapshotThresholdTime
+ && logWriter.getLineNumber() > lastSnapshotLogLineNumber) {
+ logger.info("Start creating MTree snapshot, because {} ms elaspse.", System.currentTimeMillis() - logFile.lastModified());
+ createSnapshot();
+ } else if (logWriter.getLineNumber() - lastSnapshotLogLineNumber >= mtreeSnapshotInterval) {
+ logger.info("Start creating MTree snapshot, because of {} new lines are added.", logWriter.getLineNumber() - lastSnapshotLogLineNumber);
+ createSnapshot();
} else {
- lock.readLock().lock();
- logger.info("Start creating MTree snapshot. This may take a while...");
- try {
- mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber());
- lastSnapshotLogLineNumber = logWriter.getLineNumber();
- File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
- File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
- if (snapshotFile.exists()) {
- Files.delete(snapshotFile.toPath());
- }
- if (tmpFile.renameTo(snapshotFile)) {
- logger.info("Finish creating MTree snapshot to {}.", mtreeSnapshotPath);
- }
- } catch (IOException e) {
- logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e);
- } finally {
- lock.readLock().unlock();
+ ///if (logger.isDebugEnabled()) {
+ logger.info(
+ "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {}ms",
+ logWriter.getLineNumber(), lastSnapshotLogLineNumber,
+ System.currentTimeMillis() - logFile.lastModified());
+ //}
+ }
+ }
+
+ private void createSnapshot() {
+ lock.readLock().lock();
+ long time = System.currentTimeMillis();
+ try {
+ mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber());
+ lastSnapshotLogLineNumber = logWriter.getLineNumber();
+ File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
+ File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
+ if (snapshotFile.exists()) {
+ Files.delete(snapshotFile.toPath());
}
+ if (tmpFile.renameTo(snapshotFile)) {
+ logger.info("Finish creating MTree snapshot to {}, spend {}ms.", mtreeSnapshotPath, System.currentTimeMillis() - time);
+ }
+ } catch (IOException e) {
+ logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e);
+ if (SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath).exists()) {
+ SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath).delete();
+ }
+ } finally {
+ lock.readLock().unlock();
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id/ID2NodeManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/id/ID2NodeManager.java
new file mode 100644
index 0000000..ae6dc6b
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id/ID2NodeManager.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.id;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.metadata.mnode.MNode;
+
+public class ID2NodeManager {
+ List<MNode> id2strings = new ArrayList<>();
+
+
+ /**
+ * when calling this method, you must guarantee that all ids that less than id has been put.
+ * @param id
+ * @param node
+ */
+ public void put(int id, MNode node) {
+ id2strings.set(id, node);
+ }
+
+ public MNode get(int id) {
+ return id2strings.get(id);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id/IDManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/id/IDManager.java
new file mode 100644
index 0000000..5848ae0
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id/IDManager.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.id;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+
+/**
+ * ID generator for storage groups, devices, and measurements
+ */
+public class IDManager {
+
+ private static int storageGroupIDLength = 12; //must be not greater than 32
+ private static int deviceIDLength = 32; //must be not greater than 32
+ private static int measurementLength = 20; //must be not greater than 32
+
+ private static int maxSGID;
+ private static int maxDeivceID;
+ private static int maxMeasurementID;
+ static {
+ if (storageGroupIDLength == 32) {
+ maxSGID = 0xFFFFFFFF;
+ } else {
+ maxSGID = 1 << storageGroupIDLength - 1;
+ }
+
+ if (deviceIDLength == 32) {
+ maxDeivceID = 0xFFFFFFFF;
+ } else {
+ maxDeivceID = 1 << deviceIDLength - 1;
+ }
+
+ if (measurementLength == 32) {
+ maxMeasurementID = 0xFFFFFFFF;
+ } else {
+ maxMeasurementID = 1 << measurementLength - 1;
+ }
+ }
+
+ private static AtomicInteger sgGenerator = new AtomicInteger(0);
+ private static AtomicInteger deviceGenerator = new AtomicInteger(0);
+ private static AtomicInteger measurementGenerator = new AtomicInteger(0);
+
+
+ public static int newSGNumber() throws MetadataException {
+ if (sgGenerator.get() == maxSGID) {
+ throw new MetadataException("too many storage groups: {}", sgGenerator.get());
+ }
+ return sgGenerator.incrementAndGet();
+ }
+
+ public static int newDeviceNumber() throws MetadataException {
+ if (deviceGenerator.get() == maxDeivceID) {
+ throw new MetadataException("too many devices: {}", deviceGenerator.get());
+ }
+ return deviceGenerator.incrementAndGet();
+ }
+
+ public static int newMeasurementNumber() throws MetadataException {
+ if (measurementGenerator.get() == maxMeasurementID) {
+ throw new MetadataException("too many measurements: {}", measurementGenerator.get());
+ }
+ return measurementGenerator.incrementAndGet();
+ }
+
+ public static long newID() throws MetadataException {
+ return (0L | ((long) newSGNumber()) << (deviceIDLength + measurementLength) | ((long) newDeviceNumber()) << measurementLength | newMeasurementNumber());
+ }
+
+ public static int getStorageGroupID(long fullId) {
+ return (int)(fullId >>> (deviceIDLength + measurementLength));
+ }
+
+ public static int getDeviceID(long fullId) {
+ return (int)((fullId << storageGroupIDLength) >>> (storageGroupIDLength + measurementLength));
+ }
+
+ public static int getMeasurementID(long fullId) {
+ return (int)((fullId << storageGroupIDLength + deviceIDLength) >>> (storageGroupIDLength + deviceIDLength));
+ }
+
+
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index eedf20d..dc3ffa6 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -89,7 +89,7 @@ public class IoTDB implements IoTDBMBean {
Runtime.getRuntime().addShutdownHook(new IoTDBShutdownHook());
setUncaughtExceptionHandler();
-
+ logger.info("recover the schema...");
initMManager();
registerManager.register(JMXService.getInstance());
registerManager.register(FlushManager.getInstance());
@@ -131,7 +131,10 @@ public class IoTDB implements IoTDBMBean {
}
private void initMManager() {
+ long time = System.currentTimeMillis();
MManager.getInstance().init();
+ long end = System.currentTimeMillis() - time;
+ logger.info("spend {}ms to recover schema.", end);
IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
logger.info(
"After initializing, max memTable num is {}, tsFile threshold is {}, memtableSize is {}",