You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/08/18 14:13:17 UTC
[iotdb] branch write_preformance_wal created (now ae2a4402f1)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch write_preformance_wal
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at ae2a4402f1 Avoid serializing schema in wal for better write performance of standalone
This branch includes the following new commits:
new ae2a4402f1 Avoid serializing schema in wal for better write performance of standalone
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Avoid serializing schema in wal for better write performance of standalone
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch write_preformance_wal
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae2a4402f103455c13b5d2fd2acb2b9c0b898a48
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Aug 18 22:13:02 2022 +0800
Avoid serializing schema in wal for better write performance of standalone
---
.../db/mpp/plan/planner/plan/node/write/InsertNode.java | 17 ++++++++++++++---
.../iotdb/db/wal/recover/file/TsFilePlanRedoer.java | 6 +++++-
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java
index 884e693a12..927a759ad7 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.wal.utils.WALWriteUtils;
import org.apache.iotdb.tsfile.exception.NotImplementedException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import java.io.DataInputStream;
@@ -193,7 +194,13 @@ public abstract class InsertNode extends WritePlanNode {
if (measurements[i] == null) {
continue;
}
- WALWriteUtils.write(measurementSchemas[i], buffer);
+
+ // serialize measurementId only for standalone version for better write performance
+ if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
+ WALWriteUtils.write(measurementSchemas[i], buffer);
+ } else {
+ WALWriteUtils.write(measurements[i], buffer);
+ }
}
}
@@ -203,8 +210,12 @@ public abstract class InsertNode extends WritePlanNode {
*/
protected void deserializeMeasurementSchemas(DataInputStream stream) throws IOException {
for (int i = 0; i < measurementSchemas.length; i++) {
- measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream);
- measurements[i] = measurementSchemas[i].getMeasurementId();
+ if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
+ measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream);
+ measurements[i] = measurementSchemas[i].getMeasurementId();
+ } else {
+ measurements[i] = ReadWriteIOUtils.readString(stream);
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java
index a5840c61ba..d69798fdca 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.idtable.IDTable;
import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
+import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;
@@ -138,7 +139,7 @@ public class TsFilePlanRedoer {
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- void redoInsert(InsertNode node) throws WriteProcessException, QueryProcessException {
+ void redoInsert(InsertNode node) throws WriteProcessException {
if (!node.hasValidMeasurements()) {
return;
}
@@ -165,6 +166,9 @@ public class TsFilePlanRedoer {
// TODO get device id by idTable
// idTable.getSeriesSchemas(node);
} else {
+ if (!IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
+ SchemaValidator.validate(node);
+ }
node.setDeviceID(DeviceIDFactory.getInstance().getDeviceID(node.getDevicePath()));
}