You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/08/19 00:55:39 UTC
[iotdb] branch revert-7050-write_preformance_wal created (now 4c3272c066)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch revert-7050-write_preformance_wal
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 4c3272c066 Revert "[IOTDB-3938] Avoid serializing schema in wal for better write performance of standalone version (#7050)"
This branch includes the following new commits:
new 4c3272c066 Revert "[IOTDB-3938] Avoid serializing schema in wal for better write performance of standalone version (#7050)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Revert "[IOTDB-3938] Avoid serializing schema in wal for better write performance of standalone version (#7050)"
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch revert-7050-write_preformance_wal
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4c3272c066573d9260dd0d291c6bbdccafead6bb
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Aug 19 08:55:33 2022 +0800
Revert "[IOTDB-3938] Avoid serializing schema in wal for better write performance of standalone version (#7050)"
This reverts commit 6c4a87c1af53ba1128596f7b31ebb2821a6e69a9.
---
.../plan/planner/plan/node/write/InsertNode.java | 28 ++++------------------
.../planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 7 +++++-
.../db/wal/recover/file/TsFilePlanRedoer.java | 6 +----
.../plan/node/write/InsertRowNodeSerdeTest.java | 9 +------
.../plan/node/write/InsertTabletNodeSerdeTest.java | 8 -------
.../org/apache/iotdb/db/wal/io/WALFileTest.java | 21 ++++++++++++++--
.../iotdb/db/wal/node/ConsensusReqReaderTest.java | 2 --
8 files changed, 32 insertions(+), 51 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java
index 610628b334..884e693a12 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java
@@ -31,7 +31,6 @@ import org.apache.iotdb.db.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.wal.utils.WALWriteUtils;
import org.apache.iotdb.tsfile.exception.NotImplementedException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import java.io.DataInputStream;
@@ -182,13 +181,7 @@ public abstract class InsertNode extends WritePlanNode {
if (measurements[i] == null) {
continue;
}
- if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
- byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]);
- } else {
- byteLen += ReadWriteIOUtils.sizeToWrite(measurements[i]);
- // datatype size
- byteLen++;
- }
+ byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]);
}
return byteLen;
}
@@ -200,14 +193,7 @@ public abstract class InsertNode extends WritePlanNode {
if (measurements[i] == null) {
continue;
}
-
- // serialize measurementId only for standalone version for better write performance
- if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
- WALWriteUtils.write(measurementSchemas[i], buffer);
- } else {
- WALWriteUtils.write(measurements[i], buffer);
- WALWriteUtils.write(dataTypes[i], buffer);
- }
+ WALWriteUtils.write(measurementSchemas[i], buffer);
}
}
@@ -217,14 +203,8 @@ public abstract class InsertNode extends WritePlanNode {
*/
protected void deserializeMeasurementSchemas(DataInputStream stream) throws IOException {
for (int i = 0; i < measurementSchemas.length; i++) {
- if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
- measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream);
- measurements[i] = measurementSchemas[i].getMeasurementId();
- dataTypes[i] = measurementSchemas[i].getType();
- } else {
- measurements[i] = ReadWriteIOUtils.readString(stream);
- dataTypes[i] = TSDataType.deserialize(ReadWriteIOUtils.readByte(stream));
- }
+ measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream);
+ measurements[i] = measurementSchemas[i].getMeasurementId();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowNode.java
index 3bc7e27c7a..42ea49dc35 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowNode.java
@@ -662,9 +662,9 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
measurements = new String[measurementSize];
measurementSchemas = new MeasurementSchema[measurementSize];
- dataTypes = new TSDataType[measurementSize];
deserializeMeasurementSchemas(stream);
+ dataTypes = new TSDataType[measurementSize];
values = new Object[measurementSize];
fillDataTypesAndValuesFromWAL(stream);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertTabletNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertTabletNode.java
index 1a660edc7b..5b91ca0125 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertTabletNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertTabletNode.java
@@ -916,9 +916,14 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
int measurementSize = stream.readInt();
measurements = new String[measurementSize];
measurementSchemas = new MeasurementSchema[measurementSize];
- dataTypes = new TSDataType[measurementSize];
deserializeMeasurementSchemas(stream);
+ // data types are serialized in measurement schemas
+ dataTypes = new TSDataType[measurementSize];
+ for (int i = 0; i < measurementSize; i++) {
+ dataTypes[i] = measurementSchemas[i].getType();
+ }
+
rowCount = stream.readInt();
times = new long[rowCount];
times = QueryDataSetUtils.readTimesFromStream(stream, rowCount);
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java
index d69798fdca..a5840c61ba 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.idtable.IDTable;
import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
-import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;
@@ -139,7 +138,7 @@ public class TsFilePlanRedoer {
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- void redoInsert(InsertNode node) throws WriteProcessException {
+ void redoInsert(InsertNode node) throws WriteProcessException, QueryProcessException {
if (!node.hasValidMeasurements()) {
return;
}
@@ -166,9 +165,6 @@ public class TsFilePlanRedoer {
// TODO get device id by idTable
// idTable.getSeriesSchemas(node);
} else {
- if (!IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
- SchemaValidator.validate(node);
- }
node.setDeviceID(DeviceIDFactory.getInstance().getDeviceID(node.getDevicePath()));
}
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertRowNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertRowNodeSerdeTest.java
index 55b4a61045..f57ccbd982 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertRowNodeSerdeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertRowNodeSerdeTest.java
@@ -87,14 +87,7 @@ public class InsertRowNodeSerdeTest {
InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream);
tmpNode.setPlanNodeId(insertRowNode.getPlanNodeId());
- tmpNode.setMeasurementSchemas(
- new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
- new MeasurementSchema("s3", TSDataType.INT64),
- new MeasurementSchema("s4", TSDataType.INT32),
- new MeasurementSchema("s5", TSDataType.BOOLEAN)
- });
+
Assert.assertEquals(insertRowNode, tmpNode);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertTabletNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertTabletNodeSerdeTest.java
index 83b2143166..d3f0d08066 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertTabletNodeSerdeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/InsertTabletNodeSerdeTest.java
@@ -79,14 +79,6 @@ public class InsertTabletNodeSerdeTest {
InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream);
tmpNode.setPlanNodeId(insertTabletNode.getPlanNodeId());
- tmpNode.setMeasurementSchemas(
- new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
- new MeasurementSchema("s3", TSDataType.INT64),
- new MeasurementSchema("s4", TSDataType.INT32),
- new MeasurementSchema("s5", TSDataType.BOOLEAN)
- });
Assert.assertEquals(insertTabletNode, tmpNode);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/wal/io/WALFileTest.java b/server/src/test/java/org/apache/iotdb/db/wal/io/WALFileTest.java
index e6a8ecef46..bf6460d830 100644
--- a/server/src/test/java/org/apache/iotdb/db/wal/io/WALFileTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/wal/io/WALFileTest.java
@@ -262,7 +262,15 @@ public class WALFileTest {
columns,
false);
- insertRowNode.setMeasurementSchemas(new MeasurementSchema[6]);
+ insertRowNode.setMeasurementSchemas(
+ new MeasurementSchema[] {
+ new MeasurementSchema("s1", TSDataType.DOUBLE),
+ new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("s3", TSDataType.INT64),
+ new MeasurementSchema("s4", TSDataType.INT32),
+ new MeasurementSchema("s5", TSDataType.BOOLEAN),
+ new MeasurementSchema("s6", TSDataType.TEXT)
+ });
return insertRowNode;
}
@@ -315,7 +323,16 @@ public class WALFileTest {
bitMaps,
columns,
times.length);
- insertTabletNode.setMeasurementSchemas(new MeasurementSchema[6]);
+
+ insertTabletNode.setMeasurementSchemas(
+ new MeasurementSchema[] {
+ new MeasurementSchema("s1", TSDataType.DOUBLE),
+ new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("s3", TSDataType.INT64),
+ new MeasurementSchema("s4", TSDataType.INT32),
+ new MeasurementSchema("s5", TSDataType.BOOLEAN),
+ new MeasurementSchema("s6", TSDataType.TEXT)
+ });
return insertTabletNode;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/wal/node/ConsensusReqReaderTest.java b/server/src/test/java/org/apache/iotdb/db/wal/node/ConsensusReqReaderTest.java
index 1be5e36df1..65ab61780c 100644
--- a/server/src/test/java/org/apache/iotdb/db/wal/node/ConsensusReqReaderTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/wal/node/ConsensusReqReaderTest.java
@@ -58,7 +58,6 @@ public class ConsensusReqReaderTest {
@Before
public void setUp() throws Exception {
EnvironmentUtils.cleanDir(logDirectory);
- config.setClusterMode(true);
prevMode = config.getWalMode();
config.setWalMode(WALMode.SYNC);
walNode = new WALNode(identifier, logDirectory);
@@ -68,7 +67,6 @@ public class ConsensusReqReaderTest {
public void tearDown() throws Exception {
walNode.close();
config.setWalMode(prevMode);
- config.setClusterMode(false);
EnvironmentUtils.cleanDir(logDirectory);
}