You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2024/06/17 07:05:45 UTC
(iotdb) 01/02: Indicate the encoding of String
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch wal-compress-formal-branch
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0e0236650d4703baa26418adb4f03fed206f9a63
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Mon Jun 17 14:45:58 2024 +0800
Indicate the encoding of String
---
.../dataregion/wal/buffer/WALBuffer.java | 1 +
.../db/storageengine/dataregion/wal/io/LogWriter.java | 4 +++-
.../dataregion/wal/io/WALByteBufReader.java | 1 +
.../dataregion/wal/io/WALInputStream.java | 19 +++++++++++++------
.../storageengine/dataregion/wal/io/WALMetaData.java | 3 ++-
.../db/storageengine/dataregion/wal/io/WALWriter.java | 5 +++--
6 files changed, 23 insertions(+), 10 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index 627771bde38..20970aea34b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -540,6 +540,7 @@ public class WALBuffer extends AbstractWALBuffer {
boolean forceSuccess = false;
// try to roll log writer
if (info.rollWALFileWriterListener != null
+ // TODO: Control the wal file by the number of WALEntry
|| (forceFlag
&& currentWALFileWriter.originalSize() >= config.getWalFileSizeThresholdInByte())) {
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java
index 98a42b75280..29335efde5c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java
@@ -34,6 +34,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
/**
@@ -61,7 +62,8 @@ public abstract class LogWriter implements ILogWriter {
this.logStream = new FileOutputStream(logFile, true);
this.logChannel = this.logStream.getChannel();
if (!logFile.exists() || logFile.length() == 0) {
- this.logChannel.write(ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes()));
+ this.logChannel.write(
+ ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes(StandardCharsets.UTF_8)));
size += logChannel.position();
}
if (IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
index 3e0909d6c52..882b5ea468c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
@@ -66,6 +66,7 @@ public class WALByteBufReader implements Closeable {
*/
public ByteBuffer next() throws IOException {
int size = sizeIterator.next();
+ // TODO: Reuse this buffer
ByteBuffer buffer = ByteBuffer.allocate(size);
/*
Notice, we don't need to flip the buffer after calling
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java
index 780be52671c..7cdcb63095f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java
@@ -28,6 +28,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
public class WALInputStream extends InputStream implements AutoCloseable {
@@ -78,7 +79,8 @@ public class WALInputStream extends InputStream implements AutoCloseable {
ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES);
channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_BYTES);
magicStringBuffer.flip();
- if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING)) {
+ if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8)
+ .equals(WALWriter.MAGIC_STRING)) {
// This is a broken wal file
endOffset = channel.size();
return;
@@ -89,16 +91,21 @@ public class WALInputStream extends InputStream implements AutoCloseable {
} else {
// Old version
ByteBuffer magicStringBuffer =
- ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes().length);
+ ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length);
channel.read(
- magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length);
+ magicStringBuffer,
+ channel.size() - WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length);
magicStringBuffer.flip();
- if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING_V1)) {
+ if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8)
+ .equals(WALWriter.MAGIC_STRING_V1)) {
// this is a broken wal file
endOffset = channel.size();
return;
} else {
- position = channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length - Integer.BYTES;
+ position =
+ channel.size()
+ - WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length
+ - Integer.BYTES;
}
}
// Read the meta data size
@@ -127,7 +134,7 @@ public class WALInputStream extends InputStream implements AutoCloseable {
channel.position(0);
ByteBuffer buffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES);
channel.read(buffer);
- return new String(buffer.array()).equals(WALWriter.MAGIC_STRING);
+ return new String(buffer.array(), StandardCharsets.UTF_8).equals(WALWriter.MAGIC_STRING);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
index 8824bff3a26..7fa634ffbdc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
@@ -29,6 +29,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -161,7 +162,7 @@ public class WALMetaData implements SerializedSize {
ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES);
channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES);
magicStringBytes.flip();
- String magicString = new String(magicStringBytes.array());
+ String magicString = new String(magicStringBytes.array(), StandardCharsets.UTF_8);
return magicString.equals(WALWriter.MAGIC_STRING)
|| magicString.contains(WALWriter.MAGIC_STRING_V1);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java
index c5be73892e7..7017b4be6cb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java
@@ -30,13 +30,14 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
/** WALWriter writes the binary {@link WALEntry} into .wal file. */
public class WALWriter extends LogWriter {
private static final Logger logger = LoggerFactory.getLogger(WALWriter.class);
public static final String MAGIC_STRING_V1 = "WAL";
public static final String MAGIC_STRING = "V2-WAL";
- public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes().length;
+ public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes(StandardCharsets.UTF_8).length;
private WALFileStatus walFileStatus = WALFileStatus.CONTAINS_NONE_SEARCH_INDEX;
// wal files' metadata
@@ -74,7 +75,7 @@ public class WALWriter extends LogWriter {
metaData.serialize(logFile, buffer);
buffer.putInt(metaDataSize);
// add magic string
- buffer.put(MAGIC_STRING.getBytes());
+ buffer.put(MAGIC_STRING.getBytes(StandardCharsets.UTF_8));
size += buffer.position();
writeMetadata(buffer);
}