You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/09/26 17:19:06 UTC
[incubator-iotdb] 06/12: Change all the str.length() to
str.getBytes().length while calculating the serialized byte size
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch rel/0.8
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 4fd30294a5687746d77733f26c30b8b94f355e88
Author: JackieTien97 <13...@qq.com>
AuthorDate: Fri Sep 6 10:50:24 2019 +0800
Change all the str.length() to str.getBytes().length while calculating the serialized byte size
---
.../iotdb/tsfile/file/footer/ChunkGroupFooter.java | 13 +++++----
.../iotdb/tsfile/file/header/ChunkHeader.java | 24 ++++++++++++-----
.../tsfile/file/metadata/ChunkGroupMetaData.java | 14 +++++-----
.../iotdb/tsfile/file/metadata/ChunkMetaData.java | 31 ++++++++++++++++------
.../tsfile/file/metadata/TsDeviceMetadata.java | 5 ++--
5 files changed, 60 insertions(+), 27 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
index a69468e..e440956 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
@@ -19,13 +19,16 @@
package org.apache.iotdb.tsfile.file.footer;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.read.reader.TsFileInput;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
-import org.apache.iotdb.tsfile.file.MetaMarker;
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class ChunkGroupFooter {
@@ -47,12 +50,12 @@ public class ChunkGroupFooter {
* @param dataSize data size
* @param numberOfChunks number of chunks
*/
- public ChunkGroupFooter(String deviceID, long dataSize, int numberOfChunks) {
+ public ChunkGroupFooter(String deviceID, long dataSize, int numberOfChunks) throws UnsupportedEncodingException {
this.deviceID = deviceID;
this.dataSize = dataSize;
this.numberOfChunks = numberOfChunks;
this.serializedSize =
- Byte.BYTES + Integer.BYTES + deviceID.length() + Long.BYTES + Integer.BYTES;
+ Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_ENCODING).length + Long.BYTES + Integer.BYTES;
}
public static int getSerializedSize(String deviceID) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index d7e724f..0dc5ef0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -19,19 +19,26 @@
package org.apache.iotdb.tsfile.file.header;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
public class ChunkHeader {
+ private static final Logger LOG = LoggerFactory.getLogger(ChunkHeader.class);
+
public static final byte MARKER = MetaMarker.CHUNK_HEADER;
private String measurementID;
@@ -69,7 +76,12 @@ public class ChunkHeader {
}
public static int getSerializedSize(String measurementID) {
- return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes().length);
+ try {
+ return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_ENCODING).length);
+ } catch (UnsupportedEncodingException e) {
+ LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING);
+ return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes().length);
+ }
}
private static int getSerializedSize(int measurementIdLength) {
@@ -150,7 +162,7 @@ public class ChunkHeader {
return deserializePartFrom(measurementID, buffer);
}
- private static ChunkHeader deserializePartFrom(String measurementID, ByteBuffer buffer) {
+ private static ChunkHeader deserializePartFrom(String measurementID, ByteBuffer buffer) throws UnsupportedEncodingException {
int dataSize = ReadWriteIOUtils.readInt(buffer);
TSDataType dataType = TSDataType.deserialize(ReadWriteIOUtils.readShort(buffer));
int numOfPages = ReadWriteIOUtils.readInt(buffer);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
index 01cc0ab..b869274 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
@@ -19,14 +19,15 @@
package org.apache.iotdb.tsfile.file.metadata;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* Metadata of ChunkGroup.
@@ -102,8 +103,9 @@ public class ChunkGroupMetaData {
chunkGroupMetaData.version = ReadWriteIOUtils.readLong(inputStream);
int size = ReadWriteIOUtils.readInt(inputStream);
- chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.length()
- + Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
+ chunkGroupMetaData.serializedSize = Integer.BYTES
+ + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length
+ + Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
@@ -123,7 +125,7 @@ public class ChunkGroupMetaData {
* @param buffer ByteBuffer
* @return ChunkGroupMetaData object
*/
- public static ChunkGroupMetaData deserializeFrom(ByteBuffer buffer) {
+ public static ChunkGroupMetaData deserializeFrom(ByteBuffer buffer) throws IOException {
ChunkGroupMetaData chunkGroupMetaData = new ChunkGroupMetaData();
chunkGroupMetaData.deviceID = ReadWriteIOUtils.readString(buffer);
@@ -133,7 +135,7 @@ public class ChunkGroupMetaData {
int size = ReadWriteIOUtils.readInt(buffer);
- chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.length()
+ chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length
+ Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
index 7c89cf2..c63152e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
@@ -19,19 +19,27 @@
package org.apache.iotdb.tsfile.file.metadata;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.Objects;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* MetaData of one chunk.
*/
public class ChunkMetaData {
+ private static final Logger LOG = LoggerFactory.getLogger(ChunkMetaData.class);
+
+
private String measurementUid;
/**
@@ -133,12 +141,19 @@ public class ChunkMetaData {
* @return serialized size (int type)
*/
public int getSerializedSize() {
- return (Integer.BYTES + measurementUid.length()) + // measurementUid
- 4 * Long.BYTES + // 4 long: offsetOfChunkHeader, numOfPoints, startTime, endTime
- TSDataType.getSerializedSize() + // TSDataType
- (valuesStatistics == null ? TsDigest.getNullDigestSize()
- : valuesStatistics.getSerializedSize());
-
+ int serializedSize = (Integer.BYTES +
+ 4 * Long.BYTES + // 4 long: offsetOfChunkHeader, numOfPoints, startTime, endTime
+ TSDataType.getSerializedSize() + // TSDataType
+ (valuesStatistics == null ? TsDigest.getNullDigestSize()
+ : valuesStatistics.getSerializedSize()));
+ try {
+ serializedSize += measurementUid.getBytes(TSFileConfig.STRING_ENCODING).length; // measurementUid
+ } catch (UnsupportedEncodingException e) {
+ // use the system default encoding
+ serializedSize += measurementUid.getBytes().length; // measurementUid
+ LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING);
+ }
+ return serializedSize;
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java
index a7c4a5a..df47cbe 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDeviceMetadata.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.tsfile.file.metadata;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -26,7 +28,6 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class TsDeviceMetadata {
@@ -86,7 +87,7 @@ public class TsDeviceMetadata {
* @param buffer -buffer to deserialize
* @return -device meta data
*/
- public static TsDeviceMetadata deserializeFrom(ByteBuffer buffer) {
+ public static TsDeviceMetadata deserializeFrom(ByteBuffer buffer) throws IOException {
TsDeviceMetadata deviceMetadata = new TsDeviceMetadata();
deviceMetadata.startTime = ReadWriteIOUtils.readLong(buffer);