You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/09/18 06:27:29 UTC
[incubator-iotdb] 01/01: using String.getBytes(utf-9).length to
replace string.length() in ChunkGroupMetadata for supporting Chinese
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch fix_195_chinese_ChunkGroupMetadata
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 07c36997631f2f339c0f8045192b9a09c02e4af7
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Wed Sep 18 14:27:08 2019 +0800
using String.getBytes(utf-9).length to replace string.length() in ChunkGroupMetadata for supporting Chinese
---
.../java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 2 +-
.../org/apache/iotdb/tsfile/common/conf/TSFileConfig.java | 2 ++
.../apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java | 2 +-
.../org/apache/iotdb/tsfile/file/header/ChunkHeader.java | 7 +------
.../iotdb/tsfile/file/metadata/ChunkGroupMetaData.java | 12 ++++++++----
.../apache/iotdb/tsfile/file/metadata/ChunkMetaData.java | 8 +-------
.../main/java/org/apache/iotdb/tsfile/utils/Binary.java | 10 +++-------
.../java/org/apache/iotdb/tsfile/utils/BytesUtils.java | 14 ++------------
.../org/apache/iotdb/tsfile/utils/StringContainer.java | 1 +
9 files changed, 20 insertions(+), 38 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index ad42d18..d9addec 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -81,7 +81,7 @@ public class IoTDBConfigCheck {
// get existed properties from system_properties.txt
File inputFile = SystemFileFactory.INSTANCE.getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
- properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_ENCODING));
+ properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET));
if (!properties.getProperty("timestamp_precision").equals(TIMESTAMP_PRECISION)) {
logger.error("Wrong timestamp precision, please set as: " + properties
.getProperty("timestamp_precision") + " !");
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 79b6622..39affd9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.common.conf;
+import java.nio.charset.Charset;
import org.apache.iotdb.tsfile.fileSystem.FSType;
/**
@@ -49,6 +50,7 @@ public class TSFileConfig {
*/
public static final int BYTE_SIZE_PER_CHAR = 4;
public static final String STRING_ENCODING = "UTF-8";
+ public static final Charset STRING_CHARSET = Charset.forName(STRING_ENCODING);
public static final String CONFIG_FILE_NAME = "tsfile-format.properties";
public static final String MAGIC_STRING = "TsFilev0.8.0";
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
index e440956..e1fa0b7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
@@ -55,7 +55,7 @@ public class ChunkGroupFooter {
this.dataSize = dataSize;
this.numberOfChunks = numberOfChunks;
this.serializedSize =
- Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_ENCODING).length + Long.BYTES + Integer.BYTES;
+ Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Long.BYTES + Integer.BYTES;
}
public static int getSerializedSize(String deviceID) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index 0dc5ef0..b2e7279f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -76,12 +76,7 @@ public class ChunkHeader {
}
public static int getSerializedSize(String measurementID) {
- try {
- return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_ENCODING).length);
- } catch (UnsupportedEncodingException e) {
- LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING);
- return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes().length);
- }
+ return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_CHARSET).length);
}
private static int getSerializedSize(int measurementIdLength) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
index a717eba..c81d074 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.file.metadata;
+import java.io.UnsupportedEncodingException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -28,11 +29,14 @@ import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Metadata of ChunkGroup.
*/
public class ChunkGroupMetaData {
+ private static final Logger logger = LoggerFactory.getLogger(ChunkGroupMetaData.class);
/**
* Name of device, this field is not serialized.
@@ -104,7 +108,7 @@ public class ChunkGroupMetaData {
int size = ReadWriteIOUtils.readInt(inputStream);
chunkGroupMetaData.serializedSize = Integer.BYTES
- + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length
+ + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length
+ Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
@@ -135,7 +139,7 @@ public class ChunkGroupMetaData {
int size = ReadWriteIOUtils.readInt(buffer);
- chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length
+ chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length
+ Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
@@ -154,8 +158,8 @@ public class ChunkGroupMetaData {
}
void reCalculateSerializedSize() {
- serializedSize = Integer.BYTES + deviceID.length() + Integer.BYTES
- + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList
+ serializedSize = Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Integer.BYTES
+ + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList
for (ChunkMetaData chunk : chunkMetaDataList) {
serializedSize += chunk.getSerializedSize();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
index 269e861..c94c654 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
@@ -146,13 +146,7 @@ public class ChunkMetaData {
TSDataType.getSerializedSize() + // TSDataType
(valuesStatistics == null ? TsDigest.getNullDigestSize()
: valuesStatistics.getSerializedSize()));
- try {
- serializedSize += measurementUid.getBytes(TSFileConfig.STRING_ENCODING).length; // measurementUid
- } catch (UnsupportedEncodingException e) {
- // use the system default encoding
- serializedSize += measurementUid.getBytes().length; // measurementUid
- LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING);
- }
+ serializedSize += measurementUid.getBytes(TSFileConfig.STRING_CHARSET).length; // measurementUid
return serializedSize;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java
index b071806..61d8323 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java
@@ -34,7 +34,6 @@ public class Binary implements Comparable<Binary>, Serializable {
private static final long serialVersionUID = 6394197743397020735L;
private byte[] values;
- private String textEncodingType = TSFileConfig.STRING_ENCODING;
/**
* if the bytes v is modified, the modification is visible to this binary.
@@ -44,7 +43,7 @@ public class Binary implements Comparable<Binary>, Serializable {
}
public Binary(String s) {
- this.values = (s == null) ? null : s.getBytes(Charset.forName(this.textEncodingType));
+ this.values = (s == null) ? null : s.getBytes(TSFileConfig.STRING_CHARSET);
}
public static Binary valueOf(String value) {
@@ -105,16 +104,13 @@ public class Binary implements Comparable<Binary>, Serializable {
}
public String getStringValue() {
- return new String(this.values, Charset.forName(this.textEncodingType));
+ return new String(this.values, TSFileConfig.STRING_CHARSET);
}
public String getTextEncodingType() {
- return textEncodingType;
+ return TSFileConfig.STRING_ENCODING;
}
- public void setTextEncodingType(String textEncodingType) {
- this.textEncodingType = textEncodingType;
- }
@Override
public String toString() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
index 990dba9..20c6a58 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
@@ -567,12 +567,7 @@ public class BytesUtils {
* @return byte array
*/
public static byte[] stringToBytes(String str) {
- try {
- return str.getBytes(TSFileConfig.STRING_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("catch UnsupportedEncodingException {}", str, e);
- return null;
- }
+ return str.getBytes(TSFileConfig.STRING_CHARSET);
}
/**
@@ -582,12 +577,7 @@ public class BytesUtils {
* @return string
*/
public static String bytesToString(byte[] byteStr) {
- try {
- return new String(byteStr, TSFileConfig.STRING_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("catch UnsupportedEncodingException {}", byteStr, e);
- return null;
- }
+ return new String(byteStr, TSFileConfig.STRING_CHARSET);
}
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java
index 8212db1..bbe2179 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.utils;
import java.util.ArrayList;
import java.util.List;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
/**
* this class is used to contact String effectively.It contains a StringBuider and initialize it