You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/09/18 06:27:28 UTC
[incubator-iotdb] branch fix_195_chinese_ChunkGroupMetadata created
(now 07c3699)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a change to branch fix_195_chinese_ChunkGroupMetadata
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 07c3699 using String.getBytes(utf-9).length to replace string.length() in ChunkGroupMetadata for supporting Chinese
This branch includes the following new commits:
new 07c3699 using String.getBytes(utf-9).length to replace string.length() in ChunkGroupMetadata for supporting Chinese
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: using String.getBytes(utf-9).length to
replace string.length() in ChunkGroupMetadata for supporting Chinese
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch fix_195_chinese_ChunkGroupMetadata
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 07c36997631f2f339c0f8045192b9a09c02e4af7
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Wed Sep 18 14:27:08 2019 +0800
using String.getBytes(utf-9).length to replace string.length() in ChunkGroupMetadata for supporting Chinese
---
.../java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 2 +-
.../org/apache/iotdb/tsfile/common/conf/TSFileConfig.java | 2 ++
.../apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java | 2 +-
.../org/apache/iotdb/tsfile/file/header/ChunkHeader.java | 7 +------
.../iotdb/tsfile/file/metadata/ChunkGroupMetaData.java | 12 ++++++++----
.../apache/iotdb/tsfile/file/metadata/ChunkMetaData.java | 8 +-------
.../main/java/org/apache/iotdb/tsfile/utils/Binary.java | 10 +++-------
.../java/org/apache/iotdb/tsfile/utils/BytesUtils.java | 14 ++------------
.../org/apache/iotdb/tsfile/utils/StringContainer.java | 1 +
9 files changed, 20 insertions(+), 38 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index ad42d18..d9addec 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -81,7 +81,7 @@ public class IoTDBConfigCheck {
// get existed properties from system_properties.txt
File inputFile = SystemFileFactory.INSTANCE.getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
- properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_ENCODING));
+ properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET));
if (!properties.getProperty("timestamp_precision").equals(TIMESTAMP_PRECISION)) {
logger.error("Wrong timestamp precision, please set as: " + properties
.getProperty("timestamp_precision") + " !");
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 79b6622..39affd9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.common.conf;
+import java.nio.charset.Charset;
import org.apache.iotdb.tsfile.fileSystem.FSType;
/**
@@ -49,6 +50,7 @@ public class TSFileConfig {
*/
public static final int BYTE_SIZE_PER_CHAR = 4;
public static final String STRING_ENCODING = "UTF-8";
+ public static final Charset STRING_CHARSET = Charset.forName(STRING_ENCODING);
public static final String CONFIG_FILE_NAME = "tsfile-format.properties";
public static final String MAGIC_STRING = "TsFilev0.8.0";
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
index e440956..e1fa0b7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java
@@ -55,7 +55,7 @@ public class ChunkGroupFooter {
this.dataSize = dataSize;
this.numberOfChunks = numberOfChunks;
this.serializedSize =
- Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_ENCODING).length + Long.BYTES + Integer.BYTES;
+ Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Long.BYTES + Integer.BYTES;
}
public static int getSerializedSize(String deviceID) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index 0dc5ef0..b2e7279f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -76,12 +76,7 @@ public class ChunkHeader {
}
public static int getSerializedSize(String measurementID) {
- try {
- return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_ENCODING).length);
- } catch (UnsupportedEncodingException e) {
- LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING);
- return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes().length);
- }
+ return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_CHARSET).length);
}
private static int getSerializedSize(int measurementIdLength) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
index a717eba..c81d074 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.file.metadata;
+import java.io.UnsupportedEncodingException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -28,11 +29,14 @@ import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Metadata of ChunkGroup.
*/
public class ChunkGroupMetaData {
+ private static final Logger logger = LoggerFactory.getLogger(ChunkGroupMetaData.class);
/**
* Name of device, this field is not serialized.
@@ -104,7 +108,7 @@ public class ChunkGroupMetaData {
int size = ReadWriteIOUtils.readInt(inputStream);
chunkGroupMetaData.serializedSize = Integer.BYTES
- + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length
+ + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length
+ Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
@@ -135,7 +139,7 @@ public class ChunkGroupMetaData {
int size = ReadWriteIOUtils.readInt(buffer);
- chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length
+ chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length
+ Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES;
List<ChunkMetaData> chunkMetaDataList = new ArrayList<>();
@@ -154,8 +158,8 @@ public class ChunkGroupMetaData {
}
void reCalculateSerializedSize() {
- serializedSize = Integer.BYTES + deviceID.length() + Integer.BYTES
- + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList
+ serializedSize = Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Integer.BYTES
+ + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList
for (ChunkMetaData chunk : chunkMetaDataList) {
serializedSize += chunk.getSerializedSize();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
index 269e861..c94c654 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java
@@ -146,13 +146,7 @@ public class ChunkMetaData {
TSDataType.getSerializedSize() + // TSDataType
(valuesStatistics == null ? TsDigest.getNullDigestSize()
: valuesStatistics.getSerializedSize()));
- try {
- serializedSize += measurementUid.getBytes(TSFileConfig.STRING_ENCODING).length; // measurementUid
- } catch (UnsupportedEncodingException e) {
- // use the system default encoding
- serializedSize += measurementUid.getBytes().length; // measurementUid
- LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING);
- }
+ serializedSize += measurementUid.getBytes(TSFileConfig.STRING_CHARSET).length; // measurementUid
return serializedSize;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java
index b071806..61d8323 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java
@@ -34,7 +34,6 @@ public class Binary implements Comparable<Binary>, Serializable {
private static final long serialVersionUID = 6394197743397020735L;
private byte[] values;
- private String textEncodingType = TSFileConfig.STRING_ENCODING;
/**
* if the bytes v is modified, the modification is visible to this binary.
@@ -44,7 +43,7 @@ public class Binary implements Comparable<Binary>, Serializable {
}
public Binary(String s) {
- this.values = (s == null) ? null : s.getBytes(Charset.forName(this.textEncodingType));
+ this.values = (s == null) ? null : s.getBytes(TSFileConfig.STRING_CHARSET);
}
public static Binary valueOf(String value) {
@@ -105,16 +104,13 @@ public class Binary implements Comparable<Binary>, Serializable {
}
public String getStringValue() {
- return new String(this.values, Charset.forName(this.textEncodingType));
+ return new String(this.values, TSFileConfig.STRING_CHARSET);
}
public String getTextEncodingType() {
- return textEncodingType;
+ return TSFileConfig.STRING_ENCODING;
}
- public void setTextEncodingType(String textEncodingType) {
- this.textEncodingType = textEncodingType;
- }
@Override
public String toString() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
index 990dba9..20c6a58 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java
@@ -567,12 +567,7 @@ public class BytesUtils {
* @return byte array
*/
public static byte[] stringToBytes(String str) {
- try {
- return str.getBytes(TSFileConfig.STRING_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("catch UnsupportedEncodingException {}", str, e);
- return null;
- }
+ return str.getBytes(TSFileConfig.STRING_CHARSET);
}
/**
@@ -582,12 +577,7 @@ public class BytesUtils {
* @return string
*/
public static String bytesToString(byte[] byteStr) {
- try {
- return new String(byteStr, TSFileConfig.STRING_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("catch UnsupportedEncodingException {}", byteStr, e);
- return null;
- }
+ return new String(byteStr, TSFileConfig.STRING_CHARSET);
}
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java
index 8212db1..bbe2179 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.utils;
import java.util.ArrayList;
import java.util.List;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
/**
* this class is used to contact String effectively.It contains a StringBuider and initialize it