You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ha...@apache.org on 2026/02/12 08:40:07 UTC
(tsfile) branch rc/2.2.1 created (now 8f19e6f3)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
at 8f19e6f3 update cpp py version
This branch includes the following new commits:
new e19b8ae2 perf: Optimize aligned object memory size calculation (#577)
new add5b3b3 Alter column other type to string type (#588)
new 85b0ded6 Fix camel encoding error message and add check for each datatypes of supported encoding (#591)
new eef9ad8c Modify the TsFileSequenceReaderTimeseriesMetadataIterator next function to return a LinkedHashMap (#593)
new ec219619 add encrypt param for all the write/read struct initialization (#601)
new 8f19e6f3 update cpp py version
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tsfile) 06/06: update cpp py version
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 8f19e6f3bdb8f768c0cd69f126d565f94abebb9f
Author: HTHou <ha...@apache.org>
AuthorDate: Thu Feb 12 16:38:34 2026 +0800
update cpp py version
---
cpp/CMakeLists.txt | 2 +-
python/setup.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index fff47e3b..12022634 100755
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -27,7 +27,7 @@ else()
endif ()
cmake_policy(SET CMP0079 NEW)
-set(TsFile_CPP_VERSION 2.2.0)
+set(TsFile_CPP_VERSION 2.2.1)
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall")
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
diff --git a/python/setup.py b/python/setup.py
index 6e77474f..56ee52ef 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -25,7 +25,7 @@ from Cython.Build import cythonize
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
-version = "2.2.0"
+version = "2.2.1"
system = platform.system()
(tsfile) 05/06: add encrypt param for all the write/read struct initialization (#601)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit ec219619e8bf88dda0f80b4fa2ec04ebb5d2a65d
Author: jintao zhu <10...@users.noreply.github.com>
AuthorDate: Fri Oct 10 19:02:11 2025 +0800
add encrypt param for all the write/read struct initialization (#601)
* add encrypt param for all the write/read struct initialization
* code review
---
.../org/apache/tsfile/encrypt/EncryptUtils.java | 53 ++++++++--
.../tsfile/file/metadata/TsFileMetadata.java | 44 ++++----
.../apache/tsfile/read/TsFileRestorableReader.java | 30 +++++-
.../apache/tsfile/read/TsFileSequenceReader.java | 99 +++++++++++++++++-
.../apache/tsfile/read/UnClosedTsFileReader.java | 18 ++--
.../java/org/apache/tsfile/write/TsFileWriter.java | 111 ++++++++++++---------
.../write/chunk/AlignedChunkGroupWriterImpl.java | 6 +-
.../write/v4/AbstractTableModelTsFileWriter.java | 71 +++++--------
.../write/writer/ForceAppendTsFileWriter.java | 14 ++-
.../write/writer/RestorableTsFileIOWriter.java | 34 ++++++-
.../apache/tsfile/write/writer/TsFileIOWriter.java | 85 ++++++++++------
11 files changed, 386 insertions(+), 179 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
index 97b7ad57..ae75d88b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
@@ -35,6 +35,7 @@ import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
public class EncryptUtils {
@@ -46,6 +47,9 @@ public class EncryptUtils {
private static volatile EncryptParameter encryptParam;
+ private static ConcurrentHashMap<EncryptParameter, EncryptParameter> encryptParamCache =
+ new ConcurrentHashMap<>();
+
private static final String HMAC_ALGORITHM = "HmacSHA256";
private static final int ITERATION_COUNT = 1024;
private static final int SALT_LENGTH = 16;
@@ -199,13 +203,23 @@ public class EncryptUtils {
}
md.update("IoTDB is the best".getBytes());
md.update(conf.getEncryptKey());
- byte[] data_key = Arrays.copyOfRange(md.digest(), 0, 16);
- data_key =
- IEncryptor.getEncryptor(conf.getEncryptType(), conf.getEncryptKey()).encrypt(data_key);
+ byte[] dataKey = Arrays.copyOfRange(md.digest(), 0, 16);
+ dataKey = IEncryptor.getEncryptor(conf.getEncryptType(), conf.getEncryptKey()).encrypt(dataKey);
+
+ StringBuilder valueStr = new StringBuilder();
+
+ for (byte b : dataKey) {
+ valueStr.append(b).append(",");
+ }
+
+ valueStr.deleteCharAt(valueStr.length() - 1);
+ return valueStr.toString();
+ }
+ public static String getKeyStr(byte[] key) {
StringBuilder valueStr = new StringBuilder();
- for (byte b : data_key) {
+ for (byte b : key) {
valueStr.append(b).append(",");
}
@@ -213,23 +227,44 @@ public class EncryptUtils {
return valueStr.toString();
}
+ /** Get the second EncryptParameter object according to the config file. */
public static EncryptParameter getEncryptParameter() {
if (encryptParam == null) {
synchronized (EncryptUtils.class) {
if (encryptParam == null) {
encryptParam = getEncryptParameter(TSFileDescriptor.getInstance().getConfig());
+ if (!encryptParamCache.containsKey(encryptParam)) {
+ encryptParamCache.put(
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()),
+ encryptParam);
+ }
}
}
}
return encryptParam;
}
+ /** Get the second EncryptParameter object according to the given type and first key. */
+ public static EncryptParameter getEncryptParameter(EncryptParameter param) {
+ return encryptParamCache.computeIfAbsent(param, EncryptUtils::generateEncryptParameter);
+ }
+
public static EncryptParameter getEncryptParameter(TSFileConfig conf) {
- String encryptType;
+ return generateEncryptParameter(
+ new EncryptParameter(conf.getEncryptType(), conf.getEncryptKey()));
+ }
+
+ /**
+ * Given a main EncryptParameter object, return a second EncryptParameter object with the same
+ * type but the data key generated from the given key.
+ */
+ private static EncryptParameter generateEncryptParameter(EncryptParameter param) {
+ String encryptType = param.getType();
byte[] dataEncryptKey;
- if (!Objects.equals(conf.getEncryptType(), "UNENCRYPTED")
- && !Objects.equals(conf.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
- encryptType = conf.getEncryptType();
+ if (!Objects.equals(encryptType, "UNENCRYPTED")
+ && !Objects.equals(encryptType, "org.apache.tsfile.encrypt.UNENCRYPTED")) {
final MessageDigest md;
try {
md = MessageDigest.getInstance("SHA-256");
@@ -238,7 +273,7 @@ public class EncryptUtils {
"SHA-256 algorithm not found while using SHA-256 to generate data key", e);
}
md.update("IoTDB is the best".getBytes());
- md.update(conf.getEncryptKey());
+ md.update(param.getKey());
dataEncryptKey = Arrays.copyOfRange(md.digest(), 0, 16);
} else {
encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED";
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index 5c77055c..e6a72879 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,11 +19,8 @@
package org.apache.tsfile.file.metadata;
-import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.compatibility.DeserializeConfig;
-import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.encrypt.EncryptUtils;
-import org.apache.tsfile.encrypt.IDecryptor;
import org.apache.tsfile.exception.encrypt.EncryptException;
import org.apache.tsfile.utils.BloomFilter;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
@@ -35,7 +32,6 @@ import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Objects;
import java.util.TreeMap;
/** TSFileMetaData collects all metadata info and saves in its data structure. */
@@ -55,7 +51,9 @@ public class TsFileMetadata {
// offset from MetaMarker.SEPARATOR (exclusive) to tsFileProperties
private int propertiesOffset;
- private byte[] dataEncryptKey;
+ private int encryptLevel;
+
+ private byte[] secondKey;
private String encryptType;
@@ -143,7 +141,8 @@ public class TsFileMetadata {
throw new EncryptException("TsfileMetadata null encryptKey while encryptLevel is 1");
}
String str = propertiesMap.get("encryptKey");
- fileMetaData.dataEncryptKey = EncryptUtils.getSecondKeyFromStr(str);
+ fileMetaData.encryptLevel = 1;
+ fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str);
fileMetaData.encryptType = propertiesMap.get("encryptType");
} else if (propertiesMap.get("encryptLevel").equals("2")) {
if (!propertiesMap.containsKey("encryptType")) {
@@ -155,19 +154,9 @@ public class TsFileMetadata {
if (propertiesMap.get("encryptKey") == null || propertiesMap.get("encryptKey").isEmpty()) {
throw new EncryptException("TsfileMetadata null encryptKey while encryptLevel is 2");
}
- if (Objects.equals(
- TSFileDescriptor.getInstance().getConfig().getEncryptType(),
- "org.apache.tsfile.encrypt.UNENCRYPTED")
- || Objects.equals(
- TSFileDescriptor.getInstance().getConfig().getEncryptType(), "UNENCRYPTED")) {
- throw new EncryptException("fail to decrypt encrypted tsfile in unencrypted system");
- }
- IDecryptor decryptor =
- IDecryptor.getDecryptor(
- propertiesMap.get("encryptType"),
- TSFileDescriptor.getInstance().getConfig().getEncryptKey());
+ fileMetaData.encryptLevel = 2;
String str = propertiesMap.get("encryptKey");
- fileMetaData.dataEncryptKey = decryptor.decrypt(EncryptUtils.getSecondKeyFromStr(str));
+ fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str);
fileMetaData.encryptType = propertiesMap.get("encryptType");
} else {
throw new EncryptException(
@@ -179,13 +168,6 @@ public class TsFileMetadata {
return fileMetaData;
}
- public EncryptParameter getEncryptParam() {
- if (dataEncryptKey == null) {
- return new EncryptParameter("org.apache.tsfile.encrypt.UNENCRYPTED", null);
- }
- return new EncryptParameter(encryptType, dataEncryptKey);
- }
-
public void addProperty(String key, String value) {
if (tsFileProperties == null) {
tsFileProperties = new HashMap<>();
@@ -193,6 +175,18 @@ public class TsFileMetadata {
tsFileProperties.put(key, value);
}
+ public String getEncryptType() {
+ return encryptType;
+ }
+
+ public byte[] getSecondKey() {
+ return secondKey;
+ }
+
+ public int getEncryptLevel() {
+ return encryptLevel;
+ }
+
public BloomFilter getBloomFilter() {
return bloomFilter;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java
index 147d06e9..b593e262 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java
@@ -19,6 +19,8 @@
package org.apache.tsfile.read;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.writer.RestorableTsFileIOWriter;
@@ -33,17 +35,36 @@ public class TsFileRestorableReader extends TsFileSequenceReader {
private static final Logger logger = LoggerFactory.getLogger(TsFileRestorableReader.class);
public TsFileRestorableReader(String file) throws IOException {
- this(file, true);
+ this(
+ file,
+ true,
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()));
+ }
+
+ public TsFileRestorableReader(String file, EncryptParameter param) throws IOException {
+ this(file, true, param);
}
public TsFileRestorableReader(String file, boolean autoRepair) throws IOException {
+ this(
+ file,
+ autoRepair,
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()));
+ }
+
+ public TsFileRestorableReader(String file, boolean autoRepair, EncryptParameter param)
+ throws IOException {
// if autoRepair == true, then it means the file is likely broken, so we can not
// read metadata
// otherwise, the user may consider that either the file is complete, or the
// user can accept an
// Exception when reading broken data. Therefore, we set loadMetadata as true in
// this case.
- super(file, !autoRepair);
+ super(file, param, !autoRepair);
if (autoRepair) {
try {
checkAndRepair();
@@ -62,8 +83,9 @@ public class TsFileRestorableReader extends TsFileSequenceReader {
// Try to close it
logger.info("File {} has no correct tail magic, try to repair...", file);
try (RestorableTsFileIOWriter rWriter =
- new RestorableTsFileIOWriter(FSFactoryProducer.getFSFactory().getFile(file));
- TsFileWriter writer = new TsFileWriter(rWriter)) {
+ new RestorableTsFileIOWriter(
+ FSFactoryProducer.getFSFactory().getFile(file), getFirstEncryptParam());
+ TsFileWriter writer = new TsFileWriter(rWriter, getFirstEncryptParam())) {
// This writes the right magic string
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 4330a600..bde4387c 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -140,6 +140,11 @@ public class TsFileSequenceReader implements AutoCloseable {
private DeserializeConfig deserializeConfig = new DeserializeConfig();
private volatile boolean cacheTableSchemaMap = false;
+ private EncryptParameter firstEncryptParam =
+ new EncryptParameter(config.getEncryptType(), config.getEncryptKey());
+
+ private EncryptParameter dataEncryptParam = null;
+
/**
* Create a file reader of the given file. The reader will read the tail of the file to get the
* file metadata size.Then the reader will skip the first
@@ -150,7 +155,12 @@ public class TsFileSequenceReader implements AutoCloseable {
* @throws IOException If some I/O error occurs
*/
public TsFileSequenceReader(String file) throws IOException {
- this(file, null);
+ this(file, true, null);
+ }
+
+ public TsFileSequenceReader(String file, EncryptParameter firstEncryptParam) throws IOException {
+ this(file, true, null);
+ this.firstEncryptParam = firstEncryptParam;
}
/**
@@ -167,6 +177,13 @@ public class TsFileSequenceReader implements AutoCloseable {
this(file, true, ioSizeRecorder);
}
+ public TsFileSequenceReader(
+ String file, LongConsumer ioSizeRecorder, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(file, true, ioSizeRecorder);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
/**
* construct function for TsFileSequenceReader.
*
@@ -177,6 +194,13 @@ public class TsFileSequenceReader implements AutoCloseable {
this(file, loadMetadataSize, null);
}
+ public TsFileSequenceReader(
+ String file, EncryptParameter firstEncryptParam, boolean loadMetadataSize)
+ throws IOException {
+ this(file, loadMetadataSize, null);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
/**
* construct function for TsFileSequenceReader.
*
@@ -203,6 +227,16 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
+ public TsFileSequenceReader(
+ String file,
+ boolean loadMetadataSize,
+ LongConsumer ioSizeRecorder,
+ EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(file, loadMetadataSize, ioSizeRecorder);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
// used in merge resource
public TsFileSequenceReader(String file, boolean loadMetadata, boolean cacheDeviceMetadata)
throws IOException {
@@ -210,6 +244,16 @@ public class TsFileSequenceReader implements AutoCloseable {
this.cacheDeviceMetadata = cacheDeviceMetadata;
}
+ public TsFileSequenceReader(
+ String file,
+ boolean loadMetadata,
+ boolean cacheDeviceMetadata,
+ EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(file, loadMetadata, cacheDeviceMetadata);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
/**
* Create a file reader of the given file. The reader will read the tail of the file to get the
* file metadata size.Then the reader will skip the first
@@ -222,6 +266,12 @@ public class TsFileSequenceReader implements AutoCloseable {
this(input, true);
}
+ public TsFileSequenceReader(TsFileInput input, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(input, true);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
/**
* construct function for TsFileSequenceReader.
*
@@ -241,6 +291,13 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
+ public TsFileSequenceReader(
+ TsFileInput input, boolean loadMetadataSize, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(input, loadMetadataSize);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
/**
* construct function for TsFileSequenceReader.
*
@@ -256,6 +313,15 @@ public class TsFileSequenceReader implements AutoCloseable {
this.fileMetadataSize = fileMetadataSize;
}
+ public TsFileSequenceReader(
+ TsFileInput input,
+ long fileMetadataPos,
+ int fileMetadataSize,
+ EncryptParameter firstEncryptParam) {
+ this(input, fileMetadataPos, fileMetadataSize);
+ this.firstEncryptParam = firstEncryptParam;
+ }
+
// ioSizeRecorder can be null
private void loadFileVersion(LongConsumer ioSizeRecorder) throws IOException {
try {
@@ -478,11 +544,34 @@ public class TsFileSequenceReader implements AutoCloseable {
* @param ioSizeRecorder can be null
*/
public EncryptParameter getEncryptParam(LongConsumer ioSizeRecorder) throws IOException {
- if (fileMetadataSize != 0) {
- readFileMetadata(ioSizeRecorder);
- return tsFileMetaData.getEncryptParam();
+ if (dataEncryptParam != null) {
+ return dataEncryptParam;
+ } else {
+ if (fileMetadataSize != 0) {
+ readFileMetadata(ioSizeRecorder);
+ int encryptLevel = tsFileMetaData.getEncryptLevel();
+ byte[] secondKey = tsFileMetaData.getSecondKey();
+ String encryptType = tsFileMetaData.getEncryptType();
+ if (secondKey == null) {
+ dataEncryptParam = new EncryptParameter("org.apache.tsfile.encrypt.UNENCRYPTED", null);
+ return dataEncryptParam;
+ }
+ if (encryptLevel == 1) {
+ dataEncryptParam = new EncryptParameter(encryptType, secondKey);
+ return dataEncryptParam;
+ } else if (encryptLevel == 2) {
+ IDecryptor decryptor = IDecryptor.getDecryptor(firstEncryptParam);
+ byte[] dataEncryptKey = decryptor.decrypt(secondKey);
+ dataEncryptParam = new EncryptParameter(encryptType, dataEncryptKey);
+ return dataEncryptParam;
+ }
+ }
+ return EncryptUtils.getEncryptParameter(firstEncryptParam);
}
- return EncryptUtils.getEncryptParameter();
+ }
+
+ public EncryptParameter getFirstEncryptParam() {
+ return firstEncryptParam;
}
/**
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java
index ce76fbc8..230a2e16 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.read;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.exception.NotImplementedException;
@@ -29,20 +30,23 @@ import java.util.function.LongConsumer;
/** A class for reading unclosed tsfile. */
public class UnClosedTsFileReader extends TsFileSequenceReader {
-
- private EncryptParameter encryptParam;
+ private final EncryptParameter dataEncryptParam;
// ioSizeRecorder can be null
public UnClosedTsFileReader(String file, LongConsumer ioSizeRecorder) throws IOException {
- super(file, false, ioSizeRecorder);
- encryptParam = EncryptUtils.getEncryptParameter();
+ this(
+ file,
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()),
+ ioSizeRecorder);
}
// ioSizeRecorder can be null
public UnClosedTsFileReader(
String file, EncryptParameter encryptParam, LongConsumer ioSizeRecorder) throws IOException {
- super(file, false, ioSizeRecorder);
- this.encryptParam = encryptParam;
+ super(file, false, ioSizeRecorder, encryptParam);
+ this.dataEncryptParam = EncryptUtils.getEncryptParameter(encryptParam);
}
/** unclosed file has no tail magic data. */
@@ -59,6 +63,6 @@ public class UnClosedTsFileReader extends TsFileSequenceReader {
@Override
public EncryptParameter getEncryptParam() {
- return encryptParam;
+ return dataEncryptParam;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index ff37612e..08ca25ce 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -22,9 +22,9 @@ import org.apache.tsfile.annotations.TsFileApi;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encrypt.EncryptParameter;
+import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.ColumnCategory;
-import org.apache.tsfile.exception.encrypt.EncryptException;
import org.apache.tsfile.exception.write.ConflictDataTypeException;
import org.apache.tsfile.exception.write.NoDeviceException;
import org.apache.tsfile.exception.write.NoMeasurementException;
@@ -54,10 +54,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -79,7 +76,7 @@ public class TsFileWriter implements AutoCloseable {
/** IO writer of this TsFile. */
private final TsFileIOWriter fileWriter;
- private EncryptParameter encryptParam;
+ private EncryptParameter secondEncryptParam;
private final int pageSize;
private long recordCount = 0;
@@ -118,6 +115,14 @@ public class TsFileWriter implements AutoCloseable {
this(new TsFileIOWriter(file), new Schema(), TSFileDescriptor.getInstance().getConfig());
}
+ public TsFileWriter(File file, EncryptParameter firstEncryptParam) throws IOException {
+ this(
+ new TsFileIOWriter(file),
+ new Schema(),
+ TSFileDescriptor.getInstance().getConfig(),
+ firstEncryptParam);
+ }
+
/**
* init this TsFileWriter.
*
@@ -127,6 +132,11 @@ public class TsFileWriter implements AutoCloseable {
this(fileWriter, new Schema(), TSFileDescriptor.getInstance().getConfig());
}
+ public TsFileWriter(TsFileIOWriter fileWriter, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(fileWriter, new Schema(), TSFileDescriptor.getInstance().getConfig(), firstEncryptParam);
+ }
+
/**
* init this TsFileWriter.
*
@@ -137,6 +147,15 @@ public class TsFileWriter implements AutoCloseable {
this(new TsFileIOWriter(file), schema, TSFileDescriptor.getInstance().getConfig());
}
+ public TsFileWriter(File file, Schema schema, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(
+ new TsFileIOWriter(file),
+ schema,
+ TSFileDescriptor.getInstance().getConfig(),
+ firstEncryptParam);
+ }
+
/**
* init this TsFileWriter.
*
@@ -147,6 +166,15 @@ public class TsFileWriter implements AutoCloseable {
this(new TsFileIOWriter(output), schema, TSFileDescriptor.getInstance().getConfig());
}
+ public TsFileWriter(TsFileOutput output, Schema schema, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(
+ new TsFileIOWriter(output),
+ schema,
+ TSFileDescriptor.getInstance().getConfig(),
+ firstEncryptParam);
+ }
+
/**
* init this TsFileWriter.
*
@@ -158,6 +186,12 @@ public class TsFileWriter implements AutoCloseable {
this(new TsFileIOWriter(file), schema, conf);
}
+ public TsFileWriter(
+ File file, Schema schema, TSFileConfig conf, EncryptParameter firstEncryptParam)
+ throws IOException {
+ this(new TsFileIOWriter(file), schema, conf, firstEncryptParam);
+ }
+
/**
* init this TsFileWriter.
*
@@ -167,6 +201,19 @@ public class TsFileWriter implements AutoCloseable {
*/
protected TsFileWriter(TsFileIOWriter fileWriter, Schema schema, TSFileConfig conf)
throws IOException {
+ this(
+ fileWriter,
+ schema,
+ conf,
+ new EncryptParameter(conf.getEncryptType(), conf.getEncryptKey()));
+ }
+
+ protected TsFileWriter(
+ TsFileIOWriter fileWriter,
+ Schema schema,
+ TSFileConfig conf,
+ EncryptParameter firstEncryptParam)
+ throws IOException {
if (!fileWriter.canWrite()) {
throw new IOException(
"the given file Writer does not support writing any more. Maybe it is an complete TsFile");
@@ -188,48 +235,20 @@ public class TsFileWriter implements AutoCloseable {
pageSize,
chunkGroupSizeThreshold);
}
-
+ this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam);
String encryptLevel;
- byte[] encryptKey;
- byte[] dataEncryptKey;
- String encryptType;
- if (!Objects.equals(config.getEncryptType(), "UNENCRYPTED")
- && !Objects.equals(config.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
+ if (firstEncryptParam != null
+ && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED")
+ && !Objects.equals(firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
encryptLevel = "2";
- encryptType = config.getEncryptType();
- final MessageDigest md;
- try {
- md = MessageDigest.getInstance("SHA-256");
- } catch (NoSuchAlgorithmException e) {
- throw new EncryptException(
- "SHA-256 algorithm not found while using SHA-256 to generate data key", e);
- }
- md.update("IoTDB is the best".getBytes());
- md.update(config.getEncryptKey());
- dataEncryptKey = Arrays.copyOfRange(md.digest(), 0, 16);
- encryptKey =
- IEncryptor.getEncryptor(config.getEncryptType(), config.getEncryptKey())
- .encrypt(dataEncryptKey);
+ String str =
+ EncryptUtils.getKeyStr(
+ IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey())
+ .encrypt(secondEncryptParam.getKey()));
+ fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str);
} else {
encryptLevel = "0";
- encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED";
- encryptKey = null;
- dataEncryptKey = null;
- }
- this.encryptParam = new EncryptParameter(encryptType, dataEncryptKey);
- if (encryptKey != null) {
- StringBuilder valueStr = new StringBuilder();
-
- for (byte b : encryptKey) {
- valueStr.append(b).append(",");
- }
-
- valueStr.deleteCharAt(valueStr.length() - 1);
- String str = valueStr.toString();
-
- fileWriter.setEncryptParam(encryptLevel, encryptType, str);
- } else {
- fileWriter.setEncryptParam(encryptLevel, encryptType, "");
+ fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", "");
}
}
@@ -511,8 +530,8 @@ public class TsFileWriter implements AutoCloseable {
if (isAligned) {
groupWriter =
isTableModel
- ? new TableChunkGroupWriterImpl(deviceId, encryptParam)
- : new AlignedChunkGroupWriterImpl(deviceId, encryptParam);
+ ? new TableChunkGroupWriterImpl(deviceId, secondEncryptParam)
+ : new AlignedChunkGroupWriterImpl(deviceId, secondEncryptParam);
initAllSeriesWriterForAlignedSeries(
(AlignedChunkGroupWriterImpl) groupWriter, deviceId, isTableModel);
if (!isUnseq) { // Sequence File
@@ -520,7 +539,7 @@ public class TsFileWriter implements AutoCloseable {
.setLastTime(alignedDeviceLastTimeMap.get(deviceId));
}
} else {
- groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, encryptParam);
+ groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, secondEncryptParam);
if (!isUnseq) { // Sequence File
((NonAlignedChunkGroupWriterImpl) groupWriter)
.setLastTimeMap(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index a47df975..3691f737 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -113,7 +113,8 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter {
measurementSchema.getCompressor(),
measurementSchema.getType(),
measurementSchema.getEncodingType(),
- measurementSchema.getValueEncoder());
+ measurementSchema.getValueEncoder(),
+ this.encryprParam);
valueChunkWriterMap.put(measurementName, valueChunkWriter);
tryToAddEmptyPageAndData(valueChunkWriter);
}
@@ -134,7 +135,8 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter {
schema.getCompressor(),
schema.getType(),
schema.getEncodingType(),
- schema.getValueEncoder());
+ schema.getValueEncoder(),
+ this.encryprParam);
valueChunkWriterMap.put(measurementName, valueChunkWriter);
tryToAddEmptyPageAndData(valueChunkWriter);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
index 3120bb40..260d4264 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
@@ -23,8 +23,8 @@ import org.apache.tsfile.annotations.TsFileApi;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encrypt.EncryptParameter;
+import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.encrypt.IEncryptor;
-import org.apache.tsfile.exception.encrypt.EncryptException;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl;
import org.apache.tsfile.write.chunk.IChunkGroupWriter;
@@ -38,10 +38,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -56,7 +53,7 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter {
/** IO writer of this TsFile. */
protected final TsFileIOWriter fileWriter;
- protected EncryptParameter encryptParam;
+ protected EncryptParameter secondEncryptParam;
protected final int pageSize;
protected long recordCount = 0;
@@ -85,6 +82,16 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter {
@TsFileApi
protected AbstractTableModelTsFileWriter(File file, long chunkGroupSizeThreshold)
throws IOException {
+ this(
+ file,
+ chunkGroupSizeThreshold,
+ new EncryptParameter(config.getEncryptType(), config.getEncryptKey()));
+ }
+
+ @TsFileApi
+ protected AbstractTableModelTsFileWriter(
+ File file, long chunkGroupSizeThreshold, EncryptParameter firstEncryptParam)
+ throws IOException {
Schema schema = new Schema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
this.fileWriter = new TsFileIOWriter(file);
@@ -100,48 +107,20 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter {
chunkGroupSizeThreshold);
}
+ this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam);
String encryptLevel;
- byte[] encryptKey;
- byte[] dataEncryptKey;
- String encryptType;
- if (!Objects.equals(config.getEncryptType(), "UNENCRYPTED")
- && !Objects.equals(config.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
+ if (firstEncryptParam != null
+ && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED")
+ && !Objects.equals(firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
encryptLevel = "2";
- encryptType = config.getEncryptType();
-
- final MessageDigest md;
- try {
- md = MessageDigest.getInstance("SHA-256");
- } catch (NoSuchAlgorithmException e) {
- throw new EncryptException(
- "SHA-256 algorithm not found while using SHA-256 to generate data key", e);
- }
- md.update("IoTDB is the best".getBytes());
- md.update(config.getEncryptKey());
- dataEncryptKey = Arrays.copyOfRange(md.digest(), 0, 16);
- encryptKey =
- IEncryptor.getEncryptor(config.getEncryptType(), config.getEncryptKey())
- .encrypt(dataEncryptKey);
+ String str =
+ EncryptUtils.getKeyStr(
+ IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey())
+ .encrypt(secondEncryptParam.getKey()));
+ fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str);
} else {
encryptLevel = "0";
- encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED";
- encryptKey = null;
- dataEncryptKey = null;
- }
- this.encryptParam = new EncryptParameter(encryptType, dataEncryptKey);
- if (encryptKey != null) {
- StringBuilder valueStr = new StringBuilder();
-
- for (byte b : encryptKey) {
- valueStr.append(b).append(",");
- }
-
- valueStr.deleteCharAt(valueStr.length() - 1);
- String str = valueStr.toString();
-
- fileWriter.setEncryptParam(encryptLevel, encryptType, str);
- } else {
- fileWriter.setEncryptParam(encryptLevel, encryptType, "");
+ fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", "");
}
}
@@ -152,13 +131,13 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter {
if (isAligned) {
groupWriter =
isTableModel
- ? new TableChunkGroupWriterImpl(deviceId, encryptParam)
- : new AlignedChunkGroupWriterImpl(deviceId, encryptParam);
+ ? new TableChunkGroupWriterImpl(deviceId, secondEncryptParam)
+ : new AlignedChunkGroupWriterImpl(deviceId, secondEncryptParam);
((AlignedChunkGroupWriterImpl) groupWriter)
.setLastTime(alignedDeviceLastTimeMap.get(deviceId));
initAllSeriesWriterForAlignedSeries((AlignedChunkGroupWriterImpl) groupWriter);
} else {
- groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, encryptParam);
+ groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, secondEncryptParam);
((NonAlignedChunkGroupWriterImpl) groupWriter)
.setLastTimeMap(
nonAlignedTimeseriesLastTimeMap.getOrDefault(deviceId, new HashMap<>()));
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java
index 9f2de4e8..241514bd 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -18,6 +18,8 @@
*/
package org.apache.tsfile.write.writer;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.exception.write.TsFileNotCompleteException;
import org.apache.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
@@ -44,18 +46,28 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
private static Logger logger = LoggerFactory.getLogger(ForceAppendTsFileWriter.class);
public ForceAppendTsFileWriter(File file) throws IOException {
+ this(
+ file,
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()));
+ }
+
+ public ForceAppendTsFileWriter(File file, EncryptParameter param) throws IOException {
if (logger.isDebugEnabled()) {
logger.debug("{} writer is opened.", file.getName());
}
this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true);
this.file = file;
+ setEncryptParam(param);
// file doesn't exist
if (file.length() == 0 || !file.exists()) {
throw new TsFileNotCompleteException("File " + file.getPath() + " is not a complete TsFile");
}
- try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) {
+ try (TsFileSequenceReader reader =
+ new TsFileSequenceReader(file.getAbsolutePath(), param, true)) {
// this tsfile is not complete
if (!reader.isComplete()) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java
index e01b741c..351fe58a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -19,6 +19,8 @@
package org.apache.tsfile.write.writer;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.NotCompatibleTsFileException;
import org.apache.tsfile.file.metadata.ChunkGroupMetadata;
@@ -72,6 +74,8 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
private final Map<IDeviceID, Map<String, List<ChunkMetadata>>> metadatasForQuery =
new HashMap<>();
+ private EncryptParameter param;
+
/**
* @param file a given tsfile path you want to (continue to) write
* @throws IOException if write failed, or the file is broken but autoRepair==false.
@@ -80,23 +84,49 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
this(file, true);
}
+ public RestorableTsFileIOWriter(File file, EncryptParameter param) throws IOException {
+ this(file, true, param);
+ }
+
/**
* @param file a given tsfile path you want to (continue to) write
* @throws IOException if write failed, or the file is broken but autoRepair==false.
*/
public RestorableTsFileIOWriter(File file, long maxMetadataSize) throws IOException {
- this(file, true);
+ this(
+ file,
+ true,
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()));
+ }
+
+ public RestorableTsFileIOWriter(File file, long maxMetadataSize, EncryptParameter param)
+ throws IOException {
+ this(file, true, param);
this.maxMetadataSize = maxMetadataSize;
this.chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX);
this.checkMetadataSizeAndMayFlush();
}
public RestorableTsFileIOWriter(File file, boolean truncate) throws IOException {
+ this(
+ file,
+ truncate,
+ new EncryptParameter(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey()));
+ }
+
+ public RestorableTsFileIOWriter(File file, boolean truncate, EncryptParameter param)
+ throws IOException {
if (logger.isDebugEnabled()) {
logger.debug("{} is opened.", file.getName());
}
this.file = file;
this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true);
+ this.param = param;
+ setEncryptParam(param);
// file doesn't exist
if (file.length() == 0) {
@@ -109,7 +139,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
try {
if (file.exists()) {
try (TsFileSequenceReader reader =
- new TsFileSequenceReader(file.getAbsolutePath(), false)) {
+ new TsFileSequenceReader(file.getAbsolutePath(), param, false)) {
schema.setEnabledUpdateSchema(false);
truncatedSize = reader.selfCheck(schema, chunkGroupMetadataList, true);
minPlanIndex = reader.getMinPlanIndex();
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index 31a6af52..8ac5eb2b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -21,7 +21,9 @@ package org.apache.tsfile.write.writer;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkGroupHeader;
@@ -136,17 +138,12 @@ public class TsFileIOWriter implements AutoCloseable {
/** empty construct function. */
protected TsFileIOWriter() {
- if (!Objects.equals(TS_FILE_CONFIG.getEncryptType(), "UNENCRYPTED")
- && !Objects.equals(
- TS_FILE_CONFIG.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
- this.encryptLevel = "2";
- this.encryptType = TS_FILE_CONFIG.getEncryptType();
- this.encryptKey = EncryptUtils.getNormalKeyStr();
- } else {
- this.encryptLevel = "0";
- this.encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED";
- this.encryptKey = null;
- }
+ setEncryptParam(
+ new EncryptParameter(TS_FILE_CONFIG.getEncryptType(), TS_FILE_CONFIG.getEncryptKey()));
+ }
+
+ protected TsFileIOWriter(EncryptParameter param) {
+ setEncryptParam(param);
}
/**
@@ -159,23 +156,22 @@ public class TsFileIOWriter implements AutoCloseable {
this(file, TS_FILE_CONFIG);
}
+ public TsFileIOWriter(File file, EncryptParameter param) throws IOException {
+ this(file, TS_FILE_CONFIG, param);
+ }
+
/** for test only */
public TsFileIOWriter(File file, TSFileConfig conf) throws IOException {
+ this(file, conf, new EncryptParameter(conf.getEncryptType(), conf.getEncryptKey()));
+ }
+
+ public TsFileIOWriter(File file, TSFileConfig conf, EncryptParameter param) throws IOException {
this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), false);
this.file = file;
if (resourceLogger.isDebugEnabled()) {
resourceLogger.debug("{} writer is opened.", file.getName());
}
- if (!Objects.equals(conf.getEncryptType(), "UNENCRYPTED")
- && !Objects.equals(conf.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
- this.encryptLevel = "2";
- this.encryptType = conf.getEncryptType();
- this.encryptKey = EncryptUtils.getNormalKeyStr();
- } else {
- this.encryptLevel = "0";
- this.encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED";
- this.encryptKey = null;
- }
+ setEncryptParam(param);
startFile();
}
@@ -186,23 +182,22 @@ public class TsFileIOWriter implements AutoCloseable {
*/
public TsFileIOWriter(TsFileOutput output) throws IOException {
this.out = output;
- if (!Objects.equals(TS_FILE_CONFIG.getEncryptType(), "UNENCRYPTED")
- && !Objects.equals(
- TS_FILE_CONFIG.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
- this.encryptLevel = "2";
- this.encryptType = TS_FILE_CONFIG.getEncryptType();
- this.encryptKey = EncryptUtils.getNormalKeyStr();
- } else {
- this.encryptLevel = "0";
- this.encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED";
- this.encryptKey = null;
- }
+ setEncryptParam(
+ new EncryptParameter(TS_FILE_CONFIG.getEncryptType(), TS_FILE_CONFIG.getEncryptKey()));
+ startFile();
+ }
+
+ public TsFileIOWriter(TsFileOutput output, EncryptParameter param) throws IOException {
+ this.out = output;
+ setEncryptParam(param);
startFile();
}
/** for test only */
public TsFileIOWriter(TsFileOutput output, boolean test) {
this.out = output;
+ setEncryptParam(
+ new EncryptParameter(TS_FILE_CONFIG.getEncryptType(), TS_FILE_CONFIG.getEncryptKey()));
}
/** for write with memory control */
@@ -212,12 +207,38 @@ public class TsFileIOWriter implements AutoCloseable {
chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX);
}
+ public TsFileIOWriter(File file, long maxMetadataSize, EncryptParameter param)
+ throws IOException {
+ this(file, param);
+ this.maxMetadataSize = maxMetadataSize;
+ chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX);
+ }
+
public void setEncryptParam(String encryptLevel, String encryptType, String encryptKey) {
this.encryptLevel = encryptLevel;
this.encryptType = encryptType;
this.encryptKey = encryptKey;
}
+ public void setEncryptParam(EncryptParameter param) {
+ if (param == null) {
+ setEncryptParam("0", "org.apache.tsfile.encrypt.UNENCRYPTED", null);
+ } else {
+ if (!Objects.equals(param.getType(), "UNENCRYPTED")
+ && !Objects.equals(param.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) {
+ String encryptLevel = "2";
+ String encryptType = param.getType();
+ String encryptKey =
+ EncryptUtils.getKeyStr(
+ IEncryptor.getEncryptor(param.getType(), param.getKey())
+ .encrypt(EncryptUtils.getEncryptParameter(param).getKey()));
+ setEncryptParam(encryptLevel, encryptType, encryptKey);
+ } else {
+ setEncryptParam("0", "org.apache.tsfile.encrypt.UNENCRYPTED", null);
+ }
+ }
+ }
+
public void addFlushListener(FlushChunkMetadataListener listener) {
flushListeners.add(listener);
}
(tsfile) 01/06: perf: Optimize aligned object memory size calculation (#577)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit e19b8ae25d2be166c504578f207d7bd3321bef4e
Author: Zhenyu Luo <zh...@apache.org>
AuthorDate: Wed Aug 27 10:35:47 2025 +0800
perf: Optimize aligned object memory size calculation (#577)
* Optimize aligned object memory size calculation
* spotless
---
.../src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
index 404af510..af7a8cff 100644
--- a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
+++ b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
@@ -82,6 +82,8 @@ public final class RamUsageEstimator {
*/
public static final int NUM_BYTES_OBJECT_ALIGNMENT;
+ private static final int ALIGN_MASK;
+
/**
* Approximate memory usage that we assign to all unknown queries - this maps roughly to a
* BooleanQuery with a couple term clauses.
@@ -180,6 +182,8 @@ public final class RamUsageEstimator {
NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + Integer.BYTES;
}
+ ALIGN_MASK = NUM_BYTES_OBJECT_ALIGNMENT - 1;
+
// get min/max value of cached Long class instances:
long longCacheMinValue = 0;
while (longCacheMinValue > Long.MIN_VALUE
@@ -209,8 +213,7 @@ public final class RamUsageEstimator {
/** Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}. */
public static long alignObjectSize(long size) {
- size += NUM_BYTES_OBJECT_ALIGNMENT - 1L;
- return size - (size % NUM_BYTES_OBJECT_ALIGNMENT);
+ return (size + ALIGN_MASK) & ~ALIGN_MASK;
}
/**
(tsfile) 03/06: Fix camel encoding error message and add check for each datatypes of supported encoding (#591)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 85b0ded63506d8b63716c156aa7aa76d0b21eeb5
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Sep 18 18:40:37 2025 +0800
Fix camel encoding error message and add check for each datatypes of supported encoding (#591)
* Fix camel encoding error message and add check for each datatypes of supported encoding
* Fix UT
---
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 55 +++++++++---------
.../tsfile/file/metadata/enums/TSEncoding.java | 7 ++-
.../encoding/encoder/TSEncodingBuilderTest.java | 65 ++++++++++++++++++++++
.../apache/tsfile/write/TsFileReadWriteTest.java | 10 +---
4 files changed, 102 insertions(+), 35 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java
index 7849607c..38b0731d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -30,9 +30,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
+import java.util.Objects;
/**
- * Each subclass of TSEncodingBuilder responds a enumerate value in {@linkplain TSEncoding
+ * Each subclass of TSEncodingBuilder responds an enumerate value in {@linkplain TSEncoding
* TSEncoding}, which stores several configuration related to responding encoding type to generate
* {@linkplain Encoder Encoder} instance.<br>
* Each TSEncoding has a responding TSEncodingBuilder. The design referring to visit pattern
@@ -43,6 +44,7 @@ public abstract class TSEncodingBuilder {
private static final Logger logger = LoggerFactory.getLogger(TSEncodingBuilder.class);
protected final TSFileConfig conf;
+ private static final String ERROR_MSG = "%s doesn't support data type: %s";
protected TSEncodingBuilder() {
this.conf = TSFileDescriptor.getInstance().getConfig();
@@ -64,8 +66,6 @@ public abstract class TSEncodingBuilder {
return new Ts2Diff();
case GORILLA_V1:
return new GorillaV1();
- case REGULAR:
- return new Regular();
case GORILLA:
return new GorillaV2();
case DICTIONARY:
@@ -81,7 +81,7 @@ public abstract class TSEncodingBuilder {
case CAMEL:
return new Camel();
default:
- throw new UnsupportedOperationException(type.toString());
+ throw new UnsupportedOperationException("Unsupported encoding: " + type);
}
}
@@ -124,7 +124,7 @@ public abstract class TSEncodingBuilder {
if (props == null || !props.containsKey(Encoder.MAX_STRING_LENGTH)) {
maxStringLength = TSFileDescriptor.getInstance().getConfig().getMaxStringLength();
} else {
- maxStringLength = Integer.valueOf(props.get(Encoder.MAX_STRING_LENGTH));
+ maxStringLength = Integer.parseInt(props.get(Encoder.MAX_STRING_LENGTH));
if (maxStringLength < 0) {
maxStringLength = TSFileDescriptor.getInstance().getConfig().getMaxStringLength();
logger.warn(
@@ -154,7 +154,7 @@ public abstract class TSEncodingBuilder {
case DOUBLE:
return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber);
default:
- throw new UnSupportedDataTypeException("RLE doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(String.format(ERROR_MSG, TSEncoding.RLE, type));
}
}
@@ -209,15 +209,16 @@ public abstract class TSEncodingBuilder {
case DOUBLE:
return new FloatEncoder(TSEncoding.TS_2DIFF, type, maxPointNumber);
default:
- throw new UnSupportedDataTypeException("TS_2DIFF doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(
+ String.format(ERROR_MSG, TSEncoding.TS_2DIFF, type));
}
}
- @Override
/**
* TS_2DIFF could specify <b>max_point_number</b> in given JSON Object, which means the maximum
* decimal digits for float or double data.
*/
+ @Override
public void initFromProps(Map<String, String> props) {
// set max error from initialized map or default value if not set
if (props == null || !props.containsKey(Encoder.MAX_POINT_NUMBER)) {
@@ -257,13 +258,14 @@ public abstract class TSEncodingBuilder {
case DOUBLE:
return new DoublePrecisionEncoderV1();
default:
- throw new UnSupportedDataTypeException("GORILLA_V1 doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(
+ String.format(ERROR_MSG, TSEncoding.GORILLA_V1, type));
}
}
@Override
public void initFromProps(Map<String, String> props) {
- // allowed do nothing
+ // allowed to do nothing
}
}
@@ -272,17 +274,15 @@ public abstract class TSEncodingBuilder {
@Override
public Encoder getEncoder(TSDataType type) {
- switch (type) {
- case DOUBLE:
- return new CamelEncoder();
- default:
- throw new UnSupportedDataTypeException("GORILLA_V1 doesn't support data type: " + type);
+ if (Objects.requireNonNull(type) == TSDataType.DOUBLE) {
+ return new CamelEncoder();
}
+ throw new UnSupportedDataTypeException(String.format(ERROR_MSG, TSEncoding.CAMEL, type));
}
@Override
public void initFromProps(Map<String, String> props) {
- // allowed do nothing
+ // allowed to do nothing
}
}
@@ -299,13 +299,14 @@ public abstract class TSEncodingBuilder {
case TIMESTAMP:
return new RegularDataEncoder.LongRegularEncoder();
default:
- throw new UnSupportedDataTypeException("REGULAR doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(
+ String.format(ERROR_MSG, TSEncoding.REGULAR, type));
}
}
@Override
public void initFromProps(Map<String, String> props) {
- // allowed do nothing
+ // allowed to do nothing
}
}
@@ -326,13 +327,14 @@ public abstract class TSEncodingBuilder {
case TIMESTAMP:
return new LongGorillaEncoder();
default:
- throw new UnSupportedDataTypeException("GORILLA doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(
+ String.format(ERROR_MSG, TSEncoding.GORILLA, type));
}
}
@Override
public void initFromProps(Map<String, String> props) {
- // allowed do nothing
+ // allowed to do nothing
}
}
@@ -351,7 +353,8 @@ public abstract class TSEncodingBuilder {
case DOUBLE:
return new DoubleSprintzEncoder();
default:
- throw new UnSupportedDataTypeException("Sprintz doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(
+ String.format(ERROR_MSG, TSEncoding.SPRINTZ, type));
}
}
@@ -377,7 +380,7 @@ public abstract class TSEncodingBuilder {
case DOUBLE:
return new DoubleRLBE();
default:
- throw new UnSupportedDataTypeException("RLBE doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(String.format(ERROR_MSG, TSEncoding.RLBE, type));
}
}
@@ -394,7 +397,7 @@ public abstract class TSEncodingBuilder {
if (type == TSDataType.TEXT || type == TSDataType.STRING) {
return new DictionaryEncoder();
}
- throw new UnSupportedDataTypeException("DICTIONARY doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(String.format(ERROR_MSG, TSEncoding.DICTIONARY, type));
}
@Override
@@ -415,7 +418,7 @@ public abstract class TSEncodingBuilder {
case TIMESTAMP:
return new LongZigzagEncoder();
default:
- throw new UnSupportedDataTypeException("ZIGZAG doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(String.format(ERROR_MSG, TSEncoding.ZIGZAG, type));
}
}
@@ -442,13 +445,13 @@ public abstract class TSEncodingBuilder {
case TIMESTAMP:
return new LongChimpEncoder();
default:
- throw new UnSupportedDataTypeException("CHIMP doesn't support data type: " + type);
+ throw new UnSupportedDataTypeException(String.format(ERROR_MSG, TSEncoding.CHIMP, type));
}
}
@Override
public void initFromProps(Map<String, String> props) {
- // allowed do nothing
+ // allowed to do nothing
}
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
index d2ea1931..6c02cb1f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
@@ -79,10 +79,13 @@ public enum TSEncoding {
floatSet.add(TSEncoding.CHIMP);
floatSet.add(TSEncoding.SPRINTZ);
floatSet.add(TSEncoding.RLBE);
- floatSet.add(TSEncoding.CAMEL);
TYPE_SUPPORTED_ENCODINGS.put(TSDataType.FLOAT, floatSet);
- TYPE_SUPPORTED_ENCODINGS.put(TSDataType.DOUBLE, floatSet);
+
+ Set<TSEncoding> doubleSet = new HashSet<>(floatSet);
+ doubleSet.add(TSEncoding.CAMEL);
+
+ TYPE_SUPPORTED_ENCODINGS.put(TSDataType.DOUBLE, doubleSet);
Set<TSEncoding> textSet = new HashSet<>();
textSet.add(TSEncoding.PLAIN);
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilderTest.java
new file mode 100644
index 00000000..7a866a34
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilderTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.encoding.encoder;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.write.UnSupportedDataTypeException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class TSEncodingBuilderTest {
+
+ private static final String ERROR_MSG = "Unsupported dataType: %s doesn't support data type: %s";
+
+ @Test
+ public void testTSEncodingBuilder() {
+ Set<TSDataType> supportedDataTypes =
+ Arrays.stream(TSDataType.values()).collect(Collectors.toSet());
+ supportedDataTypes.remove(TSDataType.VECTOR);
+ supportedDataTypes.remove(TSDataType.UNKNOWN);
+
+ for (TSDataType dataType : supportedDataTypes) {
+ for (TSEncoding encoding : TSEncoding.values()) {
+ if (TSEncoding.isSupported(dataType, encoding)) {
+ try {
+ TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ } catch (UnSupportedDataTypeException e) {
+ Assert.fail(e.getMessage());
+ }
+ } else {
+ try {
+ TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Assert.fail(String.format(ERROR_MSG, encoding, dataType));
+ } catch (UnsupportedOperationException e) {
+ Assert.assertEquals("Unsupported encoding: " + encoding, e.getMessage());
+ } catch (UnSupportedDataTypeException e) {
+ Assert.assertEquals(String.format(ERROR_MSG, encoding, dataType), e.getMessage());
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileReadWriteTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileReadWriteTest.java
index 3d91fffd..015a8fbd 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileReadWriteTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileReadWriteTest.java
@@ -42,6 +42,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@@ -86,7 +87,6 @@ public class TsFileReadWriteTest {
TSEncoding.PLAIN,
TSEncoding.RLE,
TSEncoding.TS_2DIFF,
- TSEncoding.REGULAR,
TSEncoding.GORILLA,
TSEncoding.ZIGZAG);
for (TSEncoding encoding : encodings) {
@@ -102,12 +102,7 @@ public class TsFileReadWriteTest {
@Test
public void longTest() throws IOException, WriteProcessException {
List<TSEncoding> encodings =
- Arrays.asList(
- TSEncoding.PLAIN,
- TSEncoding.RLE,
- TSEncoding.TS_2DIFF,
- TSEncoding.REGULAR,
- TSEncoding.GORILLA);
+ Arrays.asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.TS_2DIFF, TSEncoding.GORILLA);
for (TSEncoding encoding : encodings) {
longTest(encoding);
}
@@ -202,6 +197,7 @@ public class TsFileReadWriteTest {
}
@Test
+ @Ignore
public void readMeasurementWithRegularEncodingTest() throws IOException, WriteProcessException {
TSFileDescriptor.getInstance().getConfig().setTimeEncoder("REGULAR");
writeDataByTSRecord(
(tsfile) 04/06: Modify the TsFileSequenceReaderTimeseriesMetadataIterator next function to return a LinkedHashMap (#593)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit eef9ad8cd7c0f263372de34c5eac1a56979e380b
Author: Zhenyu Luo <zh...@apache.org>
AuthorDate: Sun Sep 28 10:29:05 2025 +0800
Modify the TsFileSequenceReaderTimeseriesMetadataIterator next function to return a LinkedHashMap (#593)
* Modify the TsFileSequenceReaderTimeseriesMetadataIterator next function to return a LinkedHashMap
* spotless
---
.../tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index 798714a8..d133b928 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -33,8 +33,8 @@ import java.nio.ByteBuffer;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
-import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -96,7 +96,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
throw new NoSuchElementException();
}
- final Map<IDeviceID, List<TimeseriesMetadata>> timeseriesMetadataMap = new HashMap<>();
+ final Map<IDeviceID, List<TimeseriesMetadata>> timeseriesMetadataMap = new LinkedHashMap<>();
while (currentTimeseriesMetadataCount < timeseriesBatchReadNumber) {
// 1. Check Buffer
(tsfile) 02/06: Alter column other type to string type (#588)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.2.1
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit add5b3b3a9944839d8567f0a62a7a392b39c6dd2
Author: libo <ze...@163.com>
AuthorDate: Wed Sep 3 11:48:38 2025 +0800
Alter column other type to string type (#588)
* Resolve may happen the problem that type conversion.
* Format code.
* Fix code.
* Fix code.
* Modify functions related type transfer in every operators class via template
* Support for what other type transfer to text/string type.
* Fix
* Allow modify value of date type, format to LocalDate result.
* Resolve the problem that BinaryColumnBuilder wrong use string of int type to transfer string of date type.
* Resolve the problem that the type of statistics don't match
* Fix
* Fix bug.
* Format code.
* Fix IT test.
* Fix IT bug.
* Adjust some problems.
* Format code.
---
.../java/org/apache/tsfile/enums/TSDataType.java | 98 +++++-
.../codegen/templates/FilterOperatorsTemplate.ftl | 356 +++++++++++++++++++--
.../file/metadata/statistics/Statistics.java | 51 ++-
.../common/block/column/BinaryColumnBuilder.java | 30 ++
.../org/apache/tsfile/read/reader/IPageReader.java | 2 +
.../reader/page/AbstractAlignedPageReader.java | 5 +
.../apache/tsfile/read/reader/page/PageReader.java | 5 +
.../tsfile/read/reader/page/ValuePageReader.java | 41 ++-
.../java/org/apache/tsfile/utils/TypeCastTest.java | 81 ++++-
9 files changed, 628 insertions(+), 41 deletions(-)
diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
index cf17aa04..492d1bab 100644
--- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
+++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.enums;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import java.io.DataOutputStream;
@@ -26,6 +27,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
@@ -99,6 +101,14 @@ public enum TSDataType {
Set<TSDataType> textCompatibleTypes = new HashSet<>();
textCompatibleTypes.add(STRING);
+ textCompatibleTypes.add(INT32);
+ textCompatibleTypes.add(INT64);
+ textCompatibleTypes.add(FLOAT);
+ textCompatibleTypes.add(DOUBLE);
+ textCompatibleTypes.add(BOOLEAN);
+ textCompatibleTypes.add(BLOB);
+ textCompatibleTypes.add(DATE);
+ textCompatibleTypes.add(TIMESTAMP);
compatibleTypes.put(TEXT, textCompatibleTypes);
compatibleTypes.put(VECTOR, Collections.emptySet());
@@ -119,6 +129,15 @@ public enum TSDataType {
Set<TSDataType> stringCompatibleTypes = new HashSet<>();
stringCompatibleTypes.add(TEXT);
+ // add
+ stringCompatibleTypes.add(INT32);
+ stringCompatibleTypes.add(INT64);
+ stringCompatibleTypes.add(FLOAT);
+ stringCompatibleTypes.add(DOUBLE);
+ stringCompatibleTypes.add(BOOLEAN);
+ stringCompatibleTypes.add(BLOB);
+ stringCompatibleTypes.add(DATE);
+ stringCompatibleTypes.add(TIMESTAMP);
compatibleTypes.put(STRING, stringCompatibleTypes);
}
@@ -232,6 +251,17 @@ public enum TSDataType {
case TEXT:
if (sourceType == TSDataType.TEXT || sourceType == TSDataType.STRING) {
return value;
+ } else if (sourceType == TSDataType.INT32
+ || sourceType == TSDataType.INT64
+ || sourceType == TSDataType.FLOAT
+ || sourceType == TSDataType.DOUBLE
+ || sourceType == TSDataType.BOOLEAN
+ || sourceType == TSDataType.TIMESTAMP) {
+ return new Binary(String.valueOf(value), StandardCharsets.UTF_8);
+ } else if (sourceType == TSDataType.DATE) {
+ return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8);
+ } else if (sourceType == TSDataType.BLOB) {
+ return new Binary(value.toString(), StandardCharsets.UTF_8);
} else {
break;
}
@@ -262,6 +292,17 @@ public enum TSDataType {
case STRING:
if (sourceType == TSDataType.STRING || sourceType == TSDataType.TEXT) {
return value;
+ } else if (sourceType == TSDataType.INT32
+ || sourceType == TSDataType.INT64
+ || sourceType == TSDataType.FLOAT
+ || sourceType == TSDataType.DOUBLE
+ || sourceType == TSDataType.BOOLEAN
+ || sourceType == TSDataType.TIMESTAMP) {
+ return new Binary(String.valueOf(value), StandardCharsets.UTF_8);
+ } else if (sourceType == TSDataType.DATE) {
+ return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8);
+ } else if (sourceType == TSDataType.BLOB) {
+ return new Binary(value.toString(), StandardCharsets.UTF_8);
} else {
break;
}
@@ -331,12 +372,6 @@ public enum TSDataType {
} else {
break;
}
- case TEXT:
- if (sourceType == TSDataType.TEXT || sourceType == STRING) {
- return array;
- } else {
- break;
- }
case TIMESTAMP:
if (sourceType == TSDataType.TIMESTAMP) {
return array;
@@ -361,9 +396,54 @@ public enum TSDataType {
} else {
break;
}
+ case TEXT:
case STRING:
- if (sourceType == TSDataType.STRING || sourceType == TSDataType.TEXT) {
+ if (sourceType == TSDataType.STRING
+ || sourceType == TSDataType.TEXT
+ || sourceType == TSDataType.BLOB) {
return array;
+ } else if (sourceType == TSDataType.INT32) {
+ int[] tmp = (int[]) array;
+ Binary[] result = new Binary[tmp.length];
+ for (int i = 0; i < tmp.length; i++) {
+ result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8);
+ }
+ return result;
+ } else if (sourceType == TSDataType.DATE) {
+ int[] tmp = (int[]) array;
+ Binary[] result = new Binary[tmp.length];
+ for (int i = 0; i < tmp.length; i++) {
+ result[i] = new Binary(TSDataType.getDateStringValue(tmp[i]), StandardCharsets.UTF_8);
+ }
+ return result;
+ } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) {
+ long[] tmp = (long[]) array;
+ Binary[] result = new Binary[tmp.length];
+ for (int i = 0; i < tmp.length; i++) {
+ result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8);
+ }
+ return result;
+ } else if (sourceType == TSDataType.FLOAT) {
+ float[] tmp = (float[]) array;
+ Binary[] result = new Binary[tmp.length];
+ for (int i = 0; i < tmp.length; i++) {
+ result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8);
+ }
+ return result;
+ } else if (sourceType == TSDataType.DOUBLE) {
+ double[] tmp = (double[]) array;
+ Binary[] result = new Binary[tmp.length];
+ for (int i = 0; i < tmp.length; i++) {
+ result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8);
+ }
+ return result;
+ } else if (sourceType == TSDataType.BOOLEAN) {
+ boolean[] tmp = (boolean[]) array;
+ Binary[] result = new Binary[tmp.length];
+ for (int i = 0; i < tmp.length; i++) {
+ result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8);
+ }
+ return result;
} else {
break;
}
@@ -487,4 +567,8 @@ public enum TSDataType {
public boolean isBinary() {
return this == TEXT || this == STRING || this == BLOB;
}
+
+ public static String getDateStringValue(int value) {
+ return String.format("%04d-%02d-%02d", value / 10000, (value % 10000) / 100, value % 100);
+ }
}
diff --git a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
index 0fe475d1..7e39b361 100644
--- a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
+++ b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
@@ -42,6 +42,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.nio.ByteBuffer;
+<#if filter.dataType == "Binary">
+import java.nio.charset.StandardCharsets;
+</#if>
<#if filter.dataType != "boolean">
import java.util.Collections;
</#if>
@@ -137,8 +140,15 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ }
+ else{
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -156,7 +166,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -166,6 +176,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0
+ || constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0
+ || constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -179,7 +205,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -189,6 +215,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) == 0
+ && constant.compareTo((${filter.dataType}) statistics.getMaxValue()) == 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) == 0
+ && constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) == 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -222,8 +264,14 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -241,7 +289,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -251,6 +299,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) == 0
+ && constant.compareTo((${filter.dataType}) statistics.getMaxValue()) == 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) == 0
+ && constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) == 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -265,7 +329,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -275,6 +339,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0
+ || constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0
+ || constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -308,8 +388,14 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -329,7 +415,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -338,6 +424,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMinValue() instanceof Binary) {
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) <= 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) <= 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -351,7 +451,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -360,6 +460,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMaxValue() instanceof Binary){
+ return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -392,8 +506,14 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -413,7 +533,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -422,6 +542,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMinValue() instanceof Binary) {
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -435,7 +569,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -444,6 +578,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMaxValue() instanceof Binary){
+ return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) >= 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) >= 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -476,8 +624,14 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -497,7 +651,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -506,6 +660,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMaxValue() instanceof Binary) {
+ return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) >= 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) >= 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -519,7 +687,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -528,6 +696,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMinValue() instanceof Binary){
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -560,8 +742,14 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -581,7 +769,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -590,6 +778,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMaxValue() instanceof Binary) {
+ return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -603,7 +805,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -612,6 +814,20 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if(statistics.getMinValue() instanceof Binary){
+ return constant.compareTo((${filter.dataType}) statistics.getMinValue()) <= 0;
+ }
+ else{
+ return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) <= 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -712,8 +928,14 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -735,7 +957,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -745,6 +967,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMaxValue() instanceof Binary) && (statistics.getMinValue() instanceof Binary)) {
+ return ((${filter.dataType}) statistics.getMaxValue()).compareTo(min) < 0
+ || ((${filter.dataType}) statistics.getMinValue()).compareTo(max) > 0;
+ }
+ else{
+ return (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(min) < 0
+ || (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(max) > 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -758,7 +996,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -768,6 +1006,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){
+ return ((${filter.dataType}) statistics.getMinValue()).compareTo(min) >= 0
+ && ((${filter.dataType}) statistics.getMaxValue()).compareTo(max) <= 0;
+ }
+ else{
+ return (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(min) >= 0
+ && (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(max) <= 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -801,8 +1055,14 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return valueSatisfy((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return valueSatisfy((${filter.dataType}) value);
+ } else {
+ return valueSatisfy(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return valueSatisfy(((Number) value).${filter.dataType}Value());
</#if>
@@ -823,7 +1083,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -833,6 +1093,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)) {
+ return ((${filter.dataType}) statistics.getMinValue()).compareTo(min) >= 0
+ && ((${filter.dataType}) statistics.getMaxValue()).compareTo(max) <= 0;
+ }
+ else{
+ return (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(min) >= 0
+ && (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(max) <= 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
// drop if value < min || value > max
if(statistics.isEmpty()){
@@ -846,7 +1122,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -856,6 +1132,22 @@ public final class ${className} {
<#else>
return false;
</#if>
+ <#elseif filter.dataType == "Binary">
+ <#if filter.javaBoxName == "String">
+ if(statistics.isEmpty()){
+ return false;
+ }
+ if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){
+ return ((${filter.dataType}) statistics.getMinValue()).compareTo(max) > 0
+ || ((${filter.dataType}) statistics.getMaxValue()).compareTo(min) < 0;
+ }
+ else{
+ return (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(max) > 0
+ || (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(min) < 0;
+ }
+ <#else>
+ return false;
+ </#if>
<#else>
if(statistics.isEmpty()){
return false;
@@ -994,8 +1286,14 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return candidates.contains((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return candidates.contains((${filter.dataType}) value);
+ } else {
+ return candidates.contains(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return candidates.contains(((Number) value).${filter.dataType}Value());
</#if>
@@ -1142,8 +1440,14 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
return !candidates.contains((${filter.dataType}) value);
+ <#elseif filter.dataType == "Binary">
+ if(value instanceof Binary){
+ return !candidates.contains((${filter.dataType}) value);
+ } else {
+ return !candidates.contains(new ${filter.dataType}(String.valueOf(value), StandardCharsets.UTF_8));
+ }
<#else>
return !candidates.contains(((Number) value).${filter.dataType}Value());
</#if>
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
index ee41b063..b1c2c463 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
@@ -24,6 +24,7 @@ import org.apache.tsfile.exception.filter.StatisticsClassException;
import org.apache.tsfile.exception.write.UnknownColumnTypeException;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -35,7 +36,9 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.util.HashSet;
import java.util.Objects;
+import java.util.Set;
/**
* This class is used for recording statistic information of each measurement in a delta file. While
@@ -63,6 +66,35 @@ public abstract class Statistics<T extends Serializable> {
static final String STATS_UNSUPPORTED_MSG = "%s statistics does not support: %s";
+ private static final Set<Pair<TSDataType, TSDataType>> CAN_NOT_MERGE_PAIRS;
+
+ static {
+ CAN_NOT_MERGE_PAIRS = new HashSet<>();
+
+ // related pair about STRING
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.INT32, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.INT64, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.FLOAT, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.DOUBLE, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.BOOLEAN, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.TIMESTAMP, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.DATE, TSDataType.STRING);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.BLOB, TSDataType.STRING);
+
+ // related pair about TEXT
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.INT32, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.INT64, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.FLOAT, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.DOUBLE, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.BOOLEAN, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.TIMESTAMP, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.DATE, TSDataType.TEXT);
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.BLOB, TSDataType.TEXT);
+
+ // related pari about TEXT and STRING
+ addSymmetricPairs(CAN_NOT_MERGE_PAIRS, TSDataType.TEXT, TSDataType.STRING);
+ }
+
/**
* static method providing statistic instance for respective data type.
*
@@ -229,7 +261,24 @@ public abstract class Statistics<T extends Serializable> {
return to.isCompatible(from)
&&
// cannot alter from TEXT to STRING because we cannot add statistic to the existing chunks
- !(from == TSDataType.TEXT && to == TSDataType.STRING);
+ isSatisfyMerge(from, to);
+ }
+
+ private static void addSymmetricPairs(
+ Set<Pair<TSDataType, TSDataType>> set, TSDataType... dataTypes) {
+ for (int i = 0; i < dataTypes.length; i++) {
+ for (int j = i + 1; j < dataTypes.length; j++) {
+ set.add(new Pair<>(dataTypes[i], dataTypes[j]));
+ set.add(new Pair<>(dataTypes[j], dataTypes[i]));
+ }
+ }
+ }
+
+ public static boolean isSatisfyMerge(TSDataType from, TSDataType to) {
+ if (from == to) {
+ return true;
+ }
+ return !CAN_NOT_MERGE_PAIRS.contains(new Pair<>(from, to));
}
public void update(long time, boolean value) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumnBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumnBuilder.java
index be952eaa..d6facd48 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumnBuilder.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumnBuilder.java
@@ -28,6 +28,7 @@ import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import static java.lang.Math.max;
@@ -100,6 +101,35 @@ public class BinaryColumnBuilder implements ColumnBuilder {
return writeBinary(column.getBinary(index));
}
+ @Override
+ public ColumnBuilder writeBoolean(boolean value) {
+ return writeBinary(new Binary(String.valueOf(value), StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ColumnBuilder writeInt(int value) {
+ return writeBinary(new Binary(String.valueOf(value), StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ColumnBuilder writeLong(long value) {
+ return writeBinary(new Binary(String.valueOf(value), StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ColumnBuilder writeFloat(float value) {
+ return writeBinary(new Binary(String.valueOf(value), StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ColumnBuilder writeDouble(double value) {
+ return writeBinary(new Binary(String.valueOf(value), StandardCharsets.UTF_8));
+ }
+
+ public ColumnBuilder writeDate(int value) {
+ return writeBinary(new Binary(TSDataType.getDateStringValue(value), StandardCharsets.UTF_8));
+ }
+
@Override
public ColumnBuilder writeTsPrimitiveType(TsPrimitiveType value) {
return writeBinary(value.getBinary());
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java
index 9d6980be..1b870f86 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java
@@ -43,6 +43,8 @@ public interface IPageReader extends IMetadata {
boolean isModified();
+ void setModified(boolean modified);
+
void initTsBlockBuilder(List<TSDataType> dataTypes);
void setLimitOffset(PaginationController paginationController);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java
index 9230d486..ad5af3b5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java
@@ -348,6 +348,11 @@ public abstract class AbstractAlignedPageReader implements IPageReader {
return isModified;
}
+ @Override
+ public void setModified(boolean modified) {
+ this.isModified = modified;
+ }
+
@Override
public void initTsBlockBuilder(List<TSDataType> dataTypes) {
if (paginationController.hasLimit()) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
index ee74f815..7ba234cf 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
@@ -413,6 +413,11 @@ public class PageReader implements IPageReader {
return pageHeader.isModified();
}
+ @Override
+ public void setModified(boolean modified) {
+ pageHeader.setModified(modified);
+ }
+
@Override
public void initTsBlockBuilder(List<TSDataType> dataTypes) {
// do nothing
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
index fc9dd375..998b2117 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
@@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.BatchDataFactory;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -318,7 +319,6 @@ public class ValuePageReader {
}
break;
case INT32:
- case DATE:
int anInt = valueDecoder.readInt(valueBuffer);
if (keepCurrentRow[i]) {
if (isDeleted[i]) {
@@ -328,6 +328,20 @@ public class ValuePageReader {
}
}
break;
+ case DATE:
+ int anDate = valueDecoder.readInt(valueBuffer);
+ if (keepCurrentRow[i]) {
+ if (isDeleted[i]) {
+ columnBuilder.appendNull();
+ } else {
+ if (columnBuilder instanceof BinaryColumnBuilder) {
+ ((BinaryColumnBuilder) columnBuilder).writeDate(anDate);
+ } else {
+ columnBuilder.writeInt(anDate);
+ }
+ }
+ }
+ break;
case INT64:
case TIMESTAMP:
long aLong = valueDecoder.readLong(valueBuffer);
@@ -403,12 +417,21 @@ public class ValuePageReader {
}
break;
case INT32:
- case DATE:
int anInt = valueDecoder.readInt(valueBuffer);
if (keepCurrentRow[i]) {
columnBuilder.writeInt(anInt);
}
break;
+ case DATE:
+ int anDate = valueDecoder.readInt(valueBuffer);
+ if (keepCurrentRow[i]) {
+ if (columnBuilder instanceof BinaryColumnBuilder) {
+ ((BinaryColumnBuilder) columnBuilder).writeDate(anDate);
+ } else {
+ columnBuilder.writeInt(anDate);
+ }
+ }
+ break;
case INT64:
case TIMESTAMP:
long aLong = valueDecoder.readLong(valueBuffer);
@@ -485,7 +508,15 @@ public class ValuePageReader {
continue;
}
int aInt = valueDecoder.readInt(valueBuffer);
- columnBuilder.writeInt(aInt);
+ if (dataType == TSDataType.INT32) {
+ columnBuilder.writeInt(aInt);
+ } else {
+ if (columnBuilder instanceof BinaryColumnBuilder) {
+ ((BinaryColumnBuilder) columnBuilder).writeDate(aInt);
+ } else {
+ columnBuilder.writeInt(aInt);
+ }
+ }
}
break;
case INT64:
@@ -584,6 +615,10 @@ public class ValuePageReader {
return pageHeader.isModified();
}
+ public void setModified(boolean modified) {
+ pageHeader.setModified(modified);
+ }
+
public boolean isDeleted(long timestamp) {
while (deleteIntervalList != null && deleteCursor < deleteIntervalList.size()) {
if (deleteIntervalList.get(deleteCursor).contains(timestamp)) {
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
index 10d26db1..4d1d0006 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
@@ -24,6 +24,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -46,7 +47,21 @@ public class TypeCastTest {
for (TSDataType to : dataTypes) {
Object src = genValue(from);
if (to.isCompatible(from)) {
- assertEquals(genValue(to), to.castFromSingleValue(from, src));
+ if (to == TSDataType.STRING || to == TSDataType.TEXT) {
+ if (from == TSDataType.DATE) {
+ assertEquals(
+ new Binary(LocalDate.ofEpochDay((int) src).toString(), StandardCharsets.UTF_8),
+ new Binary(
+ LocalDate.ofEpochDay(Long.parseLong(genValue(to).toString())).toString(),
+ StandardCharsets.UTF_8));
+ } else {
+ assertEquals(
+ new Binary(src.toString(), StandardCharsets.UTF_8),
+ to.castFromSingleValue(from, src));
+ }
+ } else {
+ assertEquals(genValue(to), to.castFromSingleValue(from, src));
+ }
} else {
assertThrows(ClassCastException.class, () -> to.castFromSingleValue(from, src));
}
@@ -66,7 +81,7 @@ public class TypeCastTest {
Object array = genValueArray(from);
if (!to.isCompatible(from)) {
assertThrows(ClassCastException.class, () -> to.castFromArray(from, array));
- return;
+ continue;
}
switch (to) {
case INT32:
@@ -84,8 +99,66 @@ public class TypeCastTest {
case STRING:
case BLOB:
case TEXT:
- assertArrayEquals(
- (Binary[]) genValueArray(to), (Binary[]) to.castFromArray(from, array));
+ switch (from) {
+ case BLOB:
+ case STRING:
+ assertArrayEquals((Binary[]) array, (Binary[]) to.castFromArray(from, array));
+ break;
+ case INT32:
+ int[] tmpInt = (int[]) array;
+ Binary[] intResult = new Binary[tmpInt.length];
+ for (int i = 0; i < tmpInt.length; i++) {
+ intResult[i] = new Binary(String.valueOf(tmpInt[i]), StandardCharsets.UTF_8);
+ }
+ assertArrayEquals(intResult, (Binary[]) to.castFromArray(from, array));
+ break;
+ case DATE:
+ int[] tmpDate = (int[]) array;
+ Binary[] dateResult = new Binary[tmpDate.length];
+ for (int i = 0; i < tmpDate.length; i++) {
+ dateResult[i] =
+ new Binary(TSDataType.getDateStringValue(tmpDate[i]), StandardCharsets.UTF_8);
+ }
+ assertArrayEquals(dateResult, (Binary[]) to.castFromArray(from, array));
+ break;
+ case INT64:
+ case TIMESTAMP:
+ long[] tmpLong = (long[]) array;
+ Binary[] longResult = new Binary[tmpLong.length];
+ for (int i = 0; i < tmpLong.length; i++) {
+ longResult[i] = new Binary(String.valueOf(tmpLong[i]), StandardCharsets.UTF_8);
+ }
+ assertArrayEquals(longResult, (Binary[]) to.castFromArray(from, array));
+ break;
+ case FLOAT:
+ float[] tmpFloat = (float[]) array;
+ Binary[] floatResult = new Binary[tmpFloat.length];
+ for (int i = 0; i < tmpFloat.length; i++) {
+ floatResult[i] = new Binary(String.valueOf(tmpFloat[i]), StandardCharsets.UTF_8);
+ }
+ assertArrayEquals(floatResult, (Binary[]) to.castFromArray(from, array));
+ break;
+ case DOUBLE:
+ double[] tmpDouble = (double[]) array;
+ Binary[] doubleResult = new Binary[tmpDouble.length];
+ for (int i = 0; i < tmpDouble.length; i++) {
+ doubleResult[i] =
+ new Binary(String.valueOf(tmpDouble[i]), StandardCharsets.UTF_8);
+ }
+ assertArrayEquals(doubleResult, (Binary[]) to.castFromArray(from, array));
+ break;
+ case BOOLEAN:
+ boolean[] tmpBoolean = (boolean[]) array;
+ Binary[] booleanResult = new Binary[tmpBoolean.length];
+ for (int i = 0; i < tmpBoolean.length; i++) {
+ booleanResult[i] =
+ new Binary(String.valueOf(tmpBoolean[i]), StandardCharsets.UTF_8);
+ }
+ assertArrayEquals(booleanResult, (Binary[]) to.castFromArray(from, array));
+ break;
+ default:
+ break;
+ }
break;
case FLOAT:
assertArrayEquals(