You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ja...@apache.org on 2024/08/31 13:23:33 UTC
(tsfile) 02/03: Add encryption module 1.0 (#213)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 1c1d5fd0ccf154a9e5e92eb8ff7412135d6e1df5
Author: jintao zhu <10...@users.noreply.github.com>
AuthorDate: Fri Aug 30 09:38:57 2024 +0800
Add encryption module 1.0 (#213)
* encrypt function v1 version realization
* Fix the wrong places and add tests for encryption module
* Fix the wrong places and add tests for encryption module
* Fix the wrong places and add tests for encryption module
* Fix the wrong places and add tests for encryption module
* modify some places in encryption module for compatibility
* code review
* code review
* code review
* code review
* code review
* code review
---
.gitignore | 2 +
.../apache/tsfile/common/conf/TSFileConfig.java | 48 +++
.../tsfile/common/conf/TSFileDescriptor.java | 3 +
.../org/apache/tsfile/encrypt/EncryptUtils.java | 150 ++++++++
.../java/org/apache/tsfile/encrypt/IDecryptor.java | 163 +++++++++
.../java/org/apache/tsfile/encrypt/IEncryptor.java | 162 +++++++++
.../main/java/org/apache/tsfile/encrypt/SM4.java | 326 +++++++++++++++++
.../java/org/apache/tsfile/encrypt/SM4Utils.java | 90 +++++
.../org/apache/tsfile/encrypt/SM4_Context.java | 34 ++
.../tsfile/exception/encrypt/EncryptException.java | 30 ++
.../encrypt/EncryptKeyLengthNotMatchException.java | 31 ++
.../tsfile/file/metadata/TsFileMetadata.java | 86 ++++-
.../tsfile/file/metadata/enums/EncryptionType.java | 81 +++++
.../apache/tsfile/read/TsFileSequenceReader.java | 74 +++-
.../java/org/apache/tsfile/read/common/Chunk.java | 29 ++
.../read/controller/CachedChunkLoaderImpl.java | 6 +-
.../read/reader/chunk/AlignedChunkReader.java | 11 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 51 ++-
.../tsfile/read/reader/page/LazyLoadPageData.java | 17 +-
.../java/org/apache/tsfile/write/TsFileWriter.java | 50 ++-
.../write/chunk/AlignedChunkGroupWriterImpl.java | 18 +
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 116 ++++++-
.../apache/tsfile/write/chunk/ChunkWriterImpl.java | 36 +-
.../chunk/NonAlignedChunkGroupWriterImpl.java | 14 +-
.../apache/tsfile/write/chunk/TimeChunkWriter.java | 33 +-
.../tsfile/write/chunk/ValueChunkWriter.java | 34 +-
.../org/apache/tsfile/write/page/PageWriter.java | 49 ++-
.../apache/tsfile/write/page/TimePageWriter.java | 37 +-
.../apache/tsfile/write/page/ValuePageWriter.java | 41 ++-
.../apache/tsfile/write/writer/TsFileIOWriter.java | 39 +++
.../java/org/apache/tsfile/encrypt/AES128Test.java | 76 ++++
.../tsfile/encrypt/AES128TsFileReadWriteTest.java | 265 ++++++++++++++
.../tsfile/encrypt/AES128TsFileWriteTest.java | 292 ++++++++++++++++
.../org/apache/tsfile/encrypt/EncryptTest.java | 385 +++++++++++++++++++++
.../java/org/apache/tsfile/encrypt/SM4128Test.java | 76 ++++
.../tsfile/encrypt/SM4128TsFileReadWriteTest.java | 265 ++++++++++++++
.../tsfile/encrypt/SM4128TsFileWriteTest.java | 291 ++++++++++++++++
37 files changed, 3464 insertions(+), 47 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7e7d3f47..c2c059c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,5 @@ python/tsfile/*.cpp
python/data
python/venv/*
cpp/cmake-build-debug-mingw/
+
+.vscode/settings.json
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
index 1cdd59e3..40fd6c37 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
@@ -19,8 +19,10 @@
package org.apache.tsfile.common.conf;
+import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.fileSystem.FSType;
import org.apache.tsfile.utils.FSUtils;
@@ -136,6 +138,15 @@ public class TSFileConfig implements Serializable {
/** Data compression method, TsFile supports UNCOMPRESSED, SNAPPY, ZSTD or LZ4. */
private CompressionType compressor = CompressionType.LZ4;
+ /** encryptFlag, true means opening the encrypt function. */
+ private boolean encryptFlag = false;
+
+ /** encryptKey, this should be 16 bytes String. */
+ private String encryptKey = "abcdefghijklmnop";
+
+ /** default encryptType is "UNENCRYPTED", TsFile supports UNENCRYPTED, SM4128 or AES128. */
+ private EncryptionType encryptType = EncryptionType.UNENCRYPTED;
+
/** Line count threshold for checking page memory occupied size. */
private int pageCheckSizeThreshold = 100;
@@ -215,6 +226,43 @@ public class TSFileConfig implements Serializable {
// do nothing because we already give default value to each field when they are being declared
}
+ public boolean getEncryptFlag() {
+ return encryptFlag;
+ }
+
+ public void setEncryptFlag(String encryptFlag) {
+ this.encryptFlag = Boolean.parseBoolean(encryptFlag);
+ }
+
+ public EncryptionType getEncryptType() {
+ return this.encryptType;
+ }
+
+ public void setEncryptType(String encryptType) {
+ this.encryptType = EncryptionType.valueOf(encryptType);
+ }
+
+ public String getEncryptKey() {
+ return this.encryptKey;
+ }
+
+ public void setEncryptKey(String encryptKey) {
+ this.encryptKey = encryptKey;
+ }
+
+ public void setEncryptKeyFromPath(String encryptKeyPath) {
+ if (!encryptFlag) {
+ return;
+ }
+ if (encryptKeyPath == null) {
+ throw new RuntimeException("encrypt key path is null");
+ }
+ if (encryptKeyPath.isEmpty()) {
+ throw new RuntimeException("encrypt key path is empty");
+ }
+ this.encryptKey = EncryptUtils.getEncryptKeyFromPath(encryptKeyPath);
+ }
+
public int getGroupSizeInByte() {
return groupSizeInByte;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java
index 3df22502..76a95db4 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java
@@ -82,6 +82,9 @@ public class TSFileDescriptor {
writer.setString(conf::setValueEncoder, "value_encoder");
writer.setString(conf::setCompressor, "compressor");
writer.setInt(conf::setBatchSize, "batch_size");
+ writer.setString(conf::setEncryptFlag, "encrypt_flag");
+ writer.setString(conf::setEncryptType, "encrypt_type");
+ writer.setString(conf::setEncryptKeyFromPath, "encrypt_key_path");
}
private static class PropertiesOverWriter {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
new file mode 100644
index 00000000..87ed1381
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.exception.encrypt.EncryptException;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.security.MessageDigest;
+
+public class EncryptUtils {
+
+ public static String normalKeyStr = getNormalKeyStr();
+
+ public static IEncryptor encryptor = getDefaultEncryptor();
+
+ public static IDecryptor decryptor = getDefaultDecryptor();
+
+ public static String getEncryptKeyFromPath(String path) {
+ try (BufferedReader br = new BufferedReader(new FileReader(path))) {
+ StringBuilder sb = new StringBuilder();
+ String line;
+ boolean first = true;
+ while ((line = br.readLine()) != null) {
+ if (first) {
+ sb.append(line);
+ first = false;
+ } else {
+ sb.append("\n").append(line);
+ }
+ }
+ return sb.toString();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static byte[] hexStringToByteArray(String hexString) {
+ int len = hexString.length();
+ byte[] byteArray = new byte[len / 2];
+
+ for (int i = 0; i < len; i += 2) {
+ byteArray[i / 2] =
+ (byte)
+ ((Character.digit(hexString.charAt(i), 16) << 4)
+ + Character.digit(hexString.charAt(i + 1), 16));
+ }
+
+ return byteArray;
+ }
+
+ public static String byteArrayToHexString(byte[] bytes) {
+ StringBuilder sb = new StringBuilder();
+
+ for (byte b : bytes) {
+ sb.append(String.format("%02X", b));
+ }
+
+ return sb.toString();
+ }
+
+ public static String getNormalKeyStr() {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update("IoTDB is the best".getBytes());
+ md.update(TSFileDescriptor.getInstance().getConfig().getEncryptKey().getBytes());
+ byte[] data_key = md.digest();
+ StringBuilder valueStr = new StringBuilder();
+
+ for (byte b : data_key) {
+ valueStr.append(b).append(",");
+ }
+
+ valueStr.deleteCharAt(valueStr.length() - 1);
+ String str = valueStr.toString();
+
+ return str;
+ } catch (Exception e) {
+ throw new EncryptException("md5 function not found while using md5 to generate data key");
+ }
+ }
+
+ public static IEncryptor getDefaultEncryptor() {
+ EncryptionType encryptType;
+ byte[] dataEncryptKey;
+ if (TSFileDescriptor.getInstance().getConfig().getEncryptFlag()) {
+ encryptType = TSFileDescriptor.getInstance().getConfig().getEncryptType();
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update("IoTDB is the best".getBytes());
+ md.update(TSFileDescriptor.getInstance().getConfig().getEncryptKey().getBytes());
+ dataEncryptKey = md.digest();
+ } catch (Exception e1) {
+ throw new EncryptException("md5 function not found while using md5 to generate data key");
+ }
+ } else {
+ encryptType = EncryptionType.UNENCRYPTED;
+ dataEncryptKey = null;
+ }
+ return IEncryptor.getEncryptor(encryptType, dataEncryptKey);
+ }
+
+ public static IDecryptor getDefaultDecryptor() {
+ EncryptionType encryptType;
+ byte[] dataEncryptKey;
+ if (TSFileDescriptor.getInstance().getConfig().getEncryptFlag()) {
+ encryptType = TSFileDescriptor.getInstance().getConfig().getEncryptType();
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update("IoTDB is the best".getBytes());
+ md.update(TSFileDescriptor.getInstance().getConfig().getEncryptKey().getBytes());
+ dataEncryptKey = md.digest();
+ } catch (Exception e1) {
+ throw new EncryptException("md5 function not found while using md5 to generate data key");
+ }
+ } else {
+ encryptType = EncryptionType.UNENCRYPTED;
+ dataEncryptKey = null;
+ }
+ return IDecryptor.getDecryptor(encryptType, dataEncryptKey);
+ }
+
+ public static byte[] getKeyFromStr(String str) {
+ String[] strArray = str.split(",");
+ byte[] key = new byte[strArray.length];
+ for (int i = 0; i < strArray.length; i++) {
+ key[i] = Byte.parseByte(strArray[i]);
+ }
+ return key;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java
new file mode 100644
index 00000000..0707ba3b
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.exception.encrypt.EncryptException;
+import org.apache.tsfile.exception.encrypt.EncryptKeyLengthNotMatchException;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+/** encrypt data according to tsfileconfig. */
+public interface IDecryptor {
+
+ static final Logger logger = LoggerFactory.getLogger(IDecryptor.class);
+
+ static IDecryptor getDecryptor(String name, byte[] key) {
+ return getDecryptor(EncryptionType.valueOf(name), key);
+ }
+
+ static IDecryptor getDecryptor(EncryptionType name, byte[] key) {
+ if (name == null) {
+ return new NoDecryptor();
+ }
+ switch (name) {
+ case UNENCRYPTED:
+ return new NoDecryptor();
+ case SM4128:
+ return new SM4128Decryptor(key);
+ case AES128:
+ return new AES128Decryptor(key);
+ default:
+ logger.warn("Unknown encryption type: {}", name);
+ return new NoDecryptor();
+ }
+ }
+
+ byte[] decrypt(byte[] data);
+
+ byte[] decrypt(byte[] data, int offset, int size);
+
+ EncryptionType getEncryptionType();
+
+ class NoDecryptor implements IDecryptor {
+
+ @Override
+ public byte[] decrypt(byte[] data) {
+ return data;
+ }
+
+ @Override
+ public byte[] decrypt(byte[] data, int offset, int size) {
+ return Arrays.copyOfRange(data, offset, offset + size);
+ }
+
+ @Override
+ public EncryptionType getEncryptionType() {
+ return EncryptionType.UNENCRYPTED;
+ }
+ }
+
+ class SM4128Decryptor implements IDecryptor {
+
+ private final SM4Utils sm4;
+
+ SM4128Decryptor(byte[] key) {
+ if (key.length != 16) {
+ throw new EncryptKeyLengthNotMatchException(16, key.length);
+ }
+ this.sm4 = new SM4Utils(key, key);
+ }
+
+ @Override
+ public byte[] decrypt(byte[] data) {
+ return sm4.cryptData_CTR(data);
+ }
+
+ @Override
+ public byte[] decrypt(byte[] data, int offset, int size) {
+ return decrypt(Arrays.copyOfRange(data, offset, offset + size));
+ }
+
+ @Override
+ public EncryptionType getEncryptionType() {
+ return EncryptionType.SM4128;
+ }
+ }
+
+ class AES128Decryptor implements IDecryptor {
+ private final Cipher AES;
+
+ AES128Decryptor(byte[] key) {
+ if (key.length != 16) {
+ throw new EncryptKeyLengthNotMatchException(16, key.length);
+ }
+ SecretKeySpec secretKeySpec = new SecretKeySpec(key, "AES");
+ // Create IV parameter
+ IvParameterSpec ivParameterSpec = new IvParameterSpec(key);
+ try {
+ // Create Cipher instance and initialize it for encryption in CTR mode without padding
+ this.AES = Cipher.getInstance("AES/CTR/NoPadding");
+ AES.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParameterSpec);
+ } catch (InvalidAlgorithmParameterException
+ | NoSuchPaddingException
+ | NoSuchAlgorithmException
+ | InvalidKeyException e) {
+ throw new EncryptException("AES128Decryptor init failed ", e);
+ }
+ }
+
+ @Override
+ public byte[] decrypt(byte[] data) {
+ try {
+ return AES.doFinal(data);
+ } catch (IllegalBlockSizeException | BadPaddingException e) {
+ throw new EncryptException("AES128Decryptor decrypt failed ", e);
+ }
+ }
+
+ @Override
+ public byte[] decrypt(byte[] data, int offset, int size) {
+ try {
+ return AES.doFinal(data, offset, size);
+ } catch (IllegalBlockSizeException | BadPaddingException e) {
+ throw new EncryptException("AES128Decryptor decrypt failed ", e);
+ }
+ }
+
+ @Override
+ public EncryptionType getEncryptionType() {
+ return EncryptionType.AES128;
+ }
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java
new file mode 100644
index 00000000..20ccb8e9
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.exception.encrypt.EncryptException;
+import org.apache.tsfile.exception.encrypt.EncryptKeyLengthNotMatchException;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+/** encrypt data according to tsfileconfig. */
+public interface IEncryptor {
+
+ static final Logger logger = LoggerFactory.getLogger(IEncryptor.class);
+
+ static IEncryptor getEncryptor(String name, byte[] key) {
+ return getEncryptor(EncryptionType.valueOf(name), key);
+ }
+
+ static IEncryptor getEncryptor(EncryptionType name, byte[] key) {
+ if (name == null) {
+ return new NoEncryptor();
+ }
+ switch (name) {
+ case SM4128:
+ return new SM4128Encryptor(key);
+ case AES128:
+ return new AES128Encryptor(key);
+ default:
+ // log a warning
+ logger.warn("Unknown encryption type: {}", name);
+ return new NoEncryptor();
+ }
+ }
+
+ byte[] encrypt(byte[] data);
+
+ byte[] encrypt(byte[] data, int offset, int size);
+
+ EncryptionType getEncryptionType();
+
+ class NoEncryptor implements IEncryptor {
+
+ @Override
+ public byte[] encrypt(byte[] data) {
+ return data;
+ }
+
+ @Override
+ public byte[] encrypt(byte[] data, int offset, int size) {
+ return Arrays.copyOfRange(data, offset, offset + size);
+ }
+
+ @Override
+ public EncryptionType getEncryptionType() {
+ return EncryptionType.UNENCRYPTED;
+ }
+ }
+
+ class SM4128Encryptor implements IEncryptor {
+
+ private final SM4Utils sm4;
+
+ SM4128Encryptor(byte[] key) {
+ if (key.length != 16) {
+ throw new EncryptKeyLengthNotMatchException(16, key.length);
+ }
+ this.sm4 = new SM4Utils(key, key);
+ }
+
+ @Override
+ public byte[] encrypt(byte[] data) {
+ return sm4.cryptData_CTR(data);
+ }
+
+ @Override
+ public byte[] encrypt(byte[] data, int offset, int size) {
+ return encrypt(Arrays.copyOfRange(data, offset, offset + size));
+ }
+
+ @Override
+ public EncryptionType getEncryptionType() {
+ return EncryptionType.SM4128;
+ }
+ }
+
+ class AES128Encryptor implements IEncryptor {
+ private final Cipher AES;
+
+ AES128Encryptor(byte[] key) {
+ if (key.length != 16) {
+ throw new EncryptKeyLengthNotMatchException(16, key.length);
+ }
+ SecretKeySpec secretKeySpec = new SecretKeySpec(key, "AES");
+ // Create IV parameter
+ IvParameterSpec ivParameterSpec = new IvParameterSpec(key);
+ try {
+ // Create Cipher instance and initialize it for encryption in CTR mode without padding
+ this.AES = Cipher.getInstance("AES/CTR/NoPadding");
+ AES.init(Cipher.ENCRYPT_MODE, secretKeySpec, ivParameterSpec);
+ } catch (InvalidAlgorithmParameterException
+ | NoSuchPaddingException
+ | NoSuchAlgorithmException
+ | InvalidKeyException e) {
+ throw new EncryptException("AES128Encryptor init failed ", e);
+ }
+ }
+
+ @Override
+ public byte[] encrypt(byte[] data) {
+ try {
+ return AES.doFinal(data);
+ } catch (IllegalBlockSizeException | BadPaddingException e) {
+ throw new EncryptException("AES128Encryptor encrypt failed ", e);
+ }
+ }
+
+ @Override
+ public byte[] encrypt(byte[] data, int offset, int size) {
+ try {
+ return AES.doFinal(data, offset, size);
+ } catch (IllegalBlockSizeException | BadPaddingException e) {
+ throw new EncryptException("AES128Encryptor encrypt failed ", e);
+ }
+ }
+
+ @Override
+ public EncryptionType getEncryptionType() {
+ return EncryptionType.AES128;
+ }
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4.java
new file mode 100644
index 00000000..fa453289
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.exception.encrypt.EncryptException;
+import org.apache.tsfile.exception.encrypt.EncryptKeyLengthNotMatchException;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public class SM4 {
+ public static final int SM4_ENCRYPT = 1;
+
+ public static final int SM4_DECRYPT = 0;
+
+ private long GET_ULONG_BE(byte[] b, int i) {
+ long n =
+ (long) (b[i] & 0xff) << 24
+ | (long) ((b[i + 1] & 0xff) << 16)
+ | (long) ((b[i + 2] & 0xff) << 8)
+ | (long) (b[i + 3] & 0xff) & 0xffffffffL;
+ return n;
+ }
+
+ private void PUT_ULONG_BE(long n, byte[] b, int i) {
+ b[i] = (byte) (int) (0xFF & n >> 24);
+ b[i + 1] = (byte) (int) (0xFF & n >> 16);
+ b[i + 2] = (byte) (int) (0xFF & n >> 8);
+ b[i + 3] = (byte) (int) (0xFF & n);
+ }
+
+ private long SHL(long x, int n) {
+ return (x & 0xFFFFFFFF) << n;
+ }
+
+ private long ROTL(long x, int n) {
+ return SHL(x, n) | x >> (32 - n);
+ }
+
+ private void SWAP(long[] sk, int i) {
+ long t = sk[i];
+ sk[i] = sk[(31 - i)];
+ sk[(31 - i)] = t;
+ }
+
+ public static final byte[] SboxTable =
+ EncryptUtils.hexStringToByteArray(
+ "D690E9FECCE13DB716B614C228FB2C052B679A762ABE04C3AA441326498606999C4250F491EF987A33540B43EDCFAC62E4B31CA9C908E89580DF94FA758F3FA64707A7FCF37317BA83593C19E6854FA8686B81B27164DA8BF8EB0F4B70569D351E240E5E6358D1A225227C3B01217887D40046579FD327524C3602E7A0C4C89EEABF8AD240C738B5A3F7F2CEF96115A1E0AE5DA49B341A55AD933230F58CB1E31DF6E22E8266CA60C02923AB0D534E6FD5DB3745DEFD8E2F03FF6A726D6C5B518D1BAF92BBDDBC7F11D95C411F105AD80AC13188A5CD7BBD2D74D012B8E5B4B08969974A0C96777E65B9F109C56EC6841 [...]
+
+ public static final int[] FK = {0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc};
+
+ public static final int[] CK = {
+ 0x00070e15,
+ 0x1c232a31,
+ 0x383f464d,
+ 0x545b6269,
+ 0x70777e85,
+ 0x8c939aa1,
+ 0xa8afb6bd,
+ 0xc4cbd2d9,
+ 0xe0e7eef5,
+ 0xfc030a11,
+ 0x181f262d,
+ 0x343b4249,
+ 0x50575e65,
+ 0x6c737a81,
+ 0x888f969d,
+ 0xa4abb2b9,
+ 0xc0c7ced5,
+ 0xdce3eaf1,
+ 0xf8ff060d,
+ 0x141b2229,
+ 0x30373e45,
+ 0x4c535a61,
+ 0x686f767d,
+ 0x848b9299,
+ 0xa0a7aeb5,
+ 0xbcc3cad1,
+ 0xd8dfe6ed,
+ 0xf4fb0209,
+ 0x10171e25,
+ 0x2c333a41,
+ 0x484f565d,
+ 0x646b7279
+ };
+
+ private byte sm4Sbox(byte inch) {
+ int i = inch & 0xFF;
+ byte retVal = SboxTable[i];
+ return retVal;
+ }
+
+ private long sm4Lt(long ka) {
+ long bb = 0L;
+ long c = 0L;
+ byte[] a = new byte[4];
+ byte[] b = new byte[4];
+ PUT_ULONG_BE(ka, a, 0);
+ b[0] = sm4Sbox(a[0]);
+ b[1] = sm4Sbox(a[1]);
+ b[2] = sm4Sbox(a[2]);
+ b[3] = sm4Sbox(a[3]);
+ bb = GET_ULONG_BE(b, 0);
+ c = bb ^ ROTL(bb, 2) ^ ROTL(bb, 10) ^ ROTL(bb, 18) ^ ROTL(bb, 24);
+ return c;
+ }
+
+ private long sm4F(long x0, long x1, long x2, long x3, long rk) {
+ return x0 ^ sm4Lt(x1 ^ x2 ^ x3 ^ rk);
+ }
+
+ private long sm4CalciRK(long ka) {
+ long bb = 0L;
+ long rk = 0L;
+ byte[] a = new byte[4];
+ byte[] b = new byte[4];
+ PUT_ULONG_BE(ka, a, 0);
+ b[0] = sm4Sbox(a[0]);
+ b[1] = sm4Sbox(a[1]);
+ b[2] = sm4Sbox(a[2]);
+ b[3] = sm4Sbox(a[3]);
+ bb = GET_ULONG_BE(b, 0);
+ rk = bb ^ ROTL(bb, 13) ^ ROTL(bb, 23);
+ return rk;
+ }
+
+ private void sm4_setkey(long[] SK, byte[] key) {
+ long[] MK = new long[4];
+ long[] k = new long[36];
+ int i = 0;
+ MK[0] = GET_ULONG_BE(key, 0);
+ MK[1] = GET_ULONG_BE(key, 4);
+ MK[2] = GET_ULONG_BE(key, 8);
+ MK[3] = GET_ULONG_BE(key, 12);
+ k[0] = MK[0] ^ (long) FK[0];
+ k[1] = MK[1] ^ (long) FK[1];
+ k[2] = MK[2] ^ (long) FK[2];
+ k[3] = MK[3] ^ (long) FK[3];
+ for (; i < 32; i++) {
+ k[(i + 4)] = (k[i] ^ sm4CalciRK(k[(i + 1)] ^ k[(i + 2)] ^ k[(i + 3)] ^ (long) CK[i]));
+ SK[i] = k[(i + 4)];
+ }
+ }
+
+ private void sm4_one_round(long[] sk, byte[] input, byte[] output) {
+ int i = 0;
+ long[] ulbuf = new long[36];
+ ulbuf[0] = GET_ULONG_BE(input, 0);
+ ulbuf[1] = GET_ULONG_BE(input, 4);
+ ulbuf[2] = GET_ULONG_BE(input, 8);
+ ulbuf[3] = GET_ULONG_BE(input, 12);
+ while (i < 32) {
+ ulbuf[(i + 4)] = sm4F(ulbuf[i], ulbuf[(i + 1)], ulbuf[(i + 2)], ulbuf[(i + 3)], sk[i]);
+ i++;
+ }
+ PUT_ULONG_BE(ulbuf[35], output, 0);
+ PUT_ULONG_BE(ulbuf[34], output, 4);
+ PUT_ULONG_BE(ulbuf[33], output, 8);
+ PUT_ULONG_BE(ulbuf[32], output, 12);
+ }
+
+ private byte[] padding(byte[] input, int mode) {
+ if (input == null) {
+ return null;
+ }
+
+ byte[] ret = (byte[]) null;
+ if (mode == SM4_ENCRYPT) {
+ int p = 16 - input.length % 16;
+ ret = new byte[input.length + p];
+ System.arraycopy(input, 0, ret, 0, input.length);
+ for (int i = 0; i < p; i++) {
+ ret[input.length + i] = (byte) p;
+ }
+ } else {
+ int p = input[input.length - 1];
+ ret = new byte[input.length - p];
+ System.arraycopy(input, 0, ret, 0, input.length - p);
+ }
+ return ret;
+ }
+
+ public void sm4_setkey_enc(SM4_Context ctx, byte[] key) {
+ if (ctx == null) {
+ throw new EncryptException("sm4 ctx is null!");
+ }
+
+ if (key == null) {
+ throw new EncryptException("sm4 key null error!");
+ }
+ if (key.length != 16) {
+ throw new EncryptKeyLengthNotMatchException(key.length, 16);
+ }
+
+ ctx.mode = SM4_ENCRYPT;
+ sm4_setkey(ctx.sk, key);
+ }
+
+ public void sm4_setkey_dec(SM4_Context ctx, byte[] key) {
+ if (ctx == null) {
+ throw new EncryptException("sm4 ctx is null!");
+ }
+
+ if (key == null) {
+ throw new EncryptException("sm4 key null error!");
+ }
+ if (key.length != 16) {
+ throw new EncryptKeyLengthNotMatchException(key.length, 16);
+ }
+
+ int i = 0;
+ ctx.mode = SM4_DECRYPT;
+ sm4_setkey(ctx.sk, key);
+ for (i = 0; i < 16; i++) {
+ SWAP(ctx.sk, i);
+ }
+ }
+
+ public byte[] sm4_crypt_ecb(SM4_Context ctx, byte[] input) {
+ try {
+ if (input == null) {
+ throw new EncryptException("sm4 input is null!");
+ }
+ if ((ctx.isPadding) && (ctx.mode == SM4_ENCRYPT)) {
+ input = padding(input, SM4_ENCRYPT);
+ }
+
+ int length = input.length;
+ ByteArrayInputStream bins = new ByteArrayInputStream(input);
+ ByteArrayOutputStream bous = new ByteArrayOutputStream();
+ for (; length > 0; length -= 16) {
+ byte[] in = new byte[16];
+ byte[] out = new byte[16];
+ bins.read(in);
+ sm4_one_round(ctx.sk, in, out);
+ bous.write(out);
+ }
+
+ byte[] output = bous.toByteArray();
+ if (ctx.isPadding && ctx.mode == SM4_DECRYPT) {
+ output = padding(output, SM4_DECRYPT);
+ }
+ bins.close();
+ bous.close();
+ return output;
+ } catch (IOException e) {
+ throw new EncryptException("sm4 crypt error");
+ }
+ }
+
+ public byte[] sm4_crypt_cbc(SM4_Context ctx, byte[] iv, byte[] input) throws Exception {
+ if (iv == null || iv.length != 16) {
+ throw new EncryptException("iv error!");
+ }
+
+ if (input == null) {
+ throw new EncryptException("input is null!");
+ }
+
+ if (ctx.isPadding && ctx.mode == SM4_ENCRYPT) {
+ input = padding(input, SM4_ENCRYPT);
+ }
+
+ int i = 0;
+ int length = input.length;
+ ByteArrayInputStream bins = new ByteArrayInputStream(input);
+ ByteArrayOutputStream bous = new ByteArrayOutputStream();
+ if (ctx.mode == SM4_ENCRYPT) {
+ for (; length > 0; length -= 16) {
+ byte[] in = new byte[16];
+ byte[] out = new byte[16];
+ byte[] out1 = new byte[16];
+
+ bins.read(in);
+ for (i = 0; i < 16; i++) {
+ out[i] = ((byte) (in[i] ^ iv[i]));
+ }
+ sm4_one_round(ctx.sk, out, out1);
+ System.arraycopy(out1, 0, iv, 0, 16);
+ bous.write(out1);
+ }
+ } else {
+ byte[] temp = new byte[16];
+ for (; length > 0; length -= 16) {
+ byte[] in = new byte[16];
+ byte[] out = new byte[16];
+ byte[] out1 = new byte[16];
+
+ bins.read(in);
+ System.arraycopy(in, 0, temp, 0, 16);
+ sm4_one_round(ctx.sk, in, out);
+ for (i = 0; i < 16; i++) {
+ out1[i] = ((byte) (out[i] ^ iv[i]));
+ }
+ System.arraycopy(temp, 0, iv, 0, 16);
+ bous.write(out1);
+ }
+ }
+
+ byte[] output = bous.toByteArray();
+ if (ctx.isPadding && ctx.mode == SM4_DECRYPT) {
+ output = padding(output, SM4_DECRYPT);
+ }
+ bins.close();
+ bous.close();
+ return output;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4Utils.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4Utils.java
new file mode 100644
index 00000000..8735275e
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4Utils.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.encrypt;
+
+import java.util.Arrays;
+
+public class SM4Utils {
+ public byte[] getSecretKey() {
+ return secretKey;
+ }
+
+ public void setSecretKey(byte[] secretKey) {
+ this.secretKey = secretKey;
+ }
+
+ private byte[] secretKey;
+
+ public byte[] getIv() {
+ return iv;
+ }
+
+ public void setIv(String iv) {
+ this.iv = iv.getBytes();
+ }
+
+ private byte[] iv;
+
+ private SM4_Context ctx;
+
+ private SM4 sm4;
+
+ public SM4Utils(byte[] key, byte[] iv) {
+ this.secretKey = key;
+ this.iv = iv;
+ this.ctx = new SM4_Context();
+ sm4 = new SM4();
+ sm4.sm4_setkey_enc(ctx, secretKey);
+ }
+
+ public byte[] cryptData_CTR(byte[] data) {
+ byte[] keyBytes = secretKey;
+ byte[] pivBytes = iv;
+ byte[] result = new byte[data.length];
+ byte[] ivBytes = Arrays.copyOfRange(pivBytes, 0, 16);
+ int begin_iv = 0;
+ for (int i = 0; i < 4; i++) {
+ begin_iv = (begin_iv << 8) | (ivBytes[i] & 0xFF);
+ }
+ int group_num = data.length / 16;
+ for (int i = 0; i < group_num; i++) {
+ ivBytes[0] = (byte) ((begin_iv >>> 24) & 0xFF);
+ ivBytes[1] = (byte) ((begin_iv >>> 16) & 0xFF);
+ ivBytes[2] = (byte) ((begin_iv >>> 8) & 0xFF);
+ ivBytes[3] = (byte) (begin_iv & 0xFF);
+ byte[] encrypted = sm4.sm4_crypt_ecb(ctx, ivBytes);
+ for (int j = 0; j < 16; j++) {
+ result[16 * i + j] = (byte) (data[16 * i + j] ^ encrypted[j]);
+ }
+ begin_iv++;
+ }
+ if (data.length % 16 != 0) {
+ ivBytes[0] = (byte) ((begin_iv >>> 24) & 0xFF);
+ ivBytes[1] = (byte) ((begin_iv >>> 16) & 0xFF);
+ ivBytes[2] = (byte) ((begin_iv >>> 8) & 0xFF);
+ ivBytes[3] = (byte) (begin_iv & 0xFF);
+ byte[] encrypted = sm4.sm4_crypt_ecb(ctx, ivBytes);
+ for (int j = 0; j < data.length % 16; j++) {
+ result[16 * group_num + j] = (byte) (data[16 * group_num + j] ^ encrypted[j]);
+ }
+ }
+ return result;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4_Context.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4_Context.java
new file mode 100644
index 00000000..53749ccb
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/SM4_Context.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.encrypt;
+
+public class SM4_Context {
+ public int mode;
+
+ public long[] sk;
+
+ public boolean isPadding;
+
+ public SM4_Context() {
+ this.mode = 1;
+ this.isPadding = false;
+ this.sk = new long[32];
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/encrypt/EncryptException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/encrypt/EncryptException.java
new file mode 100644
index 00000000..1c540ede
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/encrypt/EncryptException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.exception.encrypt;
+
+public class EncryptException extends RuntimeException {
+ public EncryptException(String message) {
+ super("Encrypt problem: " + message);
+ }
+
+ public EncryptException(String message, Throwable cause) {
+ super("Encrypt problem: " + message, cause);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/encrypt/EncryptKeyLengthNotMatchException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/encrypt/EncryptKeyLengthNotMatchException.java
new file mode 100644
index 00000000..e0ba6168
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/encrypt/EncryptKeyLengthNotMatchException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.exception.encrypt;
+
+public class EncryptKeyLengthNotMatchException extends RuntimeException {
+ public EncryptKeyLengthNotMatchException(int neededLength, int offerLength) {
+ super(
+ "Encrypt Key Length Not Match, "
+ + "need length: "
+ + neededLength
+ + ", offer length: "
+ + offerLength);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index ec3878c2..bcf5d134 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,7 +19,12 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.compatibility.DeserializeConfig;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IDecryptor;
+import org.apache.tsfile.encrypt.IEncryptor;
+import org.apache.tsfile.exception.encrypt.EncryptException;
import org.apache.tsfile.utils.BloomFilter;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -48,11 +53,15 @@ public class TsFileMetadata {
// offset from MetaMarker.SEPARATOR (exclusive) to tsFileProperties
private int propertiesOffset;
+ private byte[] dataEncryptKey;
+
+ private String encryptType;
+
/**
* deserialize data from the buffer.
*
* @param buffer -buffer use to deserialize
- * @return -a instance of TsFileMetaData
+ * @return -an instance of TsFileMetaData
*/
public static TsFileMetadata deserializeFrom(ByteBuffer buffer, DeserializeConfig context) {
TsFileMetadata fileMetaData = new TsFileMetadata();
@@ -105,12 +114,75 @@ public class TsFileMetadata {
String value = ReadWriteIOUtils.readVarIntString(buffer);
propertiesMap.put(key, value);
}
+ // if the file is not encrypted, set the default value(for compatible reason)
+ if (!propertiesMap.containsKey("encryptLevel") || propertiesMap.get("encryptLevel") == null) {
+ propertiesMap.put("encryptLevel", "0");
+ propertiesMap.put("encryptType", "UNENCRYPTED");
+ propertiesMap.put("encryptKey", "");
+ } else if (propertiesMap.get("encryptLevel").equals("0")) {
+ propertiesMap.put("encryptType", "UNENCRYPTED");
+ propertiesMap.put("encryptKey", "");
+ } else if (propertiesMap.get("encryptLevel").equals("1")) {
+ if (!propertiesMap.containsKey("encryptType")) {
+ throw new EncryptException("TsfileMetadata lack of encryptType while encryptLevel is 1");
+ }
+ if (!propertiesMap.containsKey("encryptKey")) {
+ throw new EncryptException("TsfileMetadata lack of encryptKey while encryptLevel is 1");
+ }
+ if (propertiesMap.get("encryptKey") == null || propertiesMap.get("encryptKey").isEmpty()) {
+ throw new EncryptException("TsfileMetadata null encryptKey while encryptLevel is 1");
+ }
+ String str = propertiesMap.get("encryptKey");
+ fileMetaData.dataEncryptKey = EncryptUtils.getKeyFromStr(str);
+ fileMetaData.encryptType = propertiesMap.get("encryptType");
+ } else if (propertiesMap.get("encryptLevel").equals("2")) {
+ if (!propertiesMap.containsKey("encryptType")) {
+ throw new EncryptException("TsfileMetadata lack of encryptType while encryptLevel is 2");
+ }
+ if (!propertiesMap.containsKey("encryptKey")) {
+ throw new EncryptException("TsfileMetadata lack of encryptKey while encryptLevel is 2");
+ }
+ if (propertiesMap.get("encryptKey") == null || propertiesMap.get("encryptKey").isEmpty()) {
+ throw new EncryptException("TsfileMetadata null encryptKey while encryptLevel is 2");
+ }
+ IDecryptor decryptor =
+ IDecryptor.getDecryptor(
+ TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+ TSFileDescriptor.getInstance().getConfig().getEncryptKey().getBytes());
+ String str = propertiesMap.get("encryptKey");
+ fileMetaData.dataEncryptKey = decryptor.decrypt(EncryptUtils.getKeyFromStr(str));
+ fileMetaData.encryptType = propertiesMap.get("encryptType");
+ } else {
+ throw new EncryptException(
+ "Unsupported encryptLevel: " + propertiesMap.get("encryptLevel"));
+ }
fileMetaData.tsFileProperties = propertiesMap;
}
return fileMetaData;
}
+ public IEncryptor getIEncryptor() {
+ if (dataEncryptKey == null) {
+ return IEncryptor.getEncryptor("UNENCRYPTED", null);
+ }
+ return IEncryptor.getEncryptor(encryptType, dataEncryptKey);
+ }
+
+ public IDecryptor getIDecryptor() {
+ if (dataEncryptKey == null) {
+ return IDecryptor.getDecryptor("UNENCRYPTED", null);
+ }
+ return IDecryptor.getDecryptor(encryptType, dataEncryptKey);
+ }
+
+ public void addProperty(String key, String value) {
+ if (tsFileProperties == null) {
+ tsFileProperties = new HashMap<>();
+ }
+ tsFileProperties.put(key, value);
+ }
+
public BloomFilter getBloomFilter() {
return bloomFilter;
}
@@ -177,11 +249,13 @@ public class TsFileMetadata {
int byteLen = 0;
byte[] bytes = filter.serialize();
byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(bytes.length, outputStream);
- outputStream.write(bytes);
- byteLen += bytes.length;
- byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(filter.getSize(), outputStream);
- byteLen +=
- ReadWriteForEncodingUtils.writeUnsignedVarInt(filter.getHashFunctionSize(), outputStream);
+ if (bytes.length > 0) {
+ outputStream.write(bytes);
+ byteLen += bytes.length;
+ byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(filter.getSize(), outputStream);
+ byteLen +=
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(filter.getHashFunctionSize(), outputStream);
+ }
return byteLen;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java
new file mode 100644
index 00000000..6fd4e516
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.file.metadata.enums;
+
+public enum EncryptionType {
+ /** Do not comprocess. */
+ UNENCRYPTED("UNENCRYPTED", (byte) 0),
+
+ /** SM4128. */
+ SM4128("SM4128", (byte) 1),
+
+ /** AES128. */
+ AES128("AES128", (byte) 2);
+
+ private final String extensionName;
+ private final byte index;
+
+ EncryptionType(String extensionName, byte index) {
+ this.extensionName = extensionName;
+ this.index = index;
+ }
+
+ /**
+ * deserialize byte number.
+ *
+ * @param encryptor byte number
+ * @return CompressionType
+ * @throws IllegalArgumentException illegal argument
+ */
+ public static EncryptionType deserialize(byte encryptor) {
+ switch (encryptor) {
+ case 0:
+ return EncryptionType.UNENCRYPTED;
+ case 1:
+ return EncryptionType.SM4128;
+ case 2:
+ return EncryptionType.AES128;
+ default:
+ throw new IllegalArgumentException("Invalid input: " + encryptor);
+ }
+ }
+
+ public static int getSerializedSize() {
+ return Byte.BYTES;
+ }
+
+ /**
+ * get serialized size.
+ *
+ * @return byte of index
+ */
+ public byte serialize() {
+ return this.index;
+ }
+
+ /**
+ * get extension.
+ *
+ * @return extension (string type), for example: SM4128、AES128
+ */
+ public String getExtension() {
+ return extensionName;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index b93dbb3d..a69d7872 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -26,6 +26,8 @@ import org.apache.tsfile.compatibility.CompatibilityUtils;
import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IDecryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.StopReadTsFileByInterruptException;
import org.apache.tsfile.exception.TsFileRuntimeException;
@@ -49,6 +51,7 @@ import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.file.metadata.statistics.Statistics;
@@ -339,7 +342,7 @@ public class TsFileSequenceReader implements AutoCloseable {
} catch (StopReadTsFileByInterruptException e) {
throw e;
} catch (Exception e) {
- logger.error("Something error happened while reading file metadata of file {}", file);
+ logger.error("Something error happened while reading file metadata of file {}", file, e);
throw e;
}
return tsFileMetaData;
@@ -355,6 +358,24 @@ public class TsFileSequenceReader implements AutoCloseable {
return tsFileMetaData.getBloomFilter();
}
+ /**
+ * Retrieves the decryptor for the TsFile. This method reads the file metadata to obtain the
+ * decryptor information. If an error occurs while reading the metadata, it logs the error and
+ * attempts to retrieve the decryptor based on the configuration settings.
+ *
+ * @return the decryptor for the TsFile
+ * @throws IOException if an I/O error occurs while reading the file metadata
+ */
+ public IDecryptor getDecryptor() throws IOException {
+ try {
+ readFileMetadata();
+ } catch (Exception e) {
+ logger.error("Something error happened while reading file metadata of file {}", file, e);
+ return EncryptUtils.decryptor;
+ }
+ return tsFileMetaData.getIDecryptor();
+ }
+
/**
* this function reads measurements and TimeseriesMetaDatas in given device Thread Safe
*
@@ -1659,7 +1680,7 @@ public class TsFileSequenceReader implements AutoCloseable {
try {
ChunkHeader header = readChunkHeader(offset);
ByteBuffer buffer = readChunk(offset + header.getSerializedSize(), header.getDataSize());
- return new Chunk(header, buffer);
+ return new Chunk(header, buffer, getDecryptor());
} catch (StopReadTsFileByInterruptException e) {
throw e;
} catch (Throwable t) {
@@ -1680,7 +1701,12 @@ public class TsFileSequenceReader implements AutoCloseable {
ByteBuffer buffer =
readChunk(
metaData.getOffsetOfChunkHeader() + header.getSerializedSize(), header.getDataSize());
- return new Chunk(header, buffer, metaData.getDeleteIntervalList(), metaData.getStatistics());
+ return new Chunk(
+ header,
+ buffer,
+ metaData.getDeleteIntervalList(),
+ metaData.getStatistics(),
+ getDecryptor());
} catch (StopReadTsFileByInterruptException e) {
throw e;
} catch (Throwable t) {
@@ -1702,7 +1728,11 @@ public class TsFileSequenceReader implements AutoCloseable {
chunkCacheKey.getOffsetOfChunkHeader() + header.getSerializedSize(),
header.getDataSize());
return new Chunk(
- header, buffer, chunkCacheKey.getDeleteIntervalList(), chunkCacheKey.getStatistics());
+ header,
+ buffer,
+ chunkCacheKey.getDeleteIntervalList(),
+ chunkCacheKey.getStatistics(),
+ getDecryptor());
}
/**
@@ -1774,13 +1804,37 @@ public class TsFileSequenceReader implements AutoCloseable {
public ByteBuffer readPage(PageHeader header, CompressionType type) throws IOException {
ByteBuffer buffer = readData(-1, header.getCompressedSize());
- if (header.getUncompressedSize() == 0 || type == CompressionType.UNCOMPRESSED) {
+ IDecryptor decryptor = getDecryptor();
+ if (header.getUncompressedSize() == 0) {
return buffer;
- } // FIXME if the buffer is not array-implemented.
- IUnCompressor unCompressor = IUnCompressor.getUnCompressor(type);
- ByteBuffer uncompressedBuffer = ByteBuffer.allocate(header.getUncompressedSize());
+ }
+ ByteBuffer finalBuffer = decrypt(decryptor, buffer);
+ finalBuffer = uncompress(type, finalBuffer, header.getUncompressedSize());
+ return finalBuffer;
+ }
+
+ private static ByteBuffer decrypt(IDecryptor decryptor, ByteBuffer buffer) {
+ if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) {
+ return buffer;
+ }
+ return ByteBuffer.wrap(
+ decryptor.decrypt(
+ buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()));
+ }
+
+ private static ByteBuffer uncompress(
+ CompressionType compressionType, ByteBuffer buffer, int uncompressedSize) throws IOException {
+ if (compressionType == CompressionType.UNCOMPRESSED) {
+ return buffer;
+ }
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(compressionType);
+ ByteBuffer uncompressedBuffer = ByteBuffer.allocate(uncompressedSize);
unCompressor.uncompress(
- buffer.array(), buffer.position(), buffer.remaining(), uncompressedBuffer.array(), 0);
+ buffer.array(),
+ buffer.arrayOffset() + buffer.position(),
+ buffer.remaining(),
+ uncompressedBuffer.array(),
+ 0);
return uncompressedBuffer;
}
@@ -1884,7 +1938,7 @@ public class TsFileSequenceReader implements AutoCloseable {
/**
* Self Check the file and return the position before where the data is safe.
*
- * @param newSchema the schema on each time series in the file
+ * @param schema the schema on each time series in the file
* @param chunkGroupMetadataList ChunkGroupMetadata List
* @param fastFinish if true and the file is complete, then newSchema and chunkGroupMetadataList
* parameter will be not modified.
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
index 5caab7fe..18709abc 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
@@ -19,6 +19,8 @@
package org.apache.tsfile.read.common;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IDecryptor;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.metadata.statistics.Statistics;
@@ -44,6 +46,8 @@ public class Chunk {
private ByteBuffer chunkData;
private Statistics chunkStatistic;
+ private IDecryptor decryptor;
+
/** A list of deleted intervals. */
private List<TimeRange> deleteIntervalList;
@@ -56,11 +60,36 @@ public class Chunk {
this.chunkData = buffer;
this.deleteIntervalList = deleteIntervalList;
this.chunkStatistic = chunkStatistic;
+ this.decryptor = EncryptUtils.decryptor;
+ }
+
+ public Chunk(
+ ChunkHeader header,
+ ByteBuffer buffer,
+ List<TimeRange> deleteIntervalList,
+ Statistics chunkStatistic,
+ IDecryptor decryptor) {
+ this.chunkHeader = header;
+ this.chunkData = buffer;
+ this.deleteIntervalList = deleteIntervalList;
+ this.chunkStatistic = chunkStatistic;
+ this.decryptor = decryptor;
}
public Chunk(ChunkHeader header, ByteBuffer buffer) {
this.chunkHeader = header;
this.chunkData = buffer;
+ this.decryptor = EncryptUtils.decryptor;
+ }
+
+ public Chunk(ChunkHeader header, ByteBuffer buffer, IDecryptor decryptor) {
+ this.chunkHeader = header;
+ this.chunkData = buffer;
+ this.decryptor = decryptor;
+ }
+
+ public IDecryptor getDecryptor() {
+ return decryptor;
}
public ChunkHeader getHeader() {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java
index 89d66c68..9ba86913 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java
@@ -73,7 +73,8 @@ public class CachedChunkLoaderImpl implements IChunkLoader {
chunk.getHeader(),
chunk.getData().duplicate(),
chunkMetaData.getDeleteIntervalList(),
- chunkMetaData.getStatistics());
+ chunkMetaData.getStatistics(),
+ reader.getDecryptor());
}
@Override
@@ -90,7 +91,8 @@ public class CachedChunkLoaderImpl implements IChunkLoader {
chunk.getHeader(),
chunk.getData().duplicate(),
chunkMetaData.getDeleteIntervalList(),
- chunkMetaData.getStatistics()),
+ chunkMetaData.getStatistics(),
+ reader.getDecryptor()),
globalTimeFilter);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
index 7191d853..865fdd21 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
@@ -21,6 +21,7 @@ package org.apache.tsfile.read.reader.chunk;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
+import org.apache.tsfile.encrypt.IDecryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkHeader;
@@ -52,6 +53,8 @@ public class AlignedChunkReader extends AbstractChunkReader {
// deleted intervals of all the sub sensors
private final List<List<TimeRange>> valueDeleteIntervalsList = new ArrayList<>();
+ private final IDecryptor decrytor;
+
@SuppressWarnings("unchecked")
public AlignedChunkReader(
Chunk timeChunk, List<Chunk> valueChunkList, long readStopTime, Filter queryFilter)
@@ -69,7 +72,7 @@ public class AlignedChunkReader extends AbstractChunkReader {
valueChunkStatisticsList.add(chunk == null ? null : chunk.getChunkStatistic());
});
-
+ this.decrytor = timeChunk.getDecryptor();
initAllPageReaders(timeChunk.getChunkStatistic(), valueChunkStatisticsList);
}
@@ -192,7 +195,8 @@ public class AlignedChunkReader extends AbstractChunkReader {
private AlignedPageReader constructAlignedPageReader(
PageHeader timePageHeader, List<PageHeader> rawValuePageHeaderList) throws IOException {
ByteBuffer timePageData =
- ChunkReader.deserializePageData(timePageHeader, timeChunkDataBuffer, timeChunkHeader);
+ ChunkReader.deserializePageData(
+ timePageHeader, timeChunkDataBuffer, timeChunkHeader, decrytor);
List<PageHeader> valuePageHeaderList = new ArrayList<>();
LazyLoadPageData[] lazyLoadPageDataArray = new LazyLoadPageData[rawValuePageHeaderList.size()];
@@ -232,7 +236,8 @@ public class AlignedChunkReader extends AbstractChunkReader {
new LazyLoadPageData(
valueChunkDataBufferList.get(i).array(),
currentPagePosition,
- IUnCompressor.getUnCompressor(valueChunkHeader.getCompressionType()));
+ IUnCompressor.getUnCompressor(valueChunkHeader.getCompressionType()),
+ decrytor);
valueDataTypeList.add(valueChunkHeader.getDataType());
valueDecoderList.add(
Decoder.getDecoderByType(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
index 1b747739..87d3c585 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
@@ -21,9 +21,11 @@ package org.apache.tsfile.read.reader.chunk;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
+import org.apache.tsfile.encrypt.IDecryptor;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.common.Chunk;
import org.apache.tsfile.read.common.TimeRange;
@@ -42,13 +44,15 @@ public class ChunkReader extends AbstractChunkReader {
private final ByteBuffer chunkDataBuffer;
private final List<TimeRange> deleteIntervalList;
+ private final IDecryptor decryptor;
+
@SuppressWarnings("unchecked")
public ChunkReader(Chunk chunk, long readStopTime, Filter queryFilter) {
super(readStopTime, queryFilter);
this.chunkHeader = chunk.getHeader();
this.chunkDataBuffer = chunk.getData();
this.deleteIntervalList = chunk.getDeleteIntervalList();
-
+ this.decryptor = chunk.getDecryptor();
initAllPageReaders(chunk.getChunkStatistic());
}
@@ -133,7 +137,8 @@ public class ChunkReader extends AbstractChunkReader {
PageReader reader =
new PageReader(
pageHeader,
- new LazyLoadPageData(chunkDataBuffer.array(), currentPagePosition, unCompressor),
+ new LazyLoadPageData(
+ chunkDataBuffer.array(), currentPagePosition, unCompressor, decryptor),
chunkHeader.getDataType(),
Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()),
defaultTimeDecoder,
@@ -188,6 +193,48 @@ public class ChunkReader extends AbstractChunkReader {
return ByteBuffer.wrap(uncompressedPageData);
}
+ public static ByteBuffer decryptAndUncompressPageData(
+ PageHeader pageHeader,
+ IUnCompressor unCompressor,
+ ByteBuffer compressedPageData,
+ IDecryptor decryptor)
+ throws IOException {
+ int compressedPageBodyLength = pageHeader.getCompressedSize();
+ byte[] uncompressedPageData = new byte[pageHeader.getUncompressedSize()];
+ try {
+ byte[] decryptedPageData =
+ decryptor.decrypt(
+ compressedPageData.array(),
+ compressedPageData.arrayOffset() + compressedPageData.position(),
+ compressedPageBodyLength);
+ unCompressor.uncompress(
+ decryptedPageData, 0, compressedPageBodyLength, uncompressedPageData, 0);
+ } catch (Exception e) {
+ throw new IOException(
+ "Uncompress error! uncompress size: "
+ + pageHeader.getUncompressedSize()
+ + "compressed size: "
+ + pageHeader.getCompressedSize()
+ + "page header: "
+ + pageHeader
+ + e.getMessage());
+ }
+ compressedPageData.position(compressedPageData.position() + compressedPageBodyLength);
+ return ByteBuffer.wrap(uncompressedPageData);
+ }
+
+ public static ByteBuffer deserializePageData(
+ PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader, IDecryptor decryptor)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType());
+ ByteBuffer compressedPageBody = readCompressedPageData(pageHeader, chunkBuffer);
+ if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) {
+ return uncompressPageData(pageHeader, unCompressor, compressedPageBody);
+ } else {
+ return decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageBody, decryptor);
+ }
+ }
+
public static ByteBuffer deserializePageData(
PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader) throws IOException {
IUnCompressor unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType());
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java
index d89cc2d8..481e8812 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java
@@ -20,6 +20,8 @@
package org.apache.tsfile.read.reader.page;
import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IDecryptor;
import org.apache.tsfile.file.header.PageHeader;
import java.io.IOException;
@@ -33,18 +35,31 @@ public class LazyLoadPageData {
private final IUnCompressor unCompressor;
+ private final IDecryptor decryptor;
+
public LazyLoadPageData(byte[] data, int offset, IUnCompressor unCompressor) {
this.chunkData = data;
this.pageDataOffset = offset;
this.unCompressor = unCompressor;
+ this.decryptor = EncryptUtils.decryptor;
+ }
+
+ public LazyLoadPageData(
+ byte[] data, int offset, IUnCompressor unCompressor, IDecryptor decryptor) {
+ this.chunkData = data;
+ this.pageDataOffset = offset;
+ this.unCompressor = unCompressor;
+ this.decryptor = decryptor;
}
public ByteBuffer uncompressPageData(PageHeader pageHeader) throws IOException {
int compressedPageBodyLength = pageHeader.getCompressedSize();
byte[] uncompressedPageData = new byte[pageHeader.getUncompressedSize()];
try {
+ byte[] decryptedPageData =
+ decryptor.decrypt(chunkData, pageDataOffset, compressedPageBodyLength);
unCompressor.uncompress(
- chunkData, pageDataOffset, compressedPageBodyLength, uncompressedPageData, 0);
+ decryptedPageData, 0, compressedPageBodyLength, uncompressedPageData, 0);
} catch (Exception e) {
throw new IOException(
"Uncompress error! uncompress size: "
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index f3cc2036..b1f60735 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -20,6 +20,8 @@ package org.apache.tsfile.write;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.encrypt.IEncryptor;
+import org.apache.tsfile.exception.encrypt.EncryptException;
import org.apache.tsfile.exception.write.ConflictDataTypeException;
import org.apache.tsfile.exception.write.NoDeviceException;
import org.apache.tsfile.exception.write.NoMeasurementException;
@@ -27,6 +29,7 @@ import org.apache.tsfile.exception.write.NoTableException;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.utils.MeasurementGroup;
import org.apache.tsfile.utils.Pair;
@@ -48,6 +51,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -69,6 +73,8 @@ public class TsFileWriter implements AutoCloseable {
/** IO writer of this TsFile. */
private final TsFileIOWriter fileWriter;
+ private IEncryptor encryptor;
+
private final int pageSize;
private long recordCount = 0;
@@ -175,6 +181,46 @@ public class TsFileWriter implements AutoCloseable {
pageSize,
chunkGroupSizeThreshold);
}
+
+ String encryptLevel;
+ byte[] encryptKey;
+ byte[] dataEncryptKey;
+ EncryptionType encryptType;
+ if (config.getEncryptFlag()) {
+ encryptLevel = "2";
+ encryptType = config.getEncryptType();
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update("IoTDB is the best".getBytes());
+ md.update(config.getEncryptKey().getBytes());
+ dataEncryptKey = md.digest();
+ encryptKey =
+ IEncryptor.getEncryptor(config.getEncryptType(), config.getEncryptKey().getBytes())
+ .encrypt(dataEncryptKey);
+ } catch (Exception e) {
+ throw new EncryptException("md5 function not found while using md5 to generate data key");
+ }
+ } else {
+ encryptLevel = "0";
+ encryptType = EncryptionType.UNENCRYPTED;
+ encryptKey = null;
+ dataEncryptKey = null;
+ }
+ this.encryptor = IEncryptor.getEncryptor(encryptType, dataEncryptKey);
+ if (encryptKey != null) {
+ StringBuilder valueStr = new StringBuilder();
+
+ for (byte b : encryptKey) {
+ valueStr.append(b).append(",");
+ }
+
+ valueStr.deleteCharAt(valueStr.length() - 1);
+ String str = valueStr.toString();
+
+ fileWriter.setEncryptParam(encryptLevel, encryptType.getExtension(), str);
+ } else {
+ fileWriter.setEncryptParam(encryptLevel, encryptType.getExtension(), "");
+ }
}
public void registerSchemaTemplate(
@@ -429,13 +475,13 @@ public class TsFileWriter implements AutoCloseable {
IChunkGroupWriter groupWriter = groupWriters.get(deviceId);
if (groupWriter == null) {
if (isAligned) {
- groupWriter = new AlignedChunkGroupWriterImpl(deviceId);
+ groupWriter = new AlignedChunkGroupWriterImpl(deviceId, encryptor);
if (!isUnseq) { // Sequence File
((AlignedChunkGroupWriterImpl) groupWriter)
.setLastTime(alignedDeviceLastTimeMap.getOrDefault(deviceId, -1L));
}
} else {
- groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId);
+ groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, encryptor);
if (!isUnseq) { // Sequence File
((NonAlignedChunkGroupWriterImpl) groupWriter)
.setLastTimeMap(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index 8cba8f6a..163e583b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -22,6 +22,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.encoding.encoder.Encoder;
import org.apache.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -57,6 +59,8 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter {
private final TimeChunkWriter timeChunkWriter;
+ private final IEncryptor encryptor;
+
private long lastTime = -1;
public AlignedChunkGroupWriterImpl(IDeviceID deviceId) {
@@ -67,9 +71,23 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter {
TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder());
TSDataType timeType = TSFileDescriptor.getInstance().getConfig().getTimeSeriesDataType();
Encoder encoder = TSEncodingBuilder.getEncodingBuilder(tsEncoding).getEncoder(timeType);
+ this.encryptor = EncryptUtils.encryptor;
timeChunkWriter = new TimeChunkWriter(timeMeasurementId, compressionType, tsEncoding, encoder);
}
+ public AlignedChunkGroupWriterImpl(IDeviceID deviceId, IEncryptor encryptor) {
+ this.deviceId = deviceId;
+ String timeMeasurementId = "";
+ CompressionType compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor();
+ TSEncoding tsEncoding =
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder());
+ TSDataType timeType = TSFileDescriptor.getInstance().getConfig().getTimeSeriesDataType();
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(tsEncoding).getEncoder(timeType);
+ this.encryptor = encryptor;
+ timeChunkWriter =
+ new TimeChunkWriter(timeMeasurementId, compressionType, tsEncoding, encoder, encryptor);
+ }
+
@Override
public void tryToAddSeriesWriter(IMeasurementSchema measurementSchema) throws IOException {
tryToAddSeriesWriterInternal(measurementSchema);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
index 22d310c2..87fe088a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
@@ -22,6 +22,8 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encoding.encoder.Encoder;
import org.apache.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.PageHeader;
@@ -46,6 +48,8 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
protected List<ValueChunkWriter> valueChunkWriterList;
protected int valueIndex;
+ protected IEncryptor encryptor;
+
// Used for batch writing
protected long remainingPointsNumber;
@@ -53,12 +57,45 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
// TestOnly
public AlignedChunkWriterImpl(VectorMeasurementSchema schema) {
+ this.encryptor = EncryptUtils.encryptor;
+ timeChunkWriter =
+ new TimeChunkWriter(
+ schema.getMeasurementId(),
+ schema.getCompressor(),
+ schema.getTimeTSEncoding(),
+ schema.getTimeEncoder(),
+ this.encryptor);
+
+ List<String> valueMeasurementIdList = schema.getSubMeasurementsList();
+ List<TSDataType> valueTSDataTypeList = schema.getSubMeasurementsTSDataTypeList();
+ List<TSEncoding> valueTSEncodingList = schema.getSubMeasurementsTSEncodingList();
+ List<Encoder> valueEncoderList = schema.getSubMeasurementsEncoderList();
+
+ valueChunkWriterList = new ArrayList<>(valueMeasurementIdList.size());
+ for (int i = 0; i < valueMeasurementIdList.size(); i++) {
+ valueChunkWriterList.add(
+ new ValueChunkWriter(
+ valueMeasurementIdList.get(i),
+ schema.getCompressor(),
+ valueTSDataTypeList.get(i),
+ valueTSEncodingList.get(i),
+ valueEncoderList.get(i),
+ this.encryptor));
+ }
+
+ this.valueIndex = 0;
+ this.remainingPointsNumber = timeChunkWriter.getRemainingPointNumberForCurrentPage();
+ }
+
+ public AlignedChunkWriterImpl(VectorMeasurementSchema schema, IEncryptor encryptor) {
+ this.encryptor = encryptor;
timeChunkWriter =
new TimeChunkWriter(
schema.getMeasurementId(),
schema.getCompressor(),
schema.getTimeTSEncoding(),
- schema.getTimeEncoder());
+ schema.getTimeEncoder(),
+ this.encryptor);
List<String> valueMeasurementIdList = schema.getSubMeasurementsList();
List<TSDataType> valueTSDataTypeList = schema.getSubMeasurementsTSDataTypeList();
@@ -73,7 +110,8 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
schema.getCompressor(),
valueTSDataTypeList.get(i),
valueTSEncodingList.get(i),
- valueEncoderList.get(i)));
+ valueEncoderList.get(i),
+ this.encryptor));
}
this.valueIndex = 0;
@@ -89,12 +127,43 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
*/
public AlignedChunkWriterImpl(
IMeasurementSchema timeSchema, List<IMeasurementSchema> valueSchemaList) {
+ this.encryptor = EncryptUtils.encryptor;
+ timeChunkWriter =
+ new TimeChunkWriter(
+ timeSchema.getMeasurementId(),
+ timeSchema.getCompressor(),
+ timeSchema.getEncodingType(),
+ timeSchema.getTimeEncoder(),
+ this.encryptor);
+
+ valueChunkWriterList = new ArrayList<>(valueSchemaList.size());
+ for (int i = 0; i < valueSchemaList.size(); i++) {
+ valueChunkWriterList.add(
+ new ValueChunkWriter(
+ valueSchemaList.get(i).getMeasurementId(),
+ valueSchemaList.get(i).getCompressor(),
+ valueSchemaList.get(i).getType(),
+ valueSchemaList.get(i).getEncodingType(),
+ valueSchemaList.get(i).getValueEncoder(),
+ this.encryptor));
+ }
+
+ this.valueIndex = 0;
+ this.remainingPointsNumber = timeChunkWriter.getRemainingPointNumberForCurrentPage();
+ }
+
+ public AlignedChunkWriterImpl(
+ IMeasurementSchema timeSchema,
+ List<IMeasurementSchema> valueSchemaList,
+ IEncryptor encryptor) {
+ this.encryptor = encryptor;
timeChunkWriter =
new TimeChunkWriter(
timeSchema.getMeasurementId(),
timeSchema.getCompressor(),
timeSchema.getEncodingType(),
- timeSchema.getTimeEncoder());
+ timeSchema.getTimeEncoder(),
+ this.encryptor);
valueChunkWriterList = new ArrayList<>(valueSchemaList.size());
for (int i = 0; i < valueSchemaList.size(); i++) {
@@ -104,7 +173,8 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
valueSchemaList.get(i).getCompressor(),
valueSchemaList.get(i).getType(),
valueSchemaList.get(i).getEncodingType(),
- valueSchemaList.get(i).getValueEncoder()));
+ valueSchemaList.get(i).getValueEncoder(),
+ this.encryptor));
}
this.valueIndex = 0;
@@ -119,6 +189,38 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
* @param schemaList value schema list
*/
public AlignedChunkWriterImpl(List<IMeasurementSchema> schemaList) {
+ this.encryptor = EncryptUtils.encryptor;
+ TSEncoding timeEncoding =
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder());
+ TSDataType timeType = TSFileDescriptor.getInstance().getConfig().getTimeSeriesDataType();
+ CompressionType timeCompression = TSFileDescriptor.getInstance().getConfig().getCompressor();
+ timeChunkWriter =
+ new TimeChunkWriter(
+ "",
+ timeCompression,
+ timeEncoding,
+ TSEncodingBuilder.getEncodingBuilder(timeEncoding).getEncoder(timeType),
+ this.encryptor);
+
+ valueChunkWriterList = new ArrayList<>(schemaList.size());
+ for (int i = 0; i < schemaList.size(); i++) {
+ valueChunkWriterList.add(
+ new ValueChunkWriter(
+ schemaList.get(i).getMeasurementId(),
+ schemaList.get(i).getCompressor(),
+ schemaList.get(i).getType(),
+ schemaList.get(i).getEncodingType(),
+ schemaList.get(i).getValueEncoder(),
+ this.encryptor));
+ }
+
+ this.valueIndex = 0;
+
+ this.remainingPointsNumber = timeChunkWriter.getRemainingPointNumberForCurrentPage();
+ }
+
+ public AlignedChunkWriterImpl(List<IMeasurementSchema> schemaList, IEncryptor encryptor) {
+ this.encryptor = encryptor;
TSEncoding timeEncoding =
TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder());
TSDataType timeType = TSFileDescriptor.getInstance().getConfig().getTimeSeriesDataType();
@@ -128,7 +230,8 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
"",
timeCompression,
timeEncoding,
- TSEncodingBuilder.getEncodingBuilder(timeEncoding).getEncoder(timeType));
+ TSEncodingBuilder.getEncodingBuilder(timeEncoding).getEncoder(timeType),
+ this.encryptor);
valueChunkWriterList = new ArrayList<>(schemaList.size());
for (int i = 0; i < schemaList.size(); i++) {
@@ -138,7 +241,8 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
schemaList.get(i).getCompressor(),
schemaList.get(i).getType(),
schemaList.get(i).getEncodingType(),
- schemaList.get(i).getValueEncoder()));
+ schemaList.get(i).getValueEncoder(),
+ this.encryptor));
}
this.valueIndex = 0;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java
index feb0916a..e04c7f54 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java
@@ -21,6 +21,8 @@ package org.apache.tsfile.write.chunk;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.encoding.encoder.SDTEncoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.ChunkHeader;
@@ -50,6 +52,8 @@ public class ChunkWriterImpl implements IChunkWriter {
private final ICompressor compressor;
+ private final IEncryptor encryptor;
+
/** all pages of this chunk. */
private final PublicBAOS pageBuffer;
@@ -99,6 +103,7 @@ public class ChunkWriterImpl implements IChunkWriter {
public ChunkWriterImpl(IMeasurementSchema schema) {
this.measurementSchema = schema;
this.compressor = ICompressor.getCompressor(schema.getCompressor());
+ this.encryptor = EncryptUtils.encryptor;
this.pageBuffer = new PublicBAOS();
this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
@@ -110,7 +115,31 @@ public class ChunkWriterImpl implements IChunkWriter {
// init statistics for this chunk and page
this.statistics = Statistics.getStatsByType(measurementSchema.getType());
- this.pageWriter = new PageWriter(measurementSchema);
+ this.pageWriter = new PageWriter(measurementSchema, encryptor);
+
+ this.pageWriter.setTimeEncoder(measurementSchema.getTimeEncoder());
+ this.pageWriter.setValueEncoder(measurementSchema.getValueEncoder());
+
+ // check if the measurement schema uses SDT
+ checkSdtEncoding();
+ }
+
+ public ChunkWriterImpl(IMeasurementSchema schema, IEncryptor encryptor) {
+ this.measurementSchema = schema;
+ this.compressor = ICompressor.getCompressor(schema.getCompressor());
+ this.encryptor = encryptor;
+ this.pageBuffer = new PublicBAOS();
+
+ this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ this.maxNumberOfPointsInPage =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ // initial check of memory usage. So that we have enough data to make an initial prediction
+ this.valueCountInOnePageForNextCheck = MINIMUM_RECORD_COUNT_FOR_CHECK;
+
+ // init statistics for this chunk and page
+ this.statistics = Statistics.getStatsByType(measurementSchema.getType());
+
+ this.pageWriter = new PageWriter(measurementSchema, encryptor);
this.pageWriter.setTimeEncoder(measurementSchema.getTimeEncoder());
this.pageWriter.setValueEncoder(measurementSchema.getValueEncoder());
@@ -124,6 +153,11 @@ public class ChunkWriterImpl implements IChunkWriter {
this.isMerging = isMerging;
}
+ public ChunkWriterImpl(IMeasurementSchema schema, boolean isMerging, IEncryptor encryptor) {
+ this(schema, encryptor);
+ this.isMerging = isMerging;
+ }
+
private void checkSdtEncoding() {
if (measurementSchema.getProps() != null && !isMerging) {
if (measurementSchema.getProps().getOrDefault(LOSS, "").equals(SDT)) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
index 3c453a30..223fffa1 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
@@ -19,6 +19,8 @@
package org.apache.tsfile.write.chunk;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -50,17 +52,25 @@ public class NonAlignedChunkGroupWriterImpl implements IChunkGroupWriter {
/** Map(measurementID, ChunkWriterImpl). Aligned measurementId is empty. */
private final Map<String, ChunkWriterImpl> chunkWriters = new LinkedHashMap<>();
+ private IEncryptor encryptor;
+
// measurementId -> lastTime
private Map<String, Long> lastTimeMap = new HashMap<>();
public NonAlignedChunkGroupWriterImpl(IDeviceID deviceId) {
this.deviceId = deviceId;
+ this.encryptor = EncryptUtils.encryptor;
+ }
+
+ public NonAlignedChunkGroupWriterImpl(IDeviceID deviceId, IEncryptor encryptor) {
+ this.deviceId = deviceId;
+ this.encryptor = encryptor;
}
@Override
public void tryToAddSeriesWriter(IMeasurementSchema schema) {
if (!chunkWriters.containsKey(schema.getMeasurementId())) {
- this.chunkWriters.put(schema.getMeasurementId(), new ChunkWriterImpl(schema));
+ this.chunkWriters.put(schema.getMeasurementId(), new ChunkWriterImpl(schema, encryptor));
}
}
@@ -68,7 +78,7 @@ public class NonAlignedChunkGroupWriterImpl implements IChunkGroupWriter {
public void tryToAddSeriesWriter(List<IMeasurementSchema> schemas) {
for (IMeasurementSchema schema : schemas) {
if (!chunkWriters.containsKey(schema.getMeasurementId())) {
- this.chunkWriters.put(schema.getMeasurementId(), new ChunkWriterImpl(schema));
+ this.chunkWriters.put(schema.getMeasurementId(), new ChunkWriterImpl(schema, encryptor));
}
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java
index 756031fc..4f04efb3 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java
@@ -22,6 +22,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.ChunkHeader;
@@ -53,6 +55,8 @@ public class TimeChunkWriter {
private CompressionType compressionType;
+ private IEncryptor encryptor;
+
/** all pages of this chunk. */
private PublicBAOS pageBuffer;
@@ -90,6 +94,32 @@ public class TimeChunkWriter {
this.measurementId = measurementId;
this.encodingType = encodingType;
this.compressionType = compressionType;
+ this.encryptor = EncryptUtils.encryptor;
+ this.pageBuffer = new PublicBAOS();
+
+ this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ this.maxNumberOfPointsInPage =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ // initial check of memory usage. So that we have enough data to make an initial prediction
+ this.valueCountInOnePageForNextCheck = MINIMUM_RECORD_COUNT_FOR_CHECK;
+
+ // init statistics for this chunk and page
+ this.statistics = new TimeStatistics();
+
+ this.pageWriter =
+ new TimePageWriter(timeEncoder, ICompressor.getCompressor(compressionType), this.encryptor);
+ }
+
+ public TimeChunkWriter(
+ String measurementId,
+ CompressionType compressionType,
+ TSEncoding encodingType,
+ Encoder timeEncoder,
+ IEncryptor encryptor) {
+ this.measurementId = measurementId;
+ this.encodingType = encodingType;
+ this.compressionType = compressionType;
+ this.encryptor = encryptor;
this.pageBuffer = new PublicBAOS();
this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
@@ -101,7 +131,8 @@ public class TimeChunkWriter {
// init statistics for this chunk and page
this.statistics = new TimeStatistics();
- this.pageWriter = new TimePageWriter(timeEncoder, ICompressor.getCompressor(compressionType));
+ this.pageWriter =
+ new TimePageWriter(timeEncoder, ICompressor.getCompressor(compressionType), encryptor);
}
public void write(long time) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java
index 788f741f..9ea908b2 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java
@@ -22,6 +22,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.ChunkHeader;
@@ -56,6 +58,8 @@ public class ValueChunkWriter {
private final CompressionType compressionType;
+ private final IEncryptor encryptor;
+
/** all pages of this chunk. */
private final PublicBAOS pageBuffer;
@@ -93,6 +97,33 @@ public class ValueChunkWriter {
this.encodingType = encodingType;
this.dataType = dataType;
this.compressionType = compressionType;
+ this.encryptor = EncryptUtils.encryptor;
+ this.pageBuffer = new PublicBAOS();
+ this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ this.maxNumberOfPointsInPage =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ this.valueCountInOnePageForNextCheck = MINIMUM_RECORD_COUNT_FOR_CHECK;
+
+ // init statistics for this chunk and page
+ this.statistics = Statistics.getStatsByType(dataType);
+
+ this.pageWriter =
+ new ValuePageWriter(
+ valueEncoder, ICompressor.getCompressor(compressionType), dataType, this.encryptor);
+ }
+
+ public ValueChunkWriter(
+ String measurementId,
+ CompressionType compressionType,
+ TSDataType dataType,
+ TSEncoding encodingType,
+ Encoder valueEncoder,
+ IEncryptor encryptor) {
+ this.measurementId = measurementId;
+ this.encodingType = encodingType;
+ this.dataType = dataType;
+ this.compressionType = compressionType;
+ this.encryptor = encryptor;
this.pageBuffer = new PublicBAOS();
this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
this.maxNumberOfPointsInPage =
@@ -103,7 +134,8 @@ public class ValueChunkWriter {
this.statistics = Statistics.getStatsByType(dataType);
this.pageWriter =
- new ValuePageWriter(valueEncoder, ICompressor.getCompressor(compressionType), dataType);
+ new ValuePageWriter(
+ valueEncoder, ICompressor.getCompressor(compressionType), dataType, encryptor);
}
public void write(long time, long value, boolean isNull) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java
index 6d480c00..c89d019f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java
@@ -20,8 +20,11 @@ package org.apache.tsfile.write.page;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.PublicBAOS;
@@ -47,6 +50,8 @@ public class PageWriter {
private ICompressor compressor;
+ private IEncryptor encryptor;
+
// time
private Encoder timeEncoder;
private PublicBAOS timeOut;
@@ -61,7 +66,7 @@ public class PageWriter {
private Statistics<? extends Serializable> statistics;
public PageWriter() {
- this(null, null);
+ this((Encoder) null, null);
}
public PageWriter(IMeasurementSchema measurementSchema) {
@@ -75,6 +80,25 @@ public class PageWriter {
this.valueOut = new PublicBAOS();
this.timeEncoder = timeEncoder;
this.valueEncoder = valueEncoder;
+ this.encryptor = EncryptUtils.encryptor;
+ }
+
+ public PageWriter(IEncryptor encryptor) {
+ this(null, null, encryptor);
+ }
+
+ public PageWriter(IMeasurementSchema measurementSchema, IEncryptor encryptor) {
+ this(measurementSchema.getTimeEncoder(), measurementSchema.getValueEncoder(), encryptor);
+ this.statistics = Statistics.getStatsByType(measurementSchema.getType());
+ this.compressor = ICompressor.getCompressor(measurementSchema.getCompressor());
+ }
+
+ private PageWriter(Encoder timeEncoder, Encoder valueEncoder, IEncryptor encryptor) {
+ this.timeOut = new PublicBAOS();
+ this.valueOut = new PublicBAOS();
+ this.timeEncoder = timeEncoder;
+ this.valueEncoder = valueEncoder;
+ this.encryptor = encryptor;
}
/** write a time value pair into encoder */
@@ -244,11 +268,28 @@ public class PageWriter {
// write page content to temp PBAOS
logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size());
if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) {
- try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
- channel.write(pageData);
+ if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) {
+ try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
+ channel.write(pageData);
+ }
+ } else {
+ byte[] encryptedBytes = null;
+ encryptedBytes = encryptor.encrypt(pageData.array(), pageData.position(), uncompressedSize);
+ // data is never a directByteBuffer now, so we can use data.array()
+ int encryptedSize = encryptedBytes.length;
+ pageBuffer.write(encryptedBytes, 0, encryptedSize);
}
+
} else {
- pageBuffer.write(compressedBytes, 0, compressedSize);
+ if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) {
+ pageBuffer.write(compressedBytes, 0, compressedSize);
+ } else {
+ byte[] encryptedBytes = null;
+ encryptedBytes = encryptor.encrypt(compressedBytes, 0, compressedSize);
+ // data is never a directByteBuffer now, so we can use data.array()
+ int encryptedSize = encryptedBytes.length;
+ pageBuffer.write(encryptedBytes, 0, encryptedSize);
+ }
}
logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size());
return sizeWithoutStatistic;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java
index 31f3d2a0..5faea9d9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java
@@ -20,7 +20,10 @@ package org.apache.tsfile.write.page;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.file.metadata.statistics.TimeStatistics;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
@@ -43,6 +46,8 @@ public class TimePageWriter {
private final ICompressor compressor;
+ private final IEncryptor encryptor;
+
// time
private Encoder timeEncoder;
private final PublicBAOS timeOut;
@@ -58,6 +63,15 @@ public class TimePageWriter {
this.timeEncoder = timeEncoder;
this.statistics = new TimeStatistics();
this.compressor = compressor;
+ this.encryptor = EncryptUtils.encryptor;
+ }
+
+ public TimePageWriter(Encoder timeEncoder, ICompressor compressor, IEncryptor encryptor) {
+ this.timeOut = new PublicBAOS();
+ this.timeEncoder = timeEncoder;
+ this.statistics = new TimeStatistics();
+ this.compressor = compressor;
+ this.encryptor = encryptor;
}
/** write a time into encoder */
@@ -138,11 +152,28 @@ public class TimePageWriter {
logger.trace(
"start to flush a time page data into buffer, buffer position {} ", pageBuffer.size());
if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) {
- try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
- channel.write(pageData);
+ if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) {
+ try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
+ channel.write(pageData);
+ }
+ } else {
+ byte[] encryptedBytes = null;
+ encryptedBytes = encryptor.encrypt(pageData.array(), pageData.position(), uncompressedSize);
+ // data is never a directByteBuffer now, so we can use data.array()
+ int encryptedSize = encryptedBytes.length;
+ pageBuffer.write(encryptedBytes, 0, encryptedSize);
}
+
} else {
- pageBuffer.write(compressedBytes, 0, compressedSize);
+ if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) {
+ pageBuffer.write(compressedBytes, 0, compressedSize);
+ } else {
+ byte[] encryptedBytes = null;
+ encryptedBytes = encryptor.encrypt(compressedBytes, 0, compressedSize);
+ // data is never a directByteBuffer now, so we can use data.array()
+ int encryptedSize = encryptedBytes.length;
+ pageBuffer.write(encryptedBytes, 0, encryptedSize);
+ }
}
logger.trace(
"finish flushing a time page data into buffer, buffer position {} ", pageBuffer.size());
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java
index 70547b43..1daed419 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java
@@ -20,8 +20,11 @@ package org.apache.tsfile.write.page;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.encrypt.IEncryptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.PublicBAOS;
@@ -45,6 +48,8 @@ public class ValuePageWriter {
private final ICompressor compressor;
+ private final IEncryptor encryptor;
+
// value
private Encoder valueEncoder;
private final PublicBAOS valueOut;
@@ -71,6 +76,19 @@ public class ValuePageWriter {
this.valueEncoder = valueEncoder;
this.statistics = Statistics.getStatsByType(dataType);
this.compressor = compressor;
+ this.encryptor = EncryptUtils.encryptor;
+ }
+
+ public ValuePageWriter(
+ Encoder valueEncoder, ICompressor compressor, TSDataType dataType, IEncryptor encryptor) {
+ this.valueOut = new PublicBAOS();
+ this.bitmap = 0;
+ this.size = 0;
+ this.bitmapOut = new PublicBAOS();
+ this.valueEncoder = valueEncoder;
+ this.statistics = Statistics.getStatsByType(dataType);
+ this.compressor = compressor;
+ this.encryptor = encryptor;
}
/** write a time value pair into encoder */
@@ -292,11 +310,28 @@ public class ValuePageWriter {
// write page content to temp PBAOS
logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size());
if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) {
- try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
- channel.write(pageData);
+ if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) {
+ try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) {
+ channel.write(pageData);
+ }
+ } else {
+ byte[] encryptedBytes = null;
+ encryptedBytes = encryptor.encrypt(pageData.array(), pageData.position(), uncompressedSize);
+ // data is never a directByteBuffer now, so we can use data.array()
+ int encryptedSize = encryptedBytes.length;
+ pageBuffer.write(encryptedBytes, 0, encryptedSize);
}
+
} else {
- pageBuffer.write(compressedBytes, 0, compressedSize);
+ if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) {
+ pageBuffer.write(compressedBytes, 0, compressedSize);
+ } else {
+ byte[] encryptedBytes = null;
+ encryptedBytes = encryptor.encrypt(compressedBytes, 0, compressedSize);
+ // data is never a directByteBuffer now, so we can use data.array()
+ int encryptedSize = encryptedBytes.length;
+ pageBuffer.write(encryptedBytes, 0, encryptedSize);
+ }
}
logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size());
return sizeWithoutStatistic;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index ffd30048..0de54052 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -21,6 +21,7 @@ package org.apache.tsfile.write.writer;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkGroupHeader;
@@ -124,6 +125,12 @@ public class TsFileIOWriter implements AutoCloseable {
private boolean generateTableSchema = false;
+ protected String encryptLevel = "0";
+
+ protected String encryptType = "UNENCRYPTED";
+
+ protected String encryptKey = null;
+
/** empty construct function. */
protected TsFileIOWriter() {}
@@ -134,11 +141,25 @@ public class TsFileIOWriter implements AutoCloseable {
* @throws IOException if I/O error occurs
*/
public TsFileIOWriter(File file) throws IOException {
+ this(file, TS_FILE_CONFIG);
+ }
+
+ /** for test only */
+ public TsFileIOWriter(File file, TSFileConfig conf) throws IOException {
this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), false);
this.file = file;
if (resourceLogger.isDebugEnabled()) {
resourceLogger.debug("{} writer is opened.", file.getName());
}
+ if (conf.getEncryptFlag()) {
+ this.encryptLevel = "2";
+ this.encryptType = conf.getEncryptType().getExtension();
+ this.encryptKey = EncryptUtils.normalKeyStr;
+ } else {
+ this.encryptLevel = "0";
+ this.encryptType = "UNENCRYPTED";
+ this.encryptKey = null;
+ }
startFile();
}
@@ -149,6 +170,15 @@ public class TsFileIOWriter implements AutoCloseable {
*/
public TsFileIOWriter(TsFileOutput output) throws IOException {
this.out = output;
+ if (TS_FILE_CONFIG.getEncryptFlag()) {
+ this.encryptLevel = "2";
+ this.encryptType = TS_FILE_CONFIG.getEncryptType().getExtension();
+ this.encryptKey = EncryptUtils.normalKeyStr;
+ } else {
+ this.encryptLevel = "0";
+ this.encryptType = "UNENCRYPTED";
+ this.encryptKey = null;
+ }
startFile();
}
@@ -164,6 +194,12 @@ public class TsFileIOWriter implements AutoCloseable {
chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX);
}
+ public void setEncryptParam(String encryptLevel, String encryptType, String encryptKey) {
+ this.encryptLevel = encryptLevel;
+ this.encryptType = encryptType;
+ this.encryptKey = encryptKey;
+ }
+
/**
* Writes given bytes to output stream. This method is called when total memory size exceeds the
* chunk group size threshold.
@@ -470,6 +506,9 @@ public class TsFileIOWriter implements AutoCloseable {
tsFileMetadata.setTableSchemaMap(schema.getTableSchemaMap());
tsFileMetadata.setMetaOffset(metaOffset);
tsFileMetadata.setBloomFilter(filter);
+ tsFileMetadata.addProperty("encryptLevel", encryptLevel);
+ tsFileMetadata.addProperty("encryptType", encryptType);
+ tsFileMetadata.addProperty("encryptKey", encryptKey);
int size = tsFileMetadata.serializeTo(out.wrapAsStream());
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128Test.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128Test.java
new file mode 100644
index 00000000..c0d91571
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128Test.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ThreadLocalRandom;
+
+public class AES128Test {
+ private final String key = "mkmkmkmkmkmkmkmk";
+
+ private String randomString(int length) {
+ StringBuilder builder = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ builder.append((char) (ThreadLocalRandom.current().nextInt(33, 128)));
+ }
+ return builder.toString();
+ }
+
+ @Before
+ public void setUp() {}
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void testBytes1() throws IOException {
+ String input = randomString(2000000);
+ byte[] unencrypted = input.getBytes(StandardCharsets.UTF_8);
+ long time = System.currentTimeMillis();
+ IEncryptor encryptor = new IEncryptor.AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(unencrypted);
+ System.out.println("encryption time cost:" + (System.currentTimeMillis() - time));
+ time = System.currentTimeMillis();
+ IDecryptor decryptor = new IDecryptor.AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted);
+ System.out.println("decryption time cost:" + (System.currentTimeMillis() - time));
+ Assert.assertArrayEquals(unencrypted, decrypted);
+ }
+
+ @Test
+ public void testBytes2() throws IOException {
+ String input = randomString(500000);
+ byte[] unencrypted = input.getBytes(StandardCharsets.UTF_8);
+ long time = System.currentTimeMillis();
+ IEncryptor encryptor = new IEncryptor.AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(unencrypted, 0, unencrypted.length);
+ System.out.println("encryption time cost:" + (System.currentTimeMillis() - time));
+ time = System.currentTimeMillis();
+ IDecryptor decryptor = new IDecryptor.AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted, 0, encrypted.length);
+ System.out.println("decryption time cost:" + (System.currentTimeMillis() - time));
+ Assert.assertArrayEquals(unencrypted, decrypted);
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileReadWriteTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileReadWriteTest.java
new file mode 100644
index 00000000..bc1d1220
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileReadWriteTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileReader;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Field;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.read.expression.QueryExpression;
+import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.utils.TsFileGeneratorForTest;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.datapoint.DataPoint;
+import org.apache.tsfile.write.record.datapoint.DoubleDataPoint;
+import org.apache.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.tsfile.write.record.datapoint.IntDataPoint;
+import org.apache.tsfile.write.record.datapoint.LongDataPoint;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class AES128TsFileReadWriteTest {
+ private final double delta = 0.0000001;
+ private final String path = TsFileGeneratorForTest.getTestTsFilePath("root.sg1", 0, 0, 1);
+ private File f;
+ private final IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create("device_1");
+
+ private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+
+ @Before
+ public void setUp() {
+ conf.setEncryptFlag("true");
+ conf.setEncryptType("AES128");
+ conf.setEncryptKey("thisisourtestkey");
+ f = new File(path);
+ if (f.exists()) {
+ assertTrue(f.delete());
+ }
+ if (!f.getParentFile().exists()) {
+ assertTrue(f.getParentFile().mkdirs());
+ }
+ }
+
+ @After
+ public void tearDown() {
+ conf.setEncryptKey("abcdefghijklmnop");
+ conf.setEncryptType("UNENCRYPTED");
+ conf.setEncryptFlag("false");
+ f = new File(path);
+ if (f.exists()) {
+ assertTrue(f.delete());
+ }
+ }
+
+ @Test
+ public void intTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.REGULAR,
+ TSEncoding.GORILLA,
+ TSEncoding.ZIGZAG);
+ for (TSEncoding encoding : encodings) {
+ intTest(encoding);
+ }
+ }
+
+ private void intTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(TSDataType.INT32, (i) -> new IntDataPoint("sensor_1", (int) i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getIntV()));
+ }
+
+ @Test
+ public void longTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.REGULAR,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ longTest(encoding);
+ }
+ }
+
+ public void longTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1", i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getLongV()));
+ }
+
+ @Test
+ public void floatTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.GORILLA_V1,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ floatTest(encoding);
+ }
+ }
+
+ public void floatTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(
+ TSDataType.FLOAT, (i) -> new FloatDataPoint("sensor_1", (float) i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getFloatV(), delta));
+ }
+
+ @Test
+ public void doubleTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.GORILLA_V1,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ doubleTest(encoding);
+ }
+ }
+
+ public void doubleTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(
+ TSDataType.DOUBLE, (i) -> new DoubleDataPoint("sensor_1", (double) i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getDoubleV(), delta));
+ }
+
+ // If no dataPoint in "device_1.sensor_2", it will throws a nomeasurement
+ // exception,
+ // cause no schema in tsfilemetadata anymore.
+ @Test
+ public void readEmptyMeasurementTest() throws IOException, WriteProcessException {
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f, new Schema(), conf)) {
+ // add measurements into file schema
+ tsFileWriter.registerTimeseries(
+ new Path(deviceID), new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
+ tsFileWriter.registerTimeseries(
+ new Path(deviceID),
+ new MeasurementSchema("sensor_2", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(1, deviceID);
+ DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
+ tsRecord.addTuple(dPoint1);
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
+ }
+
+ // read example : no filter
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ TsFileReader readTsFile = new TsFileReader(reader);
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path(deviceID, "sensor_2", true));
+ QueryExpression queryExpression = QueryExpression.create(paths, null);
+ try {
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ } catch (IOException e) {
+ // Assert.fail();
+ } finally {
+ reader.close();
+ }
+
+ assertTrue(f.delete());
+ }
+
+ @Test
+ public void readMeasurementWithRegularEncodingTest() throws IOException, WriteProcessException {
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("REGULAR");
+ writeDataByTSRecord(
+ TSDataType.INT64, (i) -> new LongDataPoint("sensor_1", i), TSEncoding.REGULAR);
+ readData((i, field, delta) -> assertEquals(i, field.getLongV()));
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF");
+ }
+
+ private void writeDataByTSRecord(
+ TSDataType dataType, AES128TsFileReadWriteTest.DataPointProxy proxy, TSEncoding encodingType)
+ throws IOException, WriteProcessException {
+ int floatCount = 1024 * 1024 * 13 + 1023;
+ // add measurements into file schema
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f, new Schema(), conf)) {
+ tsFileWriter.registerTimeseries(
+ new Path(deviceID), new MeasurementSchema("sensor_1", dataType, encodingType));
+ for (long i = 1; i < floatCount; i++) {
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(i, deviceID);
+ DataPoint dPoint1 = proxy.generateOne(i);
+ tsRecord.addTuple(dPoint1);
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
+ }
+ }
+ }
+
+ private void readData(AES128TsFileReadWriteTest.ReadDataPointProxy proxy) throws IOException {
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ TsFileReader readTsFile = new TsFileReader(reader);
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path(deviceID, "sensor_1", true));
+ QueryExpression queryExpression = QueryExpression.create(paths, null);
+
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ for (int j = 0; j < paths.size(); j++) {
+ assertEquals(paths.get(j), queryDataSet.getPaths().get(j));
+ }
+ int i = 1;
+ while (queryDataSet.hasNext()) {
+ RowRecord r = queryDataSet.next();
+ assertEquals(i, r.getTimestamp());
+ proxy.assertEqualProxy(i, r.getFields().get(0), delta);
+ i++;
+ }
+ reader.close();
+ }
+
+ private interface DataPointProxy {
+
+ DataPoint generateOne(long value);
+ }
+
+ private interface ReadDataPointProxy {
+
+ void assertEqualProxy(long i, Field field, double delta);
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileWriteTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileWriteTest.java
new file mode 100644
index 00000000..640f355b
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileWriteTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.common.constant.JsonFormatConstant;
+import org.apache.tsfile.constant.TestConstant;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.utils.RecordUtils;
+import org.apache.tsfile.utils.StringContainer;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.WriteTest;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class AES128TsFileWriteTest {
+ private static final Logger LOG = LoggerFactory.getLogger(WriteTest.class);
+ private final int ROW_COUNT = 2000000;
+ private TsFileWriter tsFileWriter;
+ private String inputDataFile;
+ private String outputDataFile;
+ private String errorOutputDataFile;
+ private Random rm = new Random();
+ private ArrayList<IMeasurementSchema> measurementArray;
+ private ArrayList<Path> pathArray;
+ private Schema schema;
+ private int stageSize = 4;
+ private int stageState = -1;
+ private int prePageSize;
+ private int prePageCheckThres;
+ private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+
+ private String[][] stageDeviceIds = {{"d1", "d2", "d3"}, {"d1"}, {"d2", "d3"}};
+ private String[] measurementIds = {"s0", "s1", "s2", "s3", "s4", "s5"};
+ private long longBase = System.currentTimeMillis() * 1000;
+ private String[] enums = {"MAN", "WOMAN"};
+
+ @Before
+ public void prepare() throws IOException {
+ inputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestInputData");
+ outputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestOutputData.tsfile");
+ errorOutputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestErrorOutputData.tsfile");
+ // for each row, flush page forcely
+ prePageSize = conf.getPageSizeInByte();
+ conf.setPageSizeInByte(0);
+ prePageCheckThres = conf.getPageCheckSizeThreshold();
+ conf.setPageCheckSizeThreshold(0);
+ conf.setEncryptFlag("true");
+ conf.setEncryptType("AES128");
+ conf.setEncryptKey("thisisourtestkey");
+
+ try {
+ generateSampleInputDataFile();
+ } catch (IOException e) {
+ LOG.error("meet error while creating test data " + e.getMessage());
+ fail();
+ }
+ File file = new File(outputDataFile);
+ File errorFile = new File(errorOutputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ if (errorFile.exists()) {
+ errorFile.delete();
+ }
+ measurementArray = new ArrayList<>();
+ measurementArray.add(new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.RLE));
+ measurementArray.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.TS_2DIFF));
+ HashMap<String, String> props = new HashMap<>();
+ props.put("max_point_number", "2");
+ measurementArray.add(
+ new MeasurementSchema(
+ "s2",
+ TSDataType.FLOAT,
+ TSEncoding.RLE,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
+ props));
+ props = new HashMap<>();
+ props.put("max_point_number", "3");
+ measurementArray.add(
+ new MeasurementSchema(
+ "s3",
+ TSDataType.DOUBLE,
+ TSEncoding.TS_2DIFF,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
+ props));
+ measurementArray.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.PLAIN));
+ pathArray = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ pathArray.add(new Path("d1", "s" + i, true));
+ }
+ schema = new Schema();
+ LOG.info(schema.toString());
+ tsFileWriter = new TsFileWriter(file, schema, conf);
+ }
+
+ @After
+ public void after() {
+ conf.setPageSizeInByte(prePageSize);
+ conf.setPageCheckSizeThreshold(prePageCheckThres);
+ conf.setEncryptKey("abcdefghijklmnop");
+ conf.setEncryptType("UNENCRYPTED");
+ conf.setEncryptFlag("false");
+ File file = new File(inputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ file = new File(outputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ file = new File(errorOutputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ }
+
+ private void generateSampleInputDataFile() throws IOException {
+ File file = new File(inputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ FileWriter fw = new FileWriter(file);
+
+ long startTime = System.currentTimeMillis();
+ startTime = startTime - startTime % 1000;
+
+ // first stage:int, long, float, double, boolean, enums
+ try {
+ for (int i = 0; i < ROW_COUNT; i++) {
+ // write d1
+ String d1 = "d1," + (startTime + i) + ",s1," + (i * 10 + 1) + ",s2," + (i * 10 + 2);
+ if (rm.nextInt(1000) < 100) {
+ d1 = "d1," + (startTime + i) + ",s1,,s2," + (i * 10 + 2) + ",s4,HIGH";
+ }
+ if (i % 5 == 0) {
+ d1 += ",s3," + (i * 10 + 3);
+ }
+ fw.write(d1 + "\r\n");
+
+ // write d2
+ String d2 = "d2," + (startTime + i) + ",s2," + (i * 10 + 2) + ",s3," + (i * 10 + 3);
+ if (rm.nextInt(1000) < 100) {
+ d2 = "d2," + (startTime + i) + ",s2,,s3," + (i * 10 + 3) + ",s5,MAN";
+ }
+ if (i % 5 == 0) {
+ d2 += ",s1," + (i * 10 + 1);
+ }
+ fw.write(d2 + "\r\n");
+ }
+ // write error
+ String d =
+ "d2,3,"
+ + (startTime + ROW_COUNT)
+ + ",s2,"
+ + (ROW_COUNT * 10 + 2)
+ + ",s3,"
+ + (ROW_COUNT * 10 + 3);
+ fw.write(d + "\r\n");
+ d = "d2," + (startTime + ROW_COUNT + 1) + ",2,s-1," + (ROW_COUNT * 10 + 2);
+ fw.write(d + "\r\n");
+ } finally {
+ fw.close();
+ }
+ }
+
+ @Test
+ public void writeTest() throws IOException {
+ try {
+ write();
+ } catch (WriteProcessException e) {
+ e.printStackTrace();
+ }
+ LOG.info("write processing has finished");
+ TsFileSequenceReader reader = new TsFileSequenceReader(outputDataFile);
+ TsFileMetadata metaData = reader.readFileMetadata();
+ }
+
+ public void write() throws IOException, WriteProcessException {
+ long lineCount = 0;
+ long startTime = System.currentTimeMillis();
+ String[] strings;
+ // add all measurement except the last one at before writing
+ for (int i = 0; i < measurementArray.size() - 1; i++) {
+ tsFileWriter.registerTimeseries(
+ new Path(pathArray.get(i).getIDeviceID()), measurementArray.get(i));
+ }
+ while (true) {
+ if (lineCount % stageSize == 0) {
+ LOG.info(
+ "write line:{},use time:{}s",
+ lineCount,
+ (System.currentTimeMillis() - startTime) / 1000);
+ stageState++;
+ LOG.info("stage:" + stageState);
+ if (stageState == stageDeviceIds.length) {
+ break;
+ }
+ }
+ if (lineCount == ROW_COUNT / 2) {
+ tsFileWriter.registerTimeseries(
+ new Path(pathArray.get(measurementArray.size() - 1).getIDeviceID()),
+ measurementArray.get(measurementArray.size() - 1));
+ }
+ strings = getNextRecord(lineCount, stageState);
+ for (String str : strings) {
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
+ if (record.dataPointList.isEmpty()) {
+ continue;
+ }
+ tsFileWriter.write(record);
+ }
+ lineCount++;
+ }
+ // test duplicate measurement adding
+ Path path = pathArray.get(measurementArray.size() - 1);
+ IMeasurementSchema dupTimeseries = measurementArray.get(measurementArray.size() - 1);
+ try {
+ tsFileWriter.registerTimeseries(new Path(path.getIDeviceID()), dupTimeseries);
+ } catch (WriteProcessException e) {
+ assertEquals("given timeseries has exists! " + path, e.getMessage());
+ }
+ try {
+ tsFileWriter.close();
+ } catch (IOException e) {
+ fail("close writer failed");
+ }
+ LOG.info("stage size: {}, write {} group data", stageSize, lineCount);
+ }
+
+ private String[] getNextRecord(long lineCount, int stage) {
+
+ String[] ret = new String[stageDeviceIds[stage].length];
+ for (int i = 0; i < ret.length; i++) {
+ StringContainer sc = new StringContainer(JsonFormatConstant.TSRECORD_SEPARATOR);
+ sc.addTail(stageDeviceIds[stage][i], lineCount);
+ sc.addTail(
+ measurementIds[0],
+ lineCount * 10 + i,
+ measurementIds[1],
+ longBase + lineCount * 20 + i,
+ measurementIds[2],
+ (lineCount * 30 + i) / 3.0,
+ measurementIds[3],
+ (longBase + lineCount * 40 + i) / 7.0);
+ sc.addTail(measurementIds[4], ((lineCount + i) & 1) == 0);
+ sc.addTail(measurementIds[5], enums[(int) (lineCount + i) % enums.length]);
+ ret[i] = sc.toString();
+ }
+ return ret;
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java
new file mode 100644
index 00000000..95448726
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.file.metadata.enums.EncryptionType;
+import org.apache.tsfile.utils.PublicBAOS;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class EncryptTest {
+ private final String inputString = "AES, a fast encryptor/decryptor.";
+ private final String key = "mkmkmkmkmkmkmkmk";
+
+ @Before
+ public void setUp() {}
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void NoEncryptorTest() throws IOException {
+ IEncryptor encryptor = new IEncryptor.NoEncryptor();
+ IDecryptor decryptor = new IDecryptor.NoDecryptor();
+ byte[] encrypted = encryptor.encrypt(inputString.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted);
+
+ String result = new String(decrypted, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
+
+ @Test
+ public void SM4128Test() throws IOException {
+ IEncryptor encryptor = new IEncryptor.SM4128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ IDecryptor decryptor = new IDecryptor.SM4128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(inputString.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted);
+
+ String result = new String(decrypted, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
+
+ @Test
+ public void SM4128Test1() throws IOException {
+ PublicBAOS out = new PublicBAOS();
+ out.write(inputString.getBytes(StandardCharsets.UTF_8));
+ IEncryptor encryptor = new IEncryptor.SM4128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ IDecryptor decryptor = new IDecryptor.SM4128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(out.getBuf(), 0, out.size());
+ byte[] decrypted = decryptor.decrypt(encrypted);
+
+ String result = new String(decrypted, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
+
+ @Test
+ public void AES128Test() throws IOException {
+ IEncryptor encryptor = new IEncryptor.AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ IDecryptor decryptor = new IDecryptor.AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(inputString.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted);
+
+ String result = new String(decrypted, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
+
+ @Test
+ public void AES128Test1() throws IOException {
+ PublicBAOS out = new PublicBAOS();
+ out.write(inputString.getBytes(StandardCharsets.UTF_8));
+ IEncryptor encryptor = new IEncryptor.AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ IDecryptor decryptor = new IDecryptor.AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(out.getBuf(), 0, out.size());
+ byte[] decrypted = decryptor.decrypt(encrypted);
+
+ String result = new String(decrypted, StandardCharsets.UTF_8);
+ assertEquals(inputString, result);
+ }
+
+ @Test
+ public void GetEncryptorTest() {
+ IEncryptor encryptor = IEncryptor.getEncryptor("AES128", key.getBytes(StandardCharsets.UTF_8));
+ assertEquals(encryptor.getEncryptionType(), EncryptionType.AES128);
+ IEncryptor encryptor2 =
+ IEncryptor.getEncryptor(EncryptionType.AES128, key.getBytes(StandardCharsets.UTF_8));
+ assertEquals(encryptor2.getEncryptionType(), EncryptionType.AES128);
+ }
+
+ @Test
+ public void GetEncryptorTest2() {
+ IEncryptor encryptor = IEncryptor.getEncryptor("SM4128", key.getBytes(StandardCharsets.UTF_8));
+ assertEquals(encryptor.getEncryptionType(), EncryptionType.SM4128);
+ IEncryptor encryptor2 =
+ IEncryptor.getEncryptor(EncryptionType.SM4128, key.getBytes(StandardCharsets.UTF_8));
+ assertEquals(encryptor2.getEncryptionType(), EncryptionType.SM4128);
+ }
+
+ @Test
+ public void HexStringTransverse() {
+ byte[] SboxTable = {
+ (byte) 0xd6,
+ (byte) 0x90,
+ (byte) 0xe9,
+ (byte) 0xfe,
+ (byte) 0xcc,
+ (byte) 0xe1,
+ 0x3d,
+ (byte) 0xb7,
+ 0x16,
+ (byte) 0xb6,
+ 0x14,
+ (byte) 0xc2,
+ 0x28,
+ (byte) 0xfb,
+ 0x2c,
+ 0x05,
+ 0x2b,
+ 0x67,
+ (byte) 0x9a,
+ 0x76,
+ 0x2a,
+ (byte) 0xbe,
+ 0x04,
+ (byte) 0xc3,
+ (byte) 0xaa,
+ 0x44,
+ 0x13,
+ 0x26,
+ 0x49,
+ (byte) 0x86,
+ 0x06,
+ (byte) 0x99,
+ (byte) 0x9c,
+ 0x42,
+ 0x50,
+ (byte) 0xf4,
+ (byte) 0x91,
+ (byte) 0xef,
+ (byte) 0x98,
+ 0x7a,
+ 0x33,
+ 0x54,
+ 0x0b,
+ 0x43,
+ (byte) 0xed,
+ (byte) 0xcf,
+ (byte) 0xac,
+ 0x62,
+ (byte) 0xe4,
+ (byte) 0xb3,
+ 0x1c,
+ (byte) 0xa9,
+ (byte) 0xc9,
+ 0x08,
+ (byte) 0xe8,
+ (byte) 0x95,
+ (byte) 0x80,
+ (byte) 0xdf,
+ (byte) 0x94,
+ (byte) 0xfa,
+ 0x75,
+ (byte) 0x8f,
+ 0x3f,
+ (byte) 0xa6,
+ 0x47,
+ 0x07,
+ (byte) 0xa7,
+ (byte) 0xfc,
+ (byte) 0xf3,
+ 0x73,
+ 0x17,
+ (byte) 0xba,
+ (byte) 0x83,
+ 0x59,
+ 0x3c,
+ 0x19,
+ (byte) 0xe6,
+ (byte) 0x85,
+ 0x4f,
+ (byte) 0xa8,
+ 0x68,
+ 0x6b,
+ (byte) 0x81,
+ (byte) 0xb2,
+ 0x71,
+ 0x64,
+ (byte) 0xda,
+ (byte) 0x8b,
+ (byte) 0xf8,
+ (byte) 0xeb,
+ 0x0f,
+ 0x4b,
+ 0x70,
+ 0x56,
+ (byte) 0x9d,
+ 0x35,
+ 0x1e,
+ 0x24,
+ 0x0e,
+ 0x5e,
+ 0x63,
+ 0x58,
+ (byte) 0xd1,
+ (byte) 0xa2,
+ 0x25,
+ 0x22,
+ 0x7c,
+ 0x3b,
+ 0x01,
+ 0x21,
+ 0x78,
+ (byte) 0x87,
+ (byte) 0xd4,
+ 0x00,
+ 0x46,
+ 0x57,
+ (byte) 0x9f,
+ (byte) 0xd3,
+ 0x27,
+ 0x52,
+ 0x4c,
+ 0x36,
+ 0x02,
+ (byte) 0xe7,
+ (byte) 0xa0,
+ (byte) 0xc4,
+ (byte) 0xc8,
+ (byte) 0x9e,
+ (byte) 0xea,
+ (byte) 0xbf,
+ (byte) 0x8a,
+ (byte) 0xd2,
+ 0x40,
+ (byte) 0xc7,
+ 0x38,
+ (byte) 0xb5,
+ (byte) 0xa3,
+ (byte) 0xf7,
+ (byte) 0xf2,
+ (byte) 0xce,
+ (byte) 0xf9,
+ 0x61,
+ 0x15,
+ (byte) 0xa1,
+ (byte) 0xe0,
+ (byte) 0xae,
+ 0x5d,
+ (byte) 0xa4,
+ (byte) 0x9b,
+ 0x34,
+ 0x1a,
+ 0x55,
+ (byte) 0xad,
+ (byte) 0x93,
+ 0x32,
+ 0x30,
+ (byte) 0xf5,
+ (byte) 0x8c,
+ (byte) 0xb1,
+ (byte) 0xe3,
+ 0x1d,
+ (byte) 0xf6,
+ (byte) 0xe2,
+ 0x2e,
+ (byte) 0x82,
+ 0x66,
+ (byte) 0xca,
+ 0x60,
+ (byte) 0xc0,
+ 0x29,
+ 0x23,
+ (byte) 0xab,
+ 0x0d,
+ 0x53,
+ 0x4e,
+ 0x6f,
+ (byte) 0xd5,
+ (byte) 0xdb,
+ 0x37,
+ 0x45,
+ (byte) 0xde,
+ (byte) 0xfd,
+ (byte) 0x8e,
+ 0x2f,
+ 0x03,
+ (byte) 0xff,
+ 0x6a,
+ 0x72,
+ 0x6d,
+ 0x6c,
+ 0x5b,
+ 0x51,
+ (byte) 0x8d,
+ 0x1b,
+ (byte) 0xaf,
+ (byte) 0x92,
+ (byte) 0xbb,
+ (byte) 0xdd,
+ (byte) 0xbc,
+ 0x7f,
+ 0x11,
+ (byte) 0xd9,
+ 0x5c,
+ 0x41,
+ 0x1f,
+ 0x10,
+ 0x5a,
+ (byte) 0xd8,
+ 0x0a,
+ (byte) 0xc1,
+ 0x31,
+ (byte) 0x88,
+ (byte) 0xa5,
+ (byte) 0xcd,
+ 0x7b,
+ (byte) 0xbd,
+ 0x2d,
+ 0x74,
+ (byte) 0xd0,
+ 0x12,
+ (byte) 0xb8,
+ (byte) 0xe5,
+ (byte) 0xb4,
+ (byte) 0xb0,
+ (byte) 0x89,
+ 0x69,
+ (byte) 0x97,
+ 0x4a,
+ 0x0c,
+ (byte) 0x96,
+ 0x77,
+ 0x7e,
+ 0x65,
+ (byte) 0xb9,
+ (byte) 0xf1,
+ 0x09,
+ (byte) 0xc5,
+ 0x6e,
+ (byte) 0xc6,
+ (byte) 0x84,
+ 0x18,
+ (byte) 0xf0,
+ 0x7d,
+ (byte) 0xec,
+ 0x3a,
+ (byte) 0xdc,
+ 0x4d,
+ 0x20,
+ 0x79,
+ (byte) 0xee,
+ 0x5f,
+ 0x3e,
+ (byte) 0xd7,
+ (byte) 0xcb,
+ 0x39,
+ 0x48
+ };
+ String ttt = EncryptUtils.byteArrayToHexString(SboxTable);
+ byte[] bytes = EncryptUtils.hexStringToByteArray(ttt);
+ assertArrayEquals(SboxTable, bytes);
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128Test.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128Test.java
new file mode 100644
index 00000000..af1ddbcb
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128Test.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ThreadLocalRandom;
+
+public class SM4128Test {
+ private final String key = "mkmkmkmkmkmkmkmk";
+
+ private String randomString(int length) {
+ StringBuilder builder = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ builder.append((char) (ThreadLocalRandom.current().nextInt(33, 128)));
+ }
+ return builder.toString();
+ }
+
+ @Before
+ public void setUp() {}
+
+ @After
+ public void tearDown() {}
+
+ @Test
+ public void testBytes1() throws IOException {
+ String input = randomString(2000000);
+ byte[] unencrypted = input.getBytes(StandardCharsets.UTF_8);
+ long time = System.currentTimeMillis();
+ IEncryptor encryptor = new IEncryptor.SM4128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(unencrypted);
+ System.out.println("encryption time cost:" + (System.currentTimeMillis() - time));
+ time = System.currentTimeMillis();
+ IDecryptor decryptor = new IDecryptor.SM4128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted);
+ System.out.println("decryption time cost:" + (System.currentTimeMillis() - time));
+ Assert.assertArrayEquals(unencrypted, decrypted);
+ }
+
+ @Test
+ public void testBytes2() throws IOException {
+ String input = randomString(500000);
+ byte[] unencrypted = input.getBytes(StandardCharsets.UTF_8);
+ long time = System.currentTimeMillis();
+ IEncryptor encryptor = new IEncryptor.SM4128Encryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] encrypted = encryptor.encrypt(unencrypted, 0, unencrypted.length);
+ System.out.println("encryption time cost:" + (System.currentTimeMillis() - time));
+ time = System.currentTimeMillis();
+ IDecryptor decryptor = new IDecryptor.SM4128Decryptor(key.getBytes(StandardCharsets.UTF_8));
+ byte[] decrypted = decryptor.decrypt(encrypted, 0, encrypted.length);
+ System.out.println("decryption time cost:" + (System.currentTimeMillis() - time));
+ Assert.assertArrayEquals(unencrypted, decrypted);
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128TsFileReadWriteTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128TsFileReadWriteTest.java
new file mode 100644
index 00000000..f2570707
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128TsFileReadWriteTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileReader;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Field;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.read.expression.QueryExpression;
+import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.utils.TsFileGeneratorForTest;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.datapoint.DataPoint;
+import org.apache.tsfile.write.record.datapoint.DoubleDataPoint;
+import org.apache.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.tsfile.write.record.datapoint.IntDataPoint;
+import org.apache.tsfile.write.record.datapoint.LongDataPoint;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class SM4128TsFileReadWriteTest {
+ private final double delta = 0.0000001;
+ private final String path = TsFileGeneratorForTest.getTestTsFilePath("root.sg1", 0, 0, 1);
+ private File f;
+ private final IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create("device_1");
+
+ private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+
+ @Before
+ public void setUp() {
+ conf.setEncryptFlag("true");
+ conf.setEncryptType("SM4128");
+ conf.setEncryptKey("thisisourtestkey");
+ f = new File(path);
+ if (f.exists()) {
+ assertTrue(f.delete());
+ }
+ if (!f.getParentFile().exists()) {
+ assertTrue(f.getParentFile().mkdirs());
+ }
+ }
+
+ @After
+ public void tearDown() {
+ conf.setEncryptKey("abcdefghijklmnop");
+ conf.setEncryptType("UNENCRYPTED");
+ conf.setEncryptFlag("false");
+ f = new File(path);
+ if (f.exists()) {
+ assertTrue(f.delete());
+ }
+ }
+
+ @Test
+ public void intTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.REGULAR,
+ TSEncoding.GORILLA,
+ TSEncoding.ZIGZAG);
+ for (TSEncoding encoding : encodings) {
+ intTest(encoding);
+ }
+ }
+
+ private void intTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(TSDataType.INT32, (i) -> new IntDataPoint("sensor_1", (int) i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getIntV()));
+ }
+
+ @Test
+ public void longTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.REGULAR,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ longTest(encoding);
+ }
+ }
+
+ public void longTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1", i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getLongV()));
+ }
+
+ @Test
+ public void floatTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.GORILLA_V1,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ floatTest(encoding);
+ }
+ }
+
+ public void floatTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(
+ TSDataType.FLOAT, (i) -> new FloatDataPoint("sensor_1", (float) i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getFloatV(), delta));
+ }
+
+ @Test
+ public void doubleTest() throws IOException, WriteProcessException {
+ List<TSEncoding> encodings =
+ Arrays.asList(
+ TSEncoding.PLAIN,
+ TSEncoding.RLE,
+ TSEncoding.TS_2DIFF,
+ TSEncoding.GORILLA_V1,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ doubleTest(encoding);
+ }
+ }
+
+ public void doubleTest(TSEncoding encoding) throws IOException, WriteProcessException {
+ writeDataByTSRecord(
+ TSDataType.DOUBLE, (i) -> new DoubleDataPoint("sensor_1", (double) i), encoding);
+ readData((i, field, delta) -> assertEquals(i, field.getDoubleV(), delta));
+ }
+
+ // If no dataPoint in "device_1.sensor_2", it will throws a nomeasurement
+ // exception,
+ // cause no schema in tsfilemetadata anymore.
+ @Test
+ public void readEmptyMeasurementTest() throws IOException, WriteProcessException {
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f, new Schema(), conf)) {
+ // add measurements into file schema
+ tsFileWriter.registerTimeseries(
+ new Path(deviceID), new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
+ tsFileWriter.registerTimeseries(
+ new Path(deviceID),
+ new MeasurementSchema("sensor_2", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(1, deviceID);
+ DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
+ tsRecord.addTuple(dPoint1);
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
+ }
+
+ // read example : no filter
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ TsFileReader readTsFile = new TsFileReader(reader);
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path(deviceID, "sensor_2", true));
+ QueryExpression queryExpression = QueryExpression.create(paths, null);
+ try {
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ } catch (IOException e) {
+ // Assert.fail();
+ } finally {
+ reader.close();
+ }
+
+ assertTrue(f.delete());
+ }
+
+ @Test
+ public void readMeasurementWithRegularEncodingTest() throws IOException, WriteProcessException {
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("REGULAR");
+ writeDataByTSRecord(
+ TSDataType.INT64, (i) -> new LongDataPoint("sensor_1", i), TSEncoding.REGULAR);
+ readData((i, field, delta) -> assertEquals(i, field.getLongV()));
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF");
+ }
+
+ private void writeDataByTSRecord(
+ TSDataType dataType, SM4128TsFileReadWriteTest.DataPointProxy proxy, TSEncoding encodingType)
+ throws IOException, WriteProcessException {
+ int floatCount = 1024 * 1024 * 13 + 1023;
+ // add measurements into file schema
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f, new Schema(), conf)) {
+ tsFileWriter.registerTimeseries(
+ new Path(deviceID), new MeasurementSchema("sensor_1", dataType, encodingType));
+ for (long i = 1; i < floatCount; i++) {
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(i, deviceID);
+ DataPoint dPoint1 = proxy.generateOne(i);
+ tsRecord.addTuple(dPoint1);
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
+ }
+ }
+ }
+
+ private void readData(SM4128TsFileReadWriteTest.ReadDataPointProxy proxy) throws IOException {
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ TsFileReader readTsFile = new TsFileReader(reader);
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path(deviceID, "sensor_1", true));
+ QueryExpression queryExpression = QueryExpression.create(paths, null);
+
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ for (int j = 0; j < paths.size(); j++) {
+ assertEquals(paths.get(j), queryDataSet.getPaths().get(j));
+ }
+ int i = 1;
+ while (queryDataSet.hasNext()) {
+ RowRecord r = queryDataSet.next();
+ assertEquals(i, r.getTimestamp());
+ proxy.assertEqualProxy(i, r.getFields().get(0), delta);
+ i++;
+ }
+ reader.close();
+ }
+
+ private interface DataPointProxy {
+
+ DataPoint generateOne(long value);
+ }
+
+ private interface ReadDataPointProxy {
+
+ void assertEqualProxy(long i, Field field, double delta);
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128TsFileWriteTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128TsFileWriteTest.java
new file mode 100644
index 00000000..795085bf
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/SM4128TsFileWriteTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.encrypt;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.common.constant.JsonFormatConstant;
+import org.apache.tsfile.constant.TestConstant;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.utils.RecordUtils;
+import org.apache.tsfile.utils.StringContainer;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.WriteTest;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class SM4128TsFileWriteTest {
+ private static final Logger LOG = LoggerFactory.getLogger(WriteTest.class);
+ private final int ROW_COUNT = 2000000;
+ private TsFileWriter tsFileWriter;
+ private String inputDataFile;
+ private String outputDataFile;
+ private String errorOutputDataFile;
+ private Random rm = new Random();
+ private ArrayList<IMeasurementSchema> measurementArray;
+ private ArrayList<Path> pathArray;
+ private Schema schema;
+ private int stageSize = 4;
+ private int stageState = -1;
+ private int prePageSize;
+ private int prePageCheckThres;
+ private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+
+ private String[][] stageDeviceIds = {{"d1", "d2", "d3"}, {"d1"}, {"d2", "d3"}};
+ private String[] measurementIds = {"s0", "s1", "s2", "s3", "s4", "s5"};
+ private long longBase = System.currentTimeMillis() * 1000;
+ private String[] enums = {"MAN", "WOMAN"};
+
+ @Before
+ public void prepare() throws IOException {
+ inputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestInputData");
+ outputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestOutputData.tsfile");
+ errorOutputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestErrorOutputData.tsfile");
+ // for each row, flush page forcely
+ prePageSize = conf.getPageSizeInByte();
+ conf.setPageSizeInByte(0);
+ prePageCheckThres = conf.getPageCheckSizeThreshold();
+ conf.setPageCheckSizeThreshold(0);
+ conf.setEncryptFlag("true");
+ conf.setEncryptType("SM4128");
+ conf.setEncryptKey("thisisourtestkey");
+
+ try {
+ generateSampleInputDataFile();
+ } catch (IOException e) {
+ fail();
+ }
+ File file = new File(outputDataFile);
+ File errorFile = new File(errorOutputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ if (errorFile.exists()) {
+ errorFile.delete();
+ }
+ measurementArray = new ArrayList<>();
+ measurementArray.add(new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.RLE));
+ measurementArray.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.TS_2DIFF));
+ HashMap<String, String> props = new HashMap<>();
+ props.put("max_point_number", "2");
+ measurementArray.add(
+ new MeasurementSchema(
+ "s2",
+ TSDataType.FLOAT,
+ TSEncoding.RLE,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
+ props));
+ props = new HashMap<>();
+ props.put("max_point_number", "3");
+ measurementArray.add(
+ new MeasurementSchema(
+ "s3",
+ TSDataType.DOUBLE,
+ TSEncoding.TS_2DIFF,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
+ props));
+ measurementArray.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.PLAIN));
+ pathArray = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ pathArray.add(new Path("d1", "s" + i, true));
+ }
+ schema = new Schema();
+ LOG.info(schema.toString());
+ tsFileWriter = new TsFileWriter(file, schema, conf);
+ }
+
+ @After
+ public void after() {
+ conf.setPageSizeInByte(prePageSize);
+ conf.setPageCheckSizeThreshold(prePageCheckThres);
+ conf.setEncryptKey("abcdefghijklmnop");
+ conf.setEncryptType("UNENCRYPTED");
+ conf.setEncryptFlag("false");
+ File file = new File(inputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ file = new File(outputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ file = new File(errorOutputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ }
+
+ private void generateSampleInputDataFile() throws IOException {
+ File file = new File(inputDataFile);
+ if (file.exists()) {
+ file.delete();
+ }
+ FileWriter fw = new FileWriter(file);
+
+ long startTime = System.currentTimeMillis();
+ startTime = startTime - startTime % 1000;
+
+ // first stage:int, long, float, double, boolean, enums
+ try {
+ for (int i = 0; i < ROW_COUNT; i++) {
+ // write d1
+ String d1 = "d1," + (startTime + i) + ",s1," + (i * 10 + 1) + ",s2," + (i * 10 + 2);
+ if (rm.nextInt(1000) < 100) {
+ d1 = "d1," + (startTime + i) + ",s1,,s2," + (i * 10 + 2) + ",s4,HIGH";
+ }
+ if (i % 5 == 0) {
+ d1 += ",s3," + (i * 10 + 3);
+ }
+ fw.write(d1 + "\r\n");
+
+ // write d2
+ String d2 = "d2," + (startTime + i) + ",s2," + (i * 10 + 2) + ",s3," + (i * 10 + 3);
+ if (rm.nextInt(1000) < 100) {
+ d2 = "d2," + (startTime + i) + ",s2,,s3," + (i * 10 + 3) + ",s5,MAN";
+ }
+ if (i % 5 == 0) {
+ d2 += ",s1," + (i * 10 + 1);
+ }
+ fw.write(d2 + "\r\n");
+ }
+ // write error
+ String d =
+ "d2,3,"
+ + (startTime + ROW_COUNT)
+ + ",s2,"
+ + (ROW_COUNT * 10 + 2)
+ + ",s3,"
+ + (ROW_COUNT * 10 + 3);
+ fw.write(d + "\r\n");
+ d = "d2," + (startTime + ROW_COUNT + 1) + ",2,s-1," + (ROW_COUNT * 10 + 2);
+ fw.write(d + "\r\n");
+ } finally {
+ fw.close();
+ }
+ }
+
+ @Test
+ public void writeTest() throws IOException {
+ try {
+ write();
+ } catch (WriteProcessException e) {
+ e.printStackTrace();
+ }
+ LOG.info("write processing has finished");
+ TsFileSequenceReader reader = new TsFileSequenceReader(outputDataFile);
+ TsFileMetadata metaData = reader.readFileMetadata();
+ }
+
+ public void write() throws IOException, WriteProcessException {
+ long lineCount = 0;
+ long startTime = System.currentTimeMillis();
+ String[] strings;
+ // add all measurement except the last one at before writing
+ for (int i = 0; i < measurementArray.size() - 1; i++) {
+ tsFileWriter.registerTimeseries(
+ new Path(pathArray.get(i).getIDeviceID()), measurementArray.get(i));
+ }
+ while (true) {
+ if (lineCount % stageSize == 0) {
+ LOG.info(
+ "write line:{},use time:{}s",
+ lineCount,
+ (System.currentTimeMillis() - startTime) / 1000);
+ stageState++;
+ LOG.info("stage:" + stageState);
+ if (stageState == stageDeviceIds.length) {
+ break;
+ }
+ }
+ if (lineCount == ROW_COUNT / 2) {
+ tsFileWriter.registerTimeseries(
+ new Path(pathArray.get(measurementArray.size() - 1).getIDeviceID()),
+ measurementArray.get(measurementArray.size() - 1));
+ }
+ strings = getNextRecord(lineCount, stageState);
+ for (String str : strings) {
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
+ if (record.dataPointList.isEmpty()) {
+ continue;
+ }
+ tsFileWriter.write(record);
+ }
+ lineCount++;
+ }
+ // test duplicate measurement adding
+ Path path = pathArray.get(measurementArray.size() - 1);
+ IMeasurementSchema dupTimeseries = measurementArray.get(measurementArray.size() - 1);
+ try {
+ tsFileWriter.registerTimeseries(new Path(path.getIDeviceID()), dupTimeseries);
+ } catch (WriteProcessException e) {
+ assertEquals("given timeseries has exists! " + path, e.getMessage());
+ }
+ try {
+ tsFileWriter.close();
+ } catch (IOException e) {
+ fail("close writer failed");
+ }
+ LOG.info("stage size: {}, write {} group data", stageSize, lineCount);
+ }
+
+ private String[] getNextRecord(long lineCount, int stage) {
+
+ String[] ret = new String[stageDeviceIds[stage].length];
+ for (int i = 0; i < ret.length; i++) {
+ StringContainer sc = new StringContainer(JsonFormatConstant.TSRECORD_SEPARATOR);
+ sc.addTail(stageDeviceIds[stage][i], lineCount);
+ sc.addTail(
+ measurementIds[0],
+ lineCount * 10 + i,
+ measurementIds[1],
+ longBase + lineCount * 20 + i,
+ measurementIds[2],
+ (lineCount * 30 + i) / 3.0,
+ measurementIds[3],
+ (longBase + lineCount * 40 + i) / 7.0);
+ sc.addTail(measurementIds[4], ((lineCount + i) & 1) == 0);
+ sc.addTail(measurementIds[5], enums[(int) (lineCount + i) % enums.length]);
+ ret[i] = sc.toString();
+ }
+ return ret;
+ }
+}