You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2025/05/13 03:16:50 UTC
(tsfile) 03/07: add config show.
This is an automated email from the ASF dual-hosted git repository.
colinlee pushed a commit to branch colin_fix_config
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 4b716d569b4712053f9a9ccd995e379366144b65
Author: colin <sh...@163.com>
AuthorDate: Wed Apr 23 17:27:51 2025 +0800
add config show.
---
.gitignore | 1 +
python/tests/test_write_and_read.py | 20 ++++++++++++++++++--
python/tsfile/tsfile_cpp.pxd | 4 ----
python/tsfile/tsfile_py_cpp.pyx | 20 ++++++++------------
python/tsfile/tsfile_table_writer.py | 4 ++--
python/tsfile/tsfile_writer.pyx | 2 +-
6 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/.gitignore b/.gitignore
index b9070546..4fa45012 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ python/data
python/venv/*
python/tests/__pycache__/*
python/tests/*.tsfile
+python/tsfile/include
cpp/cmake-build-debug-mingw/
cpp/third_party/googletest-release-1.12.1.zip
diff --git a/python/tests/test_write_and_read.py b/python/tests/test_write_and_read.py
index 0b666d15..ea84d898 100644
--- a/python/tests/test_write_and_read.py
+++ b/python/tests/test_write_and_read.py
@@ -26,6 +26,7 @@ from tsfile import Tablet, RowRecord, Field
from tsfile import TimeseriesSchema
from tsfile import TsFileTableWriter
from tsfile import TsFileWriter, TsFileReader, ColumnCategory
+from tsfile import Compressor
def test_row_record_write_and_read():
@@ -217,11 +218,12 @@ def test_tsfile_config():
from tsfile import get_tsfile_config, set_tsfile_config
config = get_tsfile_config()
+ assert config["chunk_group_size_threshold_"] == 0
table = TableSchema("tEst_Table",
[ColumnSchema("Device", TSDataType.STRING, ColumnCategory.TAG),
ColumnSchema("vAlue", TSDataType.DOUBLE, ColumnCategory.FIELD)])
-
+ os.remove("test1.tsfile")
with TsFileTableWriter("test1.tsfile", table) as writer:
tablet = Tablet(["device", "VALUE"], [TSDataType.STRING, TSDataType.DOUBLE])
for i in range(100):
@@ -232,8 +234,10 @@ def test_tsfile_config():
writer.write_table(tablet)
config_normal = get_tsfile_config()
+ assert config_normal["chunk_group_size_threshold_"] == 128 * 1024 * 1024
- with TsFileTableWriter("test2.tsfile", table, 100 * 100) as writer:
+ os.remove("test1.tsfile")
+ with TsFileTableWriter("test1.tsfile", table, 100 * 100) as writer:
tablet = Tablet(["device", "VALUE"], [TSDataType.STRING, TSDataType.DOUBLE])
for i in range(100):
tablet.add_timestamp(i, i)
@@ -241,6 +245,18 @@ def test_tsfile_config():
tablet.add_value_by_name("valuE", i, i * 1.1)
writer.write_table(tablet)
+ config_modified = get_tsfile_config()
+ assert config_normal != config_modified
+ assert config_modified["chunk_group_size_threshold_"] == 100 * 100
+ set_tsfile_config({'chunk_group_size_threshold_': 100 * 20})
+ assert get_tsfile_config()["chunk_group_size_threshold_"] == 100 * 20
+ with pytest.raises(TypeError):
+ set_tsfile_config({"time_compress_type_": TSDataType.DOUBLE})
+ with pytest.raises(TypeError):
+ set_tsfile_config({'chunk_group_size_threshold_': -1 *100 * 20})
+
+
+
diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd
index c726c220..e4782672 100644
--- a/python/tsfile/tsfile_cpp.pxd
+++ b/python/tsfile/tsfile_cpp.pxd
@@ -17,11 +17,7 @@
#
#cython: language_level=3
-<<<<<<< HEAD
from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t, uint8_t
-=======
-from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t
->>>>>>> 0db9e5c1 (add memory threshold in py writer.)
ctypedef int32_t ErrorCode
diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx
index a554bab9..55446ee7 100644
--- a/python/tsfile/tsfile_py_cpp.pyx
+++ b/python/tsfile/tsfile_py_cpp.pyx
@@ -384,22 +384,22 @@ cpdef void set_tsfile_config(dict new_config):
_check_double(new_config["tsfile_index_bloom_filter_error_percent_"])
g_config_value_.tsfile_index_bloom_filter_error_percent_ = new_config["tsfile_index_bloom_filter_error_percent_"]
if "time_encoding_type_" in new_config:
- if new_config["time_encoding_type_"] not in TSEncodingPy:
- raise ValueError(f"Unsupported TSEncoding: {new_config['time_encoding_type_']}")
+ if not isinstance(new_config["time_encoding_type_"], TSEncodingPy):
+ raise TypeError(f"Unsupported TSEncoding: {new_config['time_encoding_type_']}")
set_global_time_encoding(<uint8_t>(new_config["time_encoding_type_"].value))
if "time_data_type_" in new_config:
- if new_config["time_data_type_"] not in TSDataTypePy:
- raise ValueError(f"Unsupported TSDataType: {new_config['time_data_type_']}")
+ if not isinstance(new_config["time_data_type_"], TSDataTypePy):
+ raise TypeError(f"Unsupported TSDataType: {new_config['time_data_type_']}")
set_global_time_data_type(<uint8_t>(new_config["time_data_type_"].value))
if "time_compress_type_" in new_config:
- if new_config["time_compress_type_"] not in CompressorPy:
- raise ValueError(f"Unsupported Compressor: {new_config['time_compress_type_']}")
+ if not isinstance(new_config["time_compress_type_"], CompressorPy):
+ raise TypeError(f"Unsupported Compressor: {new_config['time_compress_type_']}")
set_global_time_compression(<uint8_t>(new_config["time_compress_type_"].value))
if "chunk_group_size_threshold_" in new_config:
- _check_int32(new_config["chunk_group_size_threshold_"])
+ _check_uint32(new_config["chunk_group_size_threshold_"])
g_config_value_.chunk_group_size_threshold_ = new_config["chunk_group_size_threshold_"]
if "record_count_for_next_mem_check_" in new_config:
- _check_int32(new_config["record_count_for_next_mem_check_"])
+ _check_uint32(new_config["record_count_for_next_mem_check_"])
g_config_value_.record_count_for_next_mem_check_ = new_config["record_count_for_next_mem_check_"]
if "encrypt_flag_" in new_config:
_check_bool(new_config["encrypt_flag_"])
@@ -409,10 +409,6 @@ cdef _check_uint32(value):
if not isinstance(value, int) or value < 0 or value > 0xFFFFFFFF:
raise TypeError(f"Expected uint32, got {type(value)}")
-cdef _check_int32(value):
- if not isinstance(value, int) or value < -0x80000000 or value > 0x7FFFFFFF:
- raise TypeError(f"Expected int32, got {type(value)}")
-
cdef _check_double(value):
if not isinstance(value, (int, float)):
raise TypeError(f"Expected float, got {type(value)}")
diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py
index c7ab9492..f5568dae 100644
--- a/python/tsfile/tsfile_table_writer.py
+++ b/python/tsfile/tsfile_table_writer.py
@@ -31,12 +31,12 @@ class TsFileTableWriter:
according to that schema, and serialize this data into a TsFile.
"""
- def __init__(self, path: str, table_schema: TableSchema):
+ def __init__(self, path: str, table_schema: TableSchema, memory_threshold = 128 * 1024 * 1024):
"""
:param path: The path of tsfile, will create if it doesn't exist.
:param table_schema: describes the schema of the tables they want to write.
"""
- self.writer = TsFileWriter(path)
+ self.writer = TsFileWriter(path, memory_threshold)
self.writer.register_table(table_schema)
self.exclusive_table_name_ = table_schema.get_table_name()
diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx
index 4d1f53ed..8db496cd 100644
--- a/python/tsfile/tsfile_writer.pyx
+++ b/python/tsfile/tsfile_writer.pyx
@@ -29,7 +29,7 @@ from tsfile.tablet import Tablet as TabletPy
cdef class TsFileWriterPy:
cdef TsFileWriter writer
- def __init__(self, pathname, memory_threshold = 128 * 1024 * 1024):
+ def __init__(self, pathname:str, memory_threshold:int):
self.writer = tsfile_writer_new_c(pathname, memory_threshold)
def register_timeseries(self, device_name : str, timeseries_schema : TimeseriesSchemaPy):