You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/07/09 10:07:19 UTC
(tsfile) 02/02: Update interface docs from develop
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch doc/update-interfaces-from-develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit c44b3897e8965ff233ff65daea14dc831275d7bf
Author: ColinLee <sh...@163.com>
AuthorDate: Thu Jul 9 18:06:58 2026 +0800
Update interface docs from develop
---
src/UserGuide/develop/DataFrame/TsFileDataFrame.md | 21 +-
.../InterfaceDefinition/InterfaceDefinition-C.md | 211 +++++++++++++++++++-
.../InterfaceDefinition/InterfaceDefinition-CPP.md | 147 +++++++++++++-
.../InterfaceDefinition-Python.md | 171 ++++++++++++++++-
src/UserGuide/latest/DataFrame/TsFileDataFrame.md | 21 +-
.../InterfaceDefinition/InterfaceDefinition-C.md | 211 +++++++++++++++++++-
.../InterfaceDefinition/InterfaceDefinition-CPP.md | 147 +++++++++++++-
.../InterfaceDefinition-Python.md | 171 ++++++++++++++++-
.../UserGuide/develop/DataFrame/TsFileDataFrame.md | 21 +-
.../InterfaceDefinition/InterfaceDefinition-C.md | 213 ++++++++++++++++++++-
.../InterfaceDefinition/InterfaceDefinition-CPP.md | 145 +++++++++++++-
.../InterfaceDefinition-Python.md | 165 +++++++++++++++-
.../UserGuide/latest/DataFrame/TsFileDataFrame.md | 21 +-
.../InterfaceDefinition/InterfaceDefinition-C.md | 213 ++++++++++++++++++++-
.../InterfaceDefinition/InterfaceDefinition-CPP.md | 145 +++++++++++++-
.../InterfaceDefinition-Python.md | 165 +++++++++++++++-
16 files changed, 2102 insertions(+), 86 deletions(-)
diff --git a/src/UserGuide/develop/DataFrame/TsFileDataFrame.md b/src/UserGuide/develop/DataFrame/TsFileDataFrame.md
index 91a32ee8e..dbd88160e 100644
--- a/src/UserGuide/develop/DataFrame/TsFileDataFrame.md
+++ b/src/UserGuide/develop/DataFrame/TsFileDataFrame.md
@@ -64,9 +64,11 @@ In the table below, `df` is a `TsFileDataFrame` instance, created with
| Example | Operation | Returns |
|---|---|---|
-| `TsFileDataFrame(paths)` | Load a file / list of files / directory | `TsFileDataFrame` |
+| `TsFileDataFrame(paths, show_progress=True)` | Load a file / list of files / directory. Set `show_progress=False` to silence loading progress on stderr | `TsFileDataFrame` |
| `len(df)` | Number of time series | `int` |
+| `df.model` | Loaded file model: `"table"` or `"tree"` | `str` |
| `df.list_timeseries("weather")` | Series names, optionally filtered by prefix | `List[str]` |
+| `df.list_timeseries_metadata("weather")` | Series metadata, optionally filtered by prefix | `pandas.DataFrame` |
| `df["weather.Beijing.humidity"]`, `df[0]`, `df[-1]` | One series | `Timeseries` |
| `df["city"]` | A metadata column (a tag / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` |
| `df[0:3]`, `df[[0, 2, 5]]` | Subset view by integer position: a contiguous range (`0:3`), or the listed positions (`[0, 2, 5]`); positions are the printed `index` column | `TsFileDataFrame` |
@@ -150,6 +152,7 @@ from tsfile import TsFileDataFrame
df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"])
df = TsFileDataFrame("data/") # recursively find every .tsfile under the directory
+df = TsFileDataFrame("data/", show_progress=False)
print(df)
```
@@ -203,7 +206,21 @@ optionally filtered by a prefix. Calling it with no argument returns all series.
```
To inspect metadata such as start/end time and count, print the DataFrame (or a
-subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe).
+subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe) — or call
+`list_timeseries_metadata()`.
+
+## Inspecting metadata
+
+`list_timeseries_metadata(path_prefix="")` returns a pandas DataFrame indexed by
+series name. It includes `field`, `start_time`, `end_time`, `count`, and the
+device tag columns. For table-model files it also includes the `table` column.
+
+```python
+meta = df.list_timeseries_metadata()
+weather_meta = df.list_timeseries_metadata("weather")
+
+meta[["field", "start_time", "end_time", "count"]].head()
+```
## Selecting series
diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
index 0fb0c1e36..262cb5148 100644
--- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
+++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
@@ -32,10 +32,12 @@ typedef enum {
TS_DATATYPE_FLOAT = 3,
TS_DATATYPE_DOUBLE = 4,
TS_DATATYPE_TEXT = 5,
+ TS_DATATYPE_VECTOR = 6,
TS_DATATYPE_TIMESTAMP = 8,
TS_DATATYPE_DATE = 9,
TS_DATATYPE_BLOB = 10,
TS_DATATYPE_STRING = 11,
+ TS_DATATYPE_NULL_TYPE = 254,
TS_DATATYPE_INVALID = 255
} TSDataType;
@@ -44,10 +46,18 @@ typedef enum {
TS_ENCODING_PLAIN = 0,
TS_ENCODING_DICTIONARY = 1,
TS_ENCODING_RLE = 2,
+ TS_ENCODING_DIFF = 3,
TS_ENCODING_TS_2DIFF = 4,
+ TS_ENCODING_BITMAP = 5,
+ TS_ENCODING_GORILLA_V1 = 6,
+ TS_ENCODING_REGULAR = 7,
TS_ENCODING_GORILLA = 8,
TS_ENCODING_ZIGZAG = 9,
+ TS_ENCODING_FREQ = 10,
+ TS_ENCODING_CHIMP = 11,
TS_ENCODING_SPRINTZ = 12,
+ TS_ENCODING_RLBE = 13,
+ TS_ENCODING_CAMEL = 14,
TS_ENCODING_INVALID = 255
} TSEncoding;
@@ -57,7 +67,12 @@ typedef enum {
TS_COMPRESSION_SNAPPY = 1,
TS_COMPRESSION_GZIP = 2,
TS_COMPRESSION_LZO = 3,
+ TS_COMPRESSION_SDT = 4,
+ TS_COMPRESSION_PAA = 5,
+ TS_COMPRESSION_PLA = 6,
TS_COMPRESSION_LZ4 = 7,
+ TS_COMPRESSION_ZSTD = 8,
+ TS_COMPRESSION_LZMA2 = 9,
TS_COMPRESSION_INVALID = 255
} CompressionType;
@@ -86,6 +101,19 @@ typedef struct table_schema {
int column_num;
} TableSchema;
+typedef struct timeseries_schema {
+ char* timeseries_name;
+ TSDataType data_type;
+ TSEncoding encoding;
+ CompressionType compression;
+} TimeseriesSchema;
+
+typedef struct device_schema {
+ char* device_name;
+ TimeseriesSchema* timeseries_schema;
+ int timeseries_num;
+} DeviceSchema;
+
// ResultSetMetaData: Contains metadata for a result set,
// such as column names and their data types.
typedef struct result_set_meta_data {
@@ -93,11 +121,73 @@ typedef struct result_set_meta_data {
TSDataType* data_types;
int column_num;
} ResultSetMetaData;
+
+typedef struct arrow_schema ArrowSchema;
+typedef struct arrow_array ArrowArray;
+
+typedef struct DeviceID {
+ char* path;
+ char* table_name;
+ uint32_t segment_count;
+ char** segments;
+} DeviceID;
+
+typedef struct TsFileStatisticBase {
+ bool has_statistic;
+ TSDataType type;
+ int32_t row_count;
+ int64_t start_time;
+ int64_t end_time;
+} TsFileStatisticBase;
+
+typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic;
+typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic;
+typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic;
+typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic;
+typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic;
+
+typedef union TimeseriesStatisticUnion {
+ TsFileBoolStatistic bool_s;
+ TsFileIntStatistic int_s;
+ TsFileFloatStatistic float_s;
+ TsFileStringStatistic string_s;
+ TsFileTextStatistic text_s;
+} TimeseriesStatisticUnion;
+
+typedef struct TimeseriesStatistic {
+ TimeseriesStatisticUnion u;
+} TimeseriesStatistic;
+
+#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u)
+
+typedef struct TimeseriesMetadata {
+ char* measurement_name;
+ TSDataType data_type;
+ int32_t chunk_meta_count;
+ TimeseriesStatistic statistic;
+ TimeseriesStatistic timeline_statistic;
+} TimeseriesMetadata;
+
+typedef struct DeviceTimeseriesMetadataEntry {
+ DeviceID device;
+ TimeseriesMetadata* timeseries;
+ uint32_t timeseries_count;
+} DeviceTimeseriesMetadataEntry;
+
+typedef struct DeviceTimeseriesMetadataMap {
+ DeviceTimeseriesMetadataEntry* entries;
+ uint32_t device_count;
+} DeviceTimeseriesMetadataMap;
```
> `ColumnSchema` does not carry encoding/compression: on write, columns follow the
> global defaults (see [Configuration](#configuration-encoding--compression)); on
> read, each column is decoded with the file's actual settings.
+>
+> `TimeseriesStatistic` is a tagged union in `tsfile_cwrapper.h`. Read common
+> fields through `tsfile_statistic_base(&metadata.statistic)` and then use the
+> active typed member (`int_s`, `float_s`, `bool_s`, `string_s`, or `text_s`)
+> according to the statistic data type.
## Write Interface
@@ -220,9 +310,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index,
* @param value [in] Null-terminated string. Ownership remains with caller.
* @return ERRNO.
*/
-ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index,
- const char* column_name,
- const char* value);
+ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ const char* column_name,
+ const char* value,
+ int value_len);
// Supports multiple data types
ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index,
@@ -250,9 +342,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index,
*
* @param value [in] Null-terminated string. Copied internally.
*/
-ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index,
- uint32_t column_index,
- const char* value);
+ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ uint32_t column_index,
+ const char* value,
+ int value_len);
// Supports multiple data types
@@ -329,11 +423,13 @@ Allowed encodings per data type, and the default used when you do not change it:
| Data type | Allowed encodings | Default |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` |
+| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` |
| `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` |
-Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`).
+Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`,
+`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`).
```C
// e.g. write every column with LZ4 compression
@@ -412,6 +508,31 @@ void free_tsfile_result_set(ResultSet* result_set);
```
+### Tree queries
+
+```C
+/**
+ * @brief Query tree-model data by measurement names within a time range.
+ */
+ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns,
+ uint32_t column_num, Timestamp start_time,
+ Timestamp end_time, ERRNO* err_code);
+
+/**
+ * @brief Query tree-model data by row with offset/limit.
+ *
+ * @param device_ids Array of device identifiers.
+ * @param measurement_names Array of measurement names.
+ * @param offset Leading rows to skip (>= 0).
+ * @param limit Max rows to return; < 0 means unlimited.
+ */
+ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader,
+ char** device_ids, int device_ids_len,
+ char** measurement_names,
+ int measurement_names_len, int offset,
+ int limit, ERRNO* err_code);
+```
+
### Filtering by tag
@@ -436,6 +557,8 @@ typedef enum {
TAG_FILTER_GTEQ = 5, // column >= value
TAG_FILTER_REGEXP = 6, // column matches the regex value
TAG_FILTER_NOT_REGEXP = 7, // column does not match the regex value
+ TAG_FILTER_IS_NULL = 8, // column is null
+ TAG_FILTER_IS_NOT_NULL = 9, // column is not null
} TagFilterOp;
/**
@@ -444,7 +567,7 @@ typedef enum {
* @param reader [in] Valid TsFileReader handle.
* @param table_name [in] Table whose schema defines the TAG columns.
* @param column_name [in] Name of the TAG column to filter on.
- * @param value [in] Comparison value (TAG columns are STRING).
+ * @param value [in] Comparison value (ignored for IS NULL / IS NOT NULL).
* @param op [in] Comparison operator (TagFilterOp).
* @param err_code [out] RET_OK(0) on success, or error code in errno_define_c.h.
* @return TagFilterHandle on success; NULL on failure.
@@ -465,6 +588,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader,
const char* lower, const char* upper,
bool is_not, ERRNO* err_code);
+// Convenience builders for common single-column predicates.
+TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+
// Combine predicates. AND/OR/NOT take ownership of their children; free the root only.
TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right);
TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right);
@@ -522,6 +659,18 @@ ResultSet tsfile_query_table_with_tag_filter(
TagFilterHandle tag_filter, int batch_size, ERRNO* err_code);
```
+### Read batch results as Arrow
+
+Batch query result sets (`batch_size > 0`) can be fetched as Arrow C Data
+Interface arrays and schemas. The caller owns the returned Arrow objects and
+must call their `release` callbacks when finished.
+
+```C
+ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set,
+ ArrowArray* out_array,
+ ArrowSchema* out_schema);
+```
+
Example — read `temperature` only for devices whose `region` TAG equals
`shanghai`:
@@ -664,17 +813,59 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader,
TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader,
uint32_t* size);
+/**
+ * @brief Gets all timeseries schema in the tsfile.
+ * @param size[out] number of DeviceSchema elements in the returned array.
+ * @return DeviceSchema*, an array of device schemas.
+ * @note The caller must call free_device_schema() on each element
+ * and free() the array pointer.
+ */
+DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader,
+ uint32_t* size);
+
/**
* @brief Free the tableschema's space.
* @param schema [in] the table schema to be freed.
*/
void free_table_schema(TableSchema schema);
+
+void free_device_schema(DeviceSchema schema);
```
+### Get Devices and Timeseries Metadata
+```C
+/**
+ * @brief Lists all devices in the file.
+ *
+ * @param out_devices[out] allocated array; free with tsfile_free_device_id_array().
+ * @param out_length[out] number of devices in the returned array.
+ */
+ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices,
+ uint32_t* out_length);
+void tsfile_free_device_id_array(DeviceID* devices, uint32_t length);
+void tsfile_device_id_free_contents(DeviceID* d);
+/**
+ * @brief Timeseries metadata for all devices in the file.
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap* out_map);
+/**
+ * @brief Timeseries metadata for the specified devices.
+ *
+ * length == 0 returns an empty map. For non-empty input, each DeviceID.path
+ * should contain the canonical device path.
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID* devices, uint32_t length,
+ DeviceTimeseriesMetadataMap* out_map);
+
+void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap* map);
+```
diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
index 496e48aca..d37e00777 100644
--- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
+++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
@@ -31,10 +31,14 @@ enum TSDataType : uint8_t {
FLOAT = 3,
DOUBLE = 4,
TEXT = 5,
+ VECTOR = 6,
+ UNKNOWN = 7,
TIMESTAMP = 8,
DATE = 9,
BLOB = 10,
STRING = 11,
+ NULL_TYPE = 254,
+ INVALID_DATATYPE = 255,
};
// Value encoding. See the table below for which encodings apply to which types.
@@ -42,19 +46,35 @@ enum TSEncoding : uint8_t {
PLAIN = 0,
DICTIONARY = 1,
RLE = 2,
+ DIFF = 3,
TS_2DIFF = 4,
+ BITMAP = 5,
+ GORILLA_V1 = 6,
+ REGULAR = 7,
GORILLA = 8,
ZIGZAG = 9,
+ FREQ = 10,
+ CHIMP = 11,
SPRINTZ = 12,
+ RLBE = 13,
+ CAMEL = 14,
+ INVALID_ENCODING = 255,
};
-// Compression type. SNAPPY/GZIP/LZO/LZ4 depend on build options; LZ4 is the default.
+// Compression type. SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 depend on build options;
+// LZ4 is the default.
enum CompressionType : uint8_t {
UNCOMPRESSED = 0,
SNAPPY = 1,
GZIP = 2,
LZO = 3,
+ SDT = 4,
+ PAA = 5,
+ PLA = 6,
LZ4 = 7,
+ ZSTD = 8,
+ LZMA2 = 9,
+ INVALID_COMPRESSION = 255,
};
// Column role within a table schema.
@@ -72,6 +92,8 @@ Encodings applicable to each data type:
| `GORILLA` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` |
| `ZIGZAG` | `INT32`, `INT64` |
| `SPRINTZ` | `INT32`, `INT64`, `FLOAT`, `DOUBLE` |
+| `CHIMP`, `RLBE` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` |
+| `CAMEL` | `DOUBLE` |
Default value encoding per type: `BOOLEAN → PLAIN`, `INT32 / INT64 → TS_2DIFF`,
`FLOAT / DOUBLE → GORILLA`, `TEXT / STRING / BLOB → PLAIN`. The default
@@ -134,6 +156,65 @@ class TsFileTableWriter {
};
```
+### TsFileWriter
+
+`TsFileWriter` is the lower-level writer interface from `tsfile_writer.h`. It
+supports both tree-model and table-model writes.
+
+```cpp
+extern int libtsfile_init();
+extern void libtsfile_destroy();
+
+// Writer configuration. Invalid values return common::E_INVALID_ARG and leave
+// the previous value unchanged.
+extern int set_page_max_point_count(uint32_t page_max_point_count);
+extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node);
+
+class TsFileWriter {
+ public:
+ TsFileWriter();
+ ~TsFileWriter();
+ void destroy();
+
+ int open(const std::string& file_path, int flags, mode_t mode);
+ int open(const std::string& file_path);
+ int init(storage::WriteFile* write_file);
+ int init(storage::RestorableTsFileIOWriter* rw);
+
+ void set_generate_table_schema(bool generate_table_schema);
+
+ int register_timeseries(const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_timeseries(
+ const std::string& device_path,
+ const std::vector<MeasurementSchema*>& measurement_schema_vec);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const std::vector<MeasurementSchema*>& measurement_schemas);
+ int register_table(const std::shared_ptr<TableSchema>& table_schema);
+
+ int write_record(const TsRecord& record);
+ int write_tablet(const Tablet& tablet);
+ int write_record_aligned(const TsRecord& record);
+ int write_tablet_aligned(const Tablet& tablet);
+ int write_tree(const Tablet& tablet);
+ int write_tree(const TsRecord& record);
+ int write_table(Tablet& tablet);
+
+ DeviceSchemasMap* get_schema_group_map();
+ std::shared_ptr<TableSchema> get_table_schema(
+ const std::string& table_name) const;
+ int64_t calculate_mem_size_for_all_group();
+ int64_t calculate_meta_mem_size() const;
+ int check_memory_size_and_may_flush_chunks();
+ int flush();
+ int close();
+};
+```
+
### TableSchema
Describe the data structure of the table schema
@@ -300,8 +381,14 @@ uint8_t common::get_global_compression();
// Time-column encoding/compression (the data type is fixed to INT64).
int common::set_global_time_encoding(uint8_t encoding);
int common::set_global_time_compression(uint8_t compression);
+uint8_t common::get_global_time_encoding();
+uint8_t common::get_global_time_compression();
```
+Global compression accepts `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `LZ4`,
+`ZSTD`, and `LZMA2`. The codec enum also contains legacy values such as `SDT`,
+`PAA`, and `PLA`, but the global compression setter rejects them.
+
## Read Interface
### Tsfile Reader
```cpp
@@ -339,6 +426,17 @@ class TsFileReader {
* @return Returns 0 on success, or a non-zero error code on failure.
*/
int query(storage::QueryExpression *qe, ResultSet *&ret_qds);
+ /**
+ * @brief query the tsfile by the path list, start time and end time.
+ * This method is used to query the tree model.
+ *
+ * @param [in] path_list the full path list
+ * @param [in] start_time the start time
+ * @param [in] end_time the end time
+ * @param [out] result_set the result set
+ */
+ int query(std::vector<std::string>& path_list, int64_t start_time,
+ int64_t end_time, ResultSet*& result_set);
/**
* @brief query the tsfile by the table name, columns names, start time
* and end time.
@@ -351,7 +449,7 @@ class TsFileReader {
*/
int query(const std::string &table_name,
const std::vector<std::string> &columns_names, int64_t start_time,
- int64_t end_time, ResultSet *&result_set);
+ int64_t end_time, ResultSet *&result_set, int batch_size = -1);
/**
* @brief query the tsfile by the table name, columns names, start time
@@ -366,7 +464,14 @@ class TsFileReader {
*/
int query(const std::string& table_name,
const std::vector<std::string>& columns_names, int64_t start_time,
- int64_t end_time, ResultSet*& result_set, Filter* tag_filter);
+ int64_t end_time, ResultSet*& result_set, Filter* tag_filter,
+ int batch_size = 0);
+
+ /**
+ * @brief query tree-model time series by row with offset and limit.
+ */
+ int queryByRow(std::vector<std::string>& path_list, int offset, int limit,
+ ResultSet*& result_set);
/**
* @brief query a table by row, with offset/limit pushdown and an optional
@@ -386,6 +491,13 @@ class TsFileReader {
int limit, ResultSet*& result_set,
Filter* tag_filter = nullptr, int batch_size = 0);
+ /**
+ * @brief query tree-model data by measurement names within a time range.
+ */
+ int query_table_on_tree(const std::vector<std::string>& measurement_names,
+ int64_t start_time, int64_t end_time,
+ ResultSet*& result_set);
+
/**
* @brief destroy the result set, this method should be called after the
* query is finished and result_set
@@ -393,6 +505,26 @@ class TsFileReader {
* @param qds the result set
*/
void destroy_query_data_set(ResultSet *qds);
+
+ ResultSet* read_timeseries(
+ const std::shared_ptr<IDeviceID>& device_id,
+ const std::vector<std::string>& measurement_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices(
+ std::string table_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids();
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices();
+
+ int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id,
+ std::vector<MeasurementSchema>& result);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata(
+ const std::vector<std::shared_ptr<IDeviceID>>& device_ids);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata();
+
/**
* @brief get the table schema by the table name
*
@@ -409,6 +541,11 @@ class TsFileReader {
std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas();
};
```
+
+`DeviceTimeseriesMetadataMap` is the metadata map returned by
+`get_timeseries_metadata()`: `std::map<std::shared_ptr<IDeviceID>,
+std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`.
+
### ResultSet
```cpp
/**
@@ -540,6 +677,8 @@ class TagFilterBuilder {
Filter* reg_exp(const std::string& columnName, const std::string& value);
Filter* not_reg_exp(const std::string& columnName,
const std::string& value);
+ Filter* is_null(const std::string& columnName);
+ Filter* is_not_null(const std::string& columnName);
Filter* between_and(const std::string& columnName, const std::string& lower,
const std::string& upper);
Filter* not_between_and(const std::string& columnName,
@@ -550,4 +689,4 @@ class TagFilterBuilder {
static Filter* or_filter(Filter* left, Filter* right);
static Filter* not_filter(Filter* filter);
};
-```
\ No newline at end of file
+```
diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
index bbcde49cf..806fa9115 100644
--- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
+++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
@@ -47,10 +47,17 @@ class TSEncoding(IntEnum):
PLAIN = 0 # all types
DICTIONARY = 1 # STRING, TEXT
RLE = 2 # INT32, INT64, TIMESTAMP, DATE
+ DIFF = 3
TS_2DIFF = 4 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
+ BITMAP = 5
+ GORILLA_V1 = 6
+ REGULAR = 7
GORILLA = 8 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
ZIGZAG = 9 # INT32, INT64
+ CHIMP = 11 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
SPRINTZ = 12 # INT32, INT64, FLOAT, DOUBLE
+ RLBE = 13 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
+ CAMEL = 14 # DOUBLE
class Compressor(IntEnum):
"""
@@ -60,7 +67,12 @@ class Compressor(IntEnum):
SNAPPY = 1
GZIP = 2
LZO = 3
+ SDT = 4
+ PAA = 5
+ PLA = 6
LZ4 = 7
+ ZSTD = 8
+ LZMA2 = 9
class ColumnCategory(IntEnum):
"""
@@ -103,13 +115,39 @@ class ResultSetMetaData:
def __init__(self, column_list: List[str], data_types: List[TSDataType])
+@dataclass(frozen=True)
+class DeviceID:
+ path: Optional[str]
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+
+@dataclass(frozen=True)
+class TimeseriesStatistic:
+ has_statistic: bool
+ row_count: int
+ start_time: int
+ end_time: int
+
+@dataclass(frozen=True)
+class TimeseriesMetadata:
+ measurement_name: str
+ data_type: TSDataType
+ chunk_meta_count: int
+ statistic: TimeseriesStatistic
+ timeline_statistic: TimeseriesStatistic
+
+@dataclass(frozen=True)
+class DeviceTimeseriesMetadataGroup:
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+ timeseries: list[TimeseriesMetadata]
```
## Write interface
-### TsFileWriter
+### TsFileTableWriter
```python
class TsFileTableWriter:
@@ -162,6 +200,48 @@ class TsFileTableWriter:
def __exit__(self, exc_type, exc_val, exc_tb)
```
+### TsFileWriter
+
+`TsFileWriter` is the lower-level writer exposed from `writer.pyx`. It can write
+tree-model data (`register_timeseries`, `register_device`, `write_tablet`,
+`write_row_record`) and table-model data (`register_table`, `write_table`,
+`write_dataframe`, `write_arrow_batch`).
+
+```python
+class TsFileWriter:
+ """
+ :param pathname: Destination TsFile path.
+ :param memory_threshold: bytes buffered before an automatic flush (default 128MB).
+ """
+ def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024)
+
+ def register_timeseries(self, device_name: str,
+ timeseries_schema: TimeseriesSchema)
+
+ def register_device(self, device_schema: DeviceSchema)
+
+ def register_table(self, table_schema: TableSchema)
+
+ def write_tablet(self, tablet: Tablet)
+
+ def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame,
+ tableschema: TableSchema)
+
+ def write_row_record(self, record: RowRecord)
+
+ def write_table(self, tablet: Tablet)
+
+ def write_arrow_batch(self, table_name: str, data,
+ time_col_index: int = -1)
+
+ def flush(self)
+
+ def close(self)
+
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
+```
+
### Tablet definition
@@ -231,11 +311,15 @@ encodings per data type, and the default used when you do not change it:
| Data type | Allowed encodings | Default |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` |
+| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` |
| `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` |
-Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`).
+Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`,
+`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`). The enum also exposes legacy values
+such as `SDT`, `PAA`, and `PLA`, but the global configuration setter rejects
+them.
## Read Interface
@@ -262,11 +346,39 @@ class TsFileReader:
:param column_names: A list of column names to retrieve.
:param start_time: The start time of the query range (default: minimum int64 value).
:param end_time: The end time of the query range (default: maximum int64 value).
+ :param tag_filter: Optional tag predicate for table-model TAG columns.
+ :param batch_size: <= 0 returns rows one by one; > 0 returns blocks of that size.
:return: A query result set handler.
"""
def query_table(self, table_name : str, column_names : List[str],
start_time : int = np.iinfo(np.int64).min,
- end_time: int = np.iinfo(np.int64).max) -> ResultSet
+ end_time: int = np.iinfo(np.int64).max,
+ tag_filter = None, batch_size : int = 0) -> ResultSet
+
+ """
+ Execute a time range query on tree-model measurement columns.
+
+ :param column_names: Measurement names to retrieve.
+ :param start_time: The start time of the query range.
+ :param end_time: The end time of the query range.
+ :return: A query result set handler.
+ """
+ def query_table_on_tree(self, column_names : List[str],
+ start_time : int = np.iinfo(np.int64).min,
+ end_time : int = np.iinfo(np.int64).max) -> ResultSet
+
+ """
+ Execute tree-model query by row with offset/limit.
+
+ :param device_ids: Device identifiers to query.
+ :param measurement_names: Measurement names to retrieve.
+ :param offset: Number of leading rows to skip.
+ :param limit: Maximum number of rows to return; < 0 means unlimited.
+ :return: A query result set handler.
+ """
+ def query_tree_by_row(self, device_ids : List[str],
+ measurement_names : List[str],
+ offset : int = 0, limit : int = -1) -> ResultSet
"""
Execute a table query by row, with offset/limit pushdown and an optional
@@ -287,6 +399,18 @@ class TsFileReader:
offset : int = 0, limit : int = -1,
tag_filter = None, batch_size : int = 0) -> ResultSet
+ """
+ Execute a tree-model time range query for one device.
+
+ :param device_name: Device identifier.
+ :param sensor_list: Measurement names to retrieve.
+ :param start_time: Query start time.
+ :param end_time: Query end time.
+ :return: A query result set handler.
+ """
+ def query_timeseries(self, device_name : str, sensor_list : List[str],
+ start_time : int = 0, end_time : int = 0) -> ResultSet
+
"""
Retrieves the schema of the specified table.
@@ -303,6 +427,31 @@ class TsFileReader:
"""
def get_all_table_schemas(self) ->dict[str, TableSchema]
+ """
+ Retrieves all tree-model timeseries schemas grouped by device.
+
+ :return: A list of DeviceSchema objects.
+ """
+ def get_all_timeseries_schemas(self) -> list[DeviceSchema]
+
+ """
+ Retrieves all device identifiers in the file.
+
+ :return: A list of DeviceID(path, table_name, segments).
+ """
+ def get_all_devices(self) -> List[DeviceID]
+
+ """
+ Retrieves per-timeseries metadata for all devices, or only the specified
+ devices.
+
+ :param device_ids: None for all devices, [] for an empty result, or a list
+ of DeviceID / path-compatible device identifiers.
+ :return: dict mapping device segment tuples to DeviceTimeseriesMetadataGroup.
+ """
+ def get_timeseries_metadata(
+ self, device_ids: Optional[List] = None
+ ) -> Dict[tuple, DeviceTimeseriesMetadataGroup]
"""
Closes the TsFile reader. If the reader has active result sets, they will be invalidated.
@@ -346,6 +495,13 @@ class ResultSet:
"""
def read_data_frame(self, max_row_num : int = 1024) -> DataFrame
+ """
+ Fetches the next batch result as a pyarrow.Table. Returns None when no more
+ TsBlock batches are available. This is only valid for result sets created
+ with batch_size > 0.
+ """
+ def read_arrow_batch(self) -> Optional[pyarrow.Table]
+
"""
Retrieves the value at the specified index from the query result set.
@@ -397,6 +553,9 @@ class ResultSet:
Closes the result set and releases any associated resources.
"""
def close(self)
+
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
```
### to_dataframe
@@ -467,4 +626,4 @@ def to_dataframe(file_path: str,
ColumnNotExistError
If any specified column does not exist in the table schema.
"""
-```
\ No newline at end of file
+```
diff --git a/src/UserGuide/latest/DataFrame/TsFileDataFrame.md b/src/UserGuide/latest/DataFrame/TsFileDataFrame.md
index 91a32ee8e..dbd88160e 100644
--- a/src/UserGuide/latest/DataFrame/TsFileDataFrame.md
+++ b/src/UserGuide/latest/DataFrame/TsFileDataFrame.md
@@ -64,9 +64,11 @@ In the table below, `df` is a `TsFileDataFrame` instance, created with
| Example | Operation | Returns |
|---|---|---|
-| `TsFileDataFrame(paths)` | Load a file / list of files / directory | `TsFileDataFrame` |
+| `TsFileDataFrame(paths, show_progress=True)` | Load a file / list of files / directory. Set `show_progress=False` to silence loading progress on stderr | `TsFileDataFrame` |
| `len(df)` | Number of time series | `int` |
+| `df.model` | Loaded file model: `"table"` or `"tree"` | `str` |
| `df.list_timeseries("weather")` | Series names, optionally filtered by prefix | `List[str]` |
+| `df.list_timeseries_metadata("weather")` | Series metadata, optionally filtered by prefix | `pandas.DataFrame` |
| `df["weather.Beijing.humidity"]`, `df[0]`, `df[-1]` | One series | `Timeseries` |
| `df["city"]` | A metadata column (a tag / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` |
| `df[0:3]`, `df[[0, 2, 5]]` | Subset view by integer position: a contiguous range (`0:3`), or the listed positions (`[0, 2, 5]`); positions are the printed `index` column | `TsFileDataFrame` |
@@ -150,6 +152,7 @@ from tsfile import TsFileDataFrame
df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"])
df = TsFileDataFrame("data/") # recursively find every .tsfile under the directory
+df = TsFileDataFrame("data/", show_progress=False)
print(df)
```
@@ -203,7 +206,21 @@ optionally filtered by a prefix. Calling it with no argument returns all series.
```
To inspect metadata such as start/end time and count, print the DataFrame (or a
-subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe).
+subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe) — or call
+`list_timeseries_metadata()`.
+
+## Inspecting metadata
+
+`list_timeseries_metadata(path_prefix="")` returns a pandas DataFrame indexed by
+series name. It includes `field`, `start_time`, `end_time`, `count`, and the
+device tag columns. For table-model files it also includes the `table` column.
+
+```python
+meta = df.list_timeseries_metadata()
+weather_meta = df.list_timeseries_metadata("weather")
+
+meta[["field", "start_time", "end_time", "count"]].head()
+```
## Selecting series
diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
index 0fb0c1e36..262cb5148 100644
--- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
+++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
@@ -32,10 +32,12 @@ typedef enum {
TS_DATATYPE_FLOAT = 3,
TS_DATATYPE_DOUBLE = 4,
TS_DATATYPE_TEXT = 5,
+ TS_DATATYPE_VECTOR = 6,
TS_DATATYPE_TIMESTAMP = 8,
TS_DATATYPE_DATE = 9,
TS_DATATYPE_BLOB = 10,
TS_DATATYPE_STRING = 11,
+ TS_DATATYPE_NULL_TYPE = 254,
TS_DATATYPE_INVALID = 255
} TSDataType;
@@ -44,10 +46,18 @@ typedef enum {
TS_ENCODING_PLAIN = 0,
TS_ENCODING_DICTIONARY = 1,
TS_ENCODING_RLE = 2,
+ TS_ENCODING_DIFF = 3,
TS_ENCODING_TS_2DIFF = 4,
+ TS_ENCODING_BITMAP = 5,
+ TS_ENCODING_GORILLA_V1 = 6,
+ TS_ENCODING_REGULAR = 7,
TS_ENCODING_GORILLA = 8,
TS_ENCODING_ZIGZAG = 9,
+ TS_ENCODING_FREQ = 10,
+ TS_ENCODING_CHIMP = 11,
TS_ENCODING_SPRINTZ = 12,
+ TS_ENCODING_RLBE = 13,
+ TS_ENCODING_CAMEL = 14,
TS_ENCODING_INVALID = 255
} TSEncoding;
@@ -57,7 +67,12 @@ typedef enum {
TS_COMPRESSION_SNAPPY = 1,
TS_COMPRESSION_GZIP = 2,
TS_COMPRESSION_LZO = 3,
+ TS_COMPRESSION_SDT = 4,
+ TS_COMPRESSION_PAA = 5,
+ TS_COMPRESSION_PLA = 6,
TS_COMPRESSION_LZ4 = 7,
+ TS_COMPRESSION_ZSTD = 8,
+ TS_COMPRESSION_LZMA2 = 9,
TS_COMPRESSION_INVALID = 255
} CompressionType;
@@ -86,6 +101,19 @@ typedef struct table_schema {
int column_num;
} TableSchema;
+typedef struct timeseries_schema {
+ char* timeseries_name;
+ TSDataType data_type;
+ TSEncoding encoding;
+ CompressionType compression;
+} TimeseriesSchema;
+
+typedef struct device_schema {
+ char* device_name;
+ TimeseriesSchema* timeseries_schema;
+ int timeseries_num;
+} DeviceSchema;
+
// ResultSetMetaData: Contains metadata for a result set,
// such as column names and their data types.
typedef struct result_set_meta_data {
@@ -93,11 +121,73 @@ typedef struct result_set_meta_data {
TSDataType* data_types;
int column_num;
} ResultSetMetaData;
+
+typedef struct arrow_schema ArrowSchema;
+typedef struct arrow_array ArrowArray;
+
+typedef struct DeviceID {
+ char* path;
+ char* table_name;
+ uint32_t segment_count;
+ char** segments;
+} DeviceID;
+
+typedef struct TsFileStatisticBase {
+ bool has_statistic;
+ TSDataType type;
+ int32_t row_count;
+ int64_t start_time;
+ int64_t end_time;
+} TsFileStatisticBase;
+
+typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic;
+typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic;
+typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic;
+typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic;
+typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic;
+
+typedef union TimeseriesStatisticUnion {
+ TsFileBoolStatistic bool_s;
+ TsFileIntStatistic int_s;
+ TsFileFloatStatistic float_s;
+ TsFileStringStatistic string_s;
+ TsFileTextStatistic text_s;
+} TimeseriesStatisticUnion;
+
+typedef struct TimeseriesStatistic {
+ TimeseriesStatisticUnion u;
+} TimeseriesStatistic;
+
+#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u)
+
+typedef struct TimeseriesMetadata {
+ char* measurement_name;
+ TSDataType data_type;
+ int32_t chunk_meta_count;
+ TimeseriesStatistic statistic;
+ TimeseriesStatistic timeline_statistic;
+} TimeseriesMetadata;
+
+typedef struct DeviceTimeseriesMetadataEntry {
+ DeviceID device;
+ TimeseriesMetadata* timeseries;
+ uint32_t timeseries_count;
+} DeviceTimeseriesMetadataEntry;
+
+typedef struct DeviceTimeseriesMetadataMap {
+ DeviceTimeseriesMetadataEntry* entries;
+ uint32_t device_count;
+} DeviceTimeseriesMetadataMap;
```
> `ColumnSchema` does not carry encoding/compression: on write, columns follow the
> global defaults (see [Configuration](#configuration-encoding--compression)); on
> read, each column is decoded with the file's actual settings.
+>
+> `TimeseriesStatistic` is a tagged union in `tsfile_cwrapper.h`. Read common
+> fields through `tsfile_statistic_base(&metadata.statistic)` and then use the
+> active typed member (`int_s`, `float_s`, `bool_s`, `string_s`, or `text_s`)
+> according to the statistic data type.
## Write Interface
@@ -220,9 +310,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index,
* @param value [in] Null-terminated string. Ownership remains with caller.
* @return ERRNO.
*/
-ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index,
- const char* column_name,
- const char* value);
+ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ const char* column_name,
+ const char* value,
+ int value_len);
// Supports multiple data types
ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index,
@@ -250,9 +342,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index,
*
* @param value [in] Null-terminated string. Copied internally.
*/
-ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index,
- uint32_t column_index,
- const char* value);
+ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ uint32_t column_index,
+ const char* value,
+ int value_len);
// Supports multiple data types
@@ -329,11 +423,13 @@ Allowed encodings per data type, and the default used when you do not change it:
| Data type | Allowed encodings | Default |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` |
+| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` |
| `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` |
-Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`).
+Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`,
+`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`).
```C
// e.g. write every column with LZ4 compression
@@ -412,6 +508,31 @@ void free_tsfile_result_set(ResultSet* result_set);
```
+### Tree queries
+
+```C
+/**
+ * @brief Query tree-model data by measurement names within a time range.
+ */
+ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns,
+ uint32_t column_num, Timestamp start_time,
+ Timestamp end_time, ERRNO* err_code);
+
+/**
+ * @brief Query tree-model data by row with offset/limit.
+ *
+ * @param device_ids Array of device identifiers.
+ * @param measurement_names Array of measurement names.
+ * @param offset Leading rows to skip (>= 0).
+ * @param limit Max rows to return; < 0 means unlimited.
+ */
+ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader,
+ char** device_ids, int device_ids_len,
+ char** measurement_names,
+ int measurement_names_len, int offset,
+ int limit, ERRNO* err_code);
+```
+
### Filtering by tag
@@ -436,6 +557,8 @@ typedef enum {
TAG_FILTER_GTEQ = 5, // column >= value
TAG_FILTER_REGEXP = 6, // column matches the regex value
TAG_FILTER_NOT_REGEXP = 7, // column does not match the regex value
+ TAG_FILTER_IS_NULL = 8, // column is null
+ TAG_FILTER_IS_NOT_NULL = 9, // column is not null
} TagFilterOp;
/**
@@ -444,7 +567,7 @@ typedef enum {
* @param reader [in] Valid TsFileReader handle.
* @param table_name [in] Table whose schema defines the TAG columns.
* @param column_name [in] Name of the TAG column to filter on.
- * @param value [in] Comparison value (TAG columns are STRING).
+ * @param value [in] Comparison value (ignored for IS NULL / IS NOT NULL).
* @param op [in] Comparison operator (TagFilterOp).
* @param err_code [out] RET_OK(0) on success, or error code in errno_define_c.h.
* @return TagFilterHandle on success; NULL on failure.
@@ -465,6 +588,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader,
const char* lower, const char* upper,
bool is_not, ERRNO* err_code);
+// Convenience builders for common single-column predicates.
+TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+
// Combine predicates. AND/OR/NOT take ownership of their children; free the root only.
TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right);
TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right);
@@ -522,6 +659,18 @@ ResultSet tsfile_query_table_with_tag_filter(
TagFilterHandle tag_filter, int batch_size, ERRNO* err_code);
```
+### Read batch results as Arrow
+
+Batch query result sets (`batch_size > 0`) can be fetched as Arrow C Data
+Interface arrays and schemas. The caller owns the returned Arrow objects and
+must call their `release` callbacks when finished.
+
+```C
+ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set,
+ ArrowArray* out_array,
+ ArrowSchema* out_schema);
+```
+
Example — read `temperature` only for devices whose `region` TAG equals
`shanghai`:
@@ -664,17 +813,59 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader,
TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader,
uint32_t* size);
+/**
+ * @brief Gets all timeseries schema in the tsfile.
+ * @param size[out] number of DeviceSchema elements in the returned array.
+ * @return DeviceSchema*, an array of device schemas.
+ * @note The caller must call free_device_schema() on each element
+ * and free() the array pointer.
+ */
+DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader,
+ uint32_t* size);
+
/**
* @brief Free the tableschema's space.
* @param schema [in] the table schema to be freed.
*/
void free_table_schema(TableSchema schema);
+
+void free_device_schema(DeviceSchema schema);
```
+### Get Devices and Timeseries Metadata
+```C
+/**
+ * @brief Lists all devices in the file.
+ *
+ * @param out_devices[out] allocated array; free with tsfile_free_device_id_array().
+ * @param out_length[out] number of devices in the returned array.
+ */
+ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices,
+ uint32_t* out_length);
+void tsfile_free_device_id_array(DeviceID* devices, uint32_t length);
+void tsfile_device_id_free_contents(DeviceID* d);
+/**
+ * @brief Timeseries metadata for all devices in the file.
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap* out_map);
+/**
+ * @brief Timeseries metadata for the specified devices.
+ *
+ * length == 0 returns an empty map. For non-empty input, each DeviceID.path
+ * should contain the canonical device path.
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID* devices, uint32_t length,
+ DeviceTimeseriesMetadataMap* out_map);
+
+void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap* map);
+```
diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
index 496e48aca..d37e00777 100644
--- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
+++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
@@ -31,10 +31,14 @@ enum TSDataType : uint8_t {
FLOAT = 3,
DOUBLE = 4,
TEXT = 5,
+ VECTOR = 6,
+ UNKNOWN = 7,
TIMESTAMP = 8,
DATE = 9,
BLOB = 10,
STRING = 11,
+ NULL_TYPE = 254,
+ INVALID_DATATYPE = 255,
};
// Value encoding. See the table below for which encodings apply to which types.
@@ -42,19 +46,35 @@ enum TSEncoding : uint8_t {
PLAIN = 0,
DICTIONARY = 1,
RLE = 2,
+ DIFF = 3,
TS_2DIFF = 4,
+ BITMAP = 5,
+ GORILLA_V1 = 6,
+ REGULAR = 7,
GORILLA = 8,
ZIGZAG = 9,
+ FREQ = 10,
+ CHIMP = 11,
SPRINTZ = 12,
+ RLBE = 13,
+ CAMEL = 14,
+ INVALID_ENCODING = 255,
};
-// Compression type. SNAPPY/GZIP/LZO/LZ4 depend on build options; LZ4 is the default.
+// Compression type. SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 depend on build options;
+// LZ4 is the default.
enum CompressionType : uint8_t {
UNCOMPRESSED = 0,
SNAPPY = 1,
GZIP = 2,
LZO = 3,
+ SDT = 4,
+ PAA = 5,
+ PLA = 6,
LZ4 = 7,
+ ZSTD = 8,
+ LZMA2 = 9,
+ INVALID_COMPRESSION = 255,
};
// Column role within a table schema.
@@ -72,6 +92,8 @@ Encodings applicable to each data type:
| `GORILLA` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` |
| `ZIGZAG` | `INT32`, `INT64` |
| `SPRINTZ` | `INT32`, `INT64`, `FLOAT`, `DOUBLE` |
+| `CHIMP`, `RLBE` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` |
+| `CAMEL` | `DOUBLE` |
Default value encoding per type: `BOOLEAN → PLAIN`, `INT32 / INT64 → TS_2DIFF`,
`FLOAT / DOUBLE → GORILLA`, `TEXT / STRING / BLOB → PLAIN`. The default
@@ -134,6 +156,65 @@ class TsFileTableWriter {
};
```
+### TsFileWriter
+
+`TsFileWriter` is the lower-level writer interface from `tsfile_writer.h`. It
+supports both tree-model and table-model writes.
+
+```cpp
+extern int libtsfile_init();
+extern void libtsfile_destroy();
+
+// Writer configuration. Invalid values return common::E_INVALID_ARG and leave
+// the previous value unchanged.
+extern int set_page_max_point_count(uint32_t page_max_point_count);
+extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node);
+
+class TsFileWriter {
+ public:
+ TsFileWriter();
+ ~TsFileWriter();
+ void destroy();
+
+ int open(const std::string& file_path, int flags, mode_t mode);
+ int open(const std::string& file_path);
+ int init(storage::WriteFile* write_file);
+ int init(storage::RestorableTsFileIOWriter* rw);
+
+ void set_generate_table_schema(bool generate_table_schema);
+
+ int register_timeseries(const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_timeseries(
+ const std::string& device_path,
+ const std::vector<MeasurementSchema*>& measurement_schema_vec);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const std::vector<MeasurementSchema*>& measurement_schemas);
+ int register_table(const std::shared_ptr<TableSchema>& table_schema);
+
+ int write_record(const TsRecord& record);
+ int write_tablet(const Tablet& tablet);
+ int write_record_aligned(const TsRecord& record);
+ int write_tablet_aligned(const Tablet& tablet);
+ int write_tree(const Tablet& tablet);
+ int write_tree(const TsRecord& record);
+ int write_table(Tablet& tablet);
+
+ DeviceSchemasMap* get_schema_group_map();
+ std::shared_ptr<TableSchema> get_table_schema(
+ const std::string& table_name) const;
+ int64_t calculate_mem_size_for_all_group();
+ int64_t calculate_meta_mem_size() const;
+ int check_memory_size_and_may_flush_chunks();
+ int flush();
+ int close();
+};
+```
+
### TableSchema
Describe the data structure of the table schema
@@ -300,8 +381,14 @@ uint8_t common::get_global_compression();
// Time-column encoding/compression (the data type is fixed to INT64).
int common::set_global_time_encoding(uint8_t encoding);
int common::set_global_time_compression(uint8_t compression);
+uint8_t common::get_global_time_encoding();
+uint8_t common::get_global_time_compression();
```
+Global compression accepts `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `LZ4`,
+`ZSTD`, and `LZMA2`. The codec enum also contains legacy values such as `SDT`,
+`PAA`, and `PLA`, but the global compression setter rejects them.
+
## Read Interface
### Tsfile Reader
```cpp
@@ -339,6 +426,17 @@ class TsFileReader {
* @return Returns 0 on success, or a non-zero error code on failure.
*/
int query(storage::QueryExpression *qe, ResultSet *&ret_qds);
+ /**
+ * @brief query the tsfile by the path list, start time and end time.
+ * This method is used to query the tree model.
+ *
+ * @param [in] path_list the full path list
+ * @param [in] start_time the start time
+ * @param [in] end_time the end time
+ * @param [out] result_set the result set
+ */
+ int query(std::vector<std::string>& path_list, int64_t start_time,
+ int64_t end_time, ResultSet*& result_set);
/**
* @brief query the tsfile by the table name, columns names, start time
* and end time.
@@ -351,7 +449,7 @@ class TsFileReader {
*/
int query(const std::string &table_name,
const std::vector<std::string> &columns_names, int64_t start_time,
- int64_t end_time, ResultSet *&result_set);
+ int64_t end_time, ResultSet *&result_set, int batch_size = -1);
/**
* @brief query the tsfile by the table name, columns names, start time
@@ -366,7 +464,14 @@ class TsFileReader {
*/
int query(const std::string& table_name,
const std::vector<std::string>& columns_names, int64_t start_time,
- int64_t end_time, ResultSet*& result_set, Filter* tag_filter);
+ int64_t end_time, ResultSet*& result_set, Filter* tag_filter,
+ int batch_size = 0);
+
+ /**
+ * @brief query tree-model time series by row with offset and limit.
+ */
+ int queryByRow(std::vector<std::string>& path_list, int offset, int limit,
+ ResultSet*& result_set);
/**
* @brief query a table by row, with offset/limit pushdown and an optional
@@ -386,6 +491,13 @@ class TsFileReader {
int limit, ResultSet*& result_set,
Filter* tag_filter = nullptr, int batch_size = 0);
+ /**
+ * @brief query tree-model data by measurement names within a time range.
+ */
+ int query_table_on_tree(const std::vector<std::string>& measurement_names,
+ int64_t start_time, int64_t end_time,
+ ResultSet*& result_set);
+
/**
* @brief destroy the result set, this method should be called after the
* query is finished and result_set
@@ -393,6 +505,26 @@ class TsFileReader {
* @param qds the result set
*/
void destroy_query_data_set(ResultSet *qds);
+
+ ResultSet* read_timeseries(
+ const std::shared_ptr<IDeviceID>& device_id,
+ const std::vector<std::string>& measurement_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices(
+ std::string table_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids();
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices();
+
+ int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id,
+ std::vector<MeasurementSchema>& result);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata(
+ const std::vector<std::shared_ptr<IDeviceID>>& device_ids);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata();
+
/**
* @brief get the table schema by the table name
*
@@ -409,6 +541,11 @@ class TsFileReader {
std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas();
};
```
+
+`DeviceTimeseriesMetadataMap` is the metadata map returned by
+`get_timeseries_metadata()`: `std::map<std::shared_ptr<IDeviceID>,
+std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`.
+
### ResultSet
```cpp
/**
@@ -540,6 +677,8 @@ class TagFilterBuilder {
Filter* reg_exp(const std::string& columnName, const std::string& value);
Filter* not_reg_exp(const std::string& columnName,
const std::string& value);
+ Filter* is_null(const std::string& columnName);
+ Filter* is_not_null(const std::string& columnName);
Filter* between_and(const std::string& columnName, const std::string& lower,
const std::string& upper);
Filter* not_between_and(const std::string& columnName,
@@ -550,4 +689,4 @@ class TagFilterBuilder {
static Filter* or_filter(Filter* left, Filter* right);
static Filter* not_filter(Filter* filter);
};
-```
\ No newline at end of file
+```
diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
index bbcde49cf..806fa9115 100644
--- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
+++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
@@ -47,10 +47,17 @@ class TSEncoding(IntEnum):
PLAIN = 0 # all types
DICTIONARY = 1 # STRING, TEXT
RLE = 2 # INT32, INT64, TIMESTAMP, DATE
+ DIFF = 3
TS_2DIFF = 4 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
+ BITMAP = 5
+ GORILLA_V1 = 6
+ REGULAR = 7
GORILLA = 8 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
ZIGZAG = 9 # INT32, INT64
+ CHIMP = 11 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
SPRINTZ = 12 # INT32, INT64, FLOAT, DOUBLE
+ RLBE = 13 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE
+ CAMEL = 14 # DOUBLE
class Compressor(IntEnum):
"""
@@ -60,7 +67,12 @@ class Compressor(IntEnum):
SNAPPY = 1
GZIP = 2
LZO = 3
+ SDT = 4
+ PAA = 5
+ PLA = 6
LZ4 = 7
+ ZSTD = 8
+ LZMA2 = 9
class ColumnCategory(IntEnum):
"""
@@ -103,13 +115,39 @@ class ResultSetMetaData:
def __init__(self, column_list: List[str], data_types: List[TSDataType])
+@dataclass(frozen=True)
+class DeviceID:
+ path: Optional[str]
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+
+@dataclass(frozen=True)
+class TimeseriesStatistic:
+ has_statistic: bool
+ row_count: int
+ start_time: int
+ end_time: int
+
+@dataclass(frozen=True)
+class TimeseriesMetadata:
+ measurement_name: str
+ data_type: TSDataType
+ chunk_meta_count: int
+ statistic: TimeseriesStatistic
+ timeline_statistic: TimeseriesStatistic
+
+@dataclass(frozen=True)
+class DeviceTimeseriesMetadataGroup:
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+ timeseries: list[TimeseriesMetadata]
```
## Write interface
-### TsFileWriter
+### TsFileTableWriter
```python
class TsFileTableWriter:
@@ -162,6 +200,48 @@ class TsFileTableWriter:
def __exit__(self, exc_type, exc_val, exc_tb)
```
+### TsFileWriter
+
+`TsFileWriter` is the lower-level writer exposed from `writer.pyx`. It can write
+tree-model data (`register_timeseries`, `register_device`, `write_tablet`,
+`write_row_record`) and table-model data (`register_table`, `write_table`,
+`write_dataframe`, `write_arrow_batch`).
+
+```python
+class TsFileWriter:
+ """
+ :param pathname: Destination TsFile path.
+ :param memory_threshold: bytes buffered before an automatic flush (default 128MB).
+ """
+ def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024)
+
+ def register_timeseries(self, device_name: str,
+ timeseries_schema: TimeseriesSchema)
+
+ def register_device(self, device_schema: DeviceSchema)
+
+ def register_table(self, table_schema: TableSchema)
+
+ def write_tablet(self, tablet: Tablet)
+
+ def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame,
+ tableschema: TableSchema)
+
+ def write_row_record(self, record: RowRecord)
+
+ def write_table(self, tablet: Tablet)
+
+ def write_arrow_batch(self, table_name: str, data,
+ time_col_index: int = -1)
+
+ def flush(self)
+
+ def close(self)
+
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
+```
+
### Tablet definition
@@ -231,11 +311,15 @@ encodings per data type, and the default used when you do not change it:
| Data type | Allowed encodings | Default |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` |
+| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` |
| `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` |
-Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`).
+Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`,
+`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`). The enum also exposes legacy values
+such as `SDT`, `PAA`, and `PLA`, but the global configuration setter rejects
+them.
## Read Interface
@@ -262,11 +346,39 @@ class TsFileReader:
:param column_names: A list of column names to retrieve.
:param start_time: The start time of the query range (default: minimum int64 value).
:param end_time: The end time of the query range (default: maximum int64 value).
+ :param tag_filter: Optional tag predicate for table-model TAG columns.
+ :param batch_size: <= 0 returns rows one by one; > 0 returns blocks of that size.
:return: A query result set handler.
"""
def query_table(self, table_name : str, column_names : List[str],
start_time : int = np.iinfo(np.int64).min,
- end_time: int = np.iinfo(np.int64).max) -> ResultSet
+ end_time: int = np.iinfo(np.int64).max,
+ tag_filter = None, batch_size : int = 0) -> ResultSet
+
+ """
+ Execute a time range query on tree-model measurement columns.
+
+ :param column_names: Measurement names to retrieve.
+ :param start_time: The start time of the query range.
+ :param end_time: The end time of the query range.
+ :return: A query result set handler.
+ """
+ def query_table_on_tree(self, column_names : List[str],
+ start_time : int = np.iinfo(np.int64).min,
+ end_time : int = np.iinfo(np.int64).max) -> ResultSet
+
+ """
+ Execute tree-model query by row with offset/limit.
+
+ :param device_ids: Device identifiers to query.
+ :param measurement_names: Measurement names to retrieve.
+ :param offset: Number of leading rows to skip.
+ :param limit: Maximum number of rows to return; < 0 means unlimited.
+ :return: A query result set handler.
+ """
+ def query_tree_by_row(self, device_ids : List[str],
+ measurement_names : List[str],
+ offset : int = 0, limit : int = -1) -> ResultSet
"""
Execute a table query by row, with offset/limit pushdown and an optional
@@ -287,6 +399,18 @@ class TsFileReader:
offset : int = 0, limit : int = -1,
tag_filter = None, batch_size : int = 0) -> ResultSet
+ """
+ Execute a tree-model time range query for one device.
+
+ :param device_name: Device identifier.
+ :param sensor_list: Measurement names to retrieve.
+ :param start_time: Query start time.
+ :param end_time: Query end time.
+ :return: A query result set handler.
+ """
+ def query_timeseries(self, device_name : str, sensor_list : List[str],
+ start_time : int = 0, end_time : int = 0) -> ResultSet
+
"""
Retrieves the schema of the specified table.
@@ -303,6 +427,31 @@ class TsFileReader:
"""
def get_all_table_schemas(self) ->dict[str, TableSchema]
+ """
+ Retrieves all tree-model timeseries schemas grouped by device.
+
+ :return: A list of DeviceSchema objects.
+ """
+ def get_all_timeseries_schemas(self) -> list[DeviceSchema]
+
+ """
+ Retrieves all device identifiers in the file.
+
+ :return: A list of DeviceID(path, table_name, segments).
+ """
+ def get_all_devices(self) -> List[DeviceID]
+
+ """
+ Retrieves per-timeseries metadata for all devices, or only the specified
+ devices.
+
+ :param device_ids: None for all devices, [] for an empty result, or a list
+ of DeviceID / path-compatible device identifiers.
+ :return: dict mapping device segment tuples to DeviceTimeseriesMetadataGroup.
+ """
+ def get_timeseries_metadata(
+ self, device_ids: Optional[List] = None
+ ) -> Dict[tuple, DeviceTimeseriesMetadataGroup]
"""
Closes the TsFile reader. If the reader has active result sets, they will be invalidated.
@@ -346,6 +495,13 @@ class ResultSet:
"""
def read_data_frame(self, max_row_num : int = 1024) -> DataFrame
+ """
+ Fetches the next batch result as a pyarrow.Table. Returns None when no more
+ TsBlock batches are available. This is only valid for result sets created
+ with batch_size > 0.
+ """
+ def read_arrow_batch(self) -> Optional[pyarrow.Table]
+
"""
Retrieves the value at the specified index from the query result set.
@@ -397,6 +553,9 @@ class ResultSet:
Closes the result set and releases any associated resources.
"""
def close(self)
+
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
```
### to_dataframe
@@ -467,4 +626,4 @@ def to_dataframe(file_path: str,
ColumnNotExistError
If any specified column does not exist in the table schema.
"""
-```
\ No newline at end of file
+```
diff --git a/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md b/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md
index 816a8cca6..2a7bf9616 100644
--- a/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md
+++ b/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md
@@ -59,9 +59,11 @@ data.values # -> np.ndarray, shape (N, 2):N
| 示例 | 操作 | 返回类型 |
|---|---|---|
-| `TsFileDataFrame(paths)` | 加载文件 / 文件列表 / 目录 | `TsFileDataFrame` |
+| `TsFileDataFrame(paths, show_progress=True)` | 加载文件 / 文件列表 / 目录。设置 `show_progress=False` 可关闭 stderr 上的加载进度 | `TsFileDataFrame` |
| `len(df)` | 时间序列总数 | `int` |
+| `df.model` | 已加载文件模型:`"table"` 或 `"tree"` | `str` |
| `df.list_timeseries("weather")` | 获取序列名,可按前缀筛选 | `List[str]` |
+| `df.list_timeseries_metadata("weather")` | 获取序列元数据,可按前缀筛选 | `pandas.DataFrame` |
| `df["weather.Beijing.humidity"]`、`df[0]`、`df[-1]` | 获取单条序列 | `Timeseries` |
| `df["city"]` | 获取某元数据列(标签 / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` |
| `df[0:3]`、`df[[0, 2, 5]]` | 按整数位置取子集视图:连续区间(`0:3`)或所列位置(`[0, 2, 5]`);位置即打印的 `index` 列 | `TsFileDataFrame` |
@@ -132,6 +134,7 @@ from tsfile import TsFileDataFrame
df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"])
df = TsFileDataFrame("data/") # 递归查找目录下所有 .tsfile
+df = TsFileDataFrame("data/", show_progress=False)
print(df)
```
@@ -172,7 +175,21 @@ TsFileDataFrame(table model, 972 time series, 5 files)
['weather.Beijing.humidity', 'weather.Beijing.temperature']
```
-若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)。
+若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)——也可以调用
+`list_timeseries_metadata()`。
+
+## 查看元数据
+
+`list_timeseries_metadata(path_prefix="")` 返回以序列名为索引的 pandas DataFrame。
+其中包含 `field`、`start_time`、`end_time`、`count` 以及设备标签列。对于
+table 模型文件,还会包含 `table` 列。
+
+```python
+meta = df.list_timeseries_metadata()
+weather_meta = df.list_timeseries_metadata("weather")
+
+meta[["field", "start_time", "end_time", "count"]].head()
+```
## 选取序列
diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
index de6d39796..1edc34e54 100644
--- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
+++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
@@ -32,10 +32,12 @@ typedef enum {
TS_DATATYPE_FLOAT = 3,
TS_DATATYPE_DOUBLE = 4,
TS_DATATYPE_TEXT = 5,
+ TS_DATATYPE_VECTOR = 6,
TS_DATATYPE_TIMESTAMP = 8,
TS_DATATYPE_DATE = 9,
TS_DATATYPE_BLOB = 10,
TS_DATATYPE_STRING = 11,
+ TS_DATATYPE_NULL_TYPE = 254,
TS_DATATYPE_INVALID = 255
} TSDataType;
@@ -44,10 +46,18 @@ typedef enum {
TS_ENCODING_PLAIN = 0,
TS_ENCODING_DICTIONARY = 1,
TS_ENCODING_RLE = 2,
+ TS_ENCODING_DIFF = 3,
TS_ENCODING_TS_2DIFF = 4,
+ TS_ENCODING_BITMAP = 5,
+ TS_ENCODING_GORILLA_V1 = 6,
+ TS_ENCODING_REGULAR = 7,
TS_ENCODING_GORILLA = 8,
TS_ENCODING_ZIGZAG = 9,
+ TS_ENCODING_FREQ = 10,
+ TS_ENCODING_CHIMP = 11,
TS_ENCODING_SPRINTZ = 12,
+ TS_ENCODING_RLBE = 13,
+ TS_ENCODING_CAMEL = 14,
TS_ENCODING_INVALID = 255
} TSEncoding;
@@ -57,7 +67,12 @@ typedef enum {
TS_COMPRESSION_SNAPPY = 1,
TS_COMPRESSION_GZIP = 2,
TS_COMPRESSION_LZO = 3,
+ TS_COMPRESSION_SDT = 4,
+ TS_COMPRESSION_PAA = 5,
+ TS_COMPRESSION_PLA = 6,
TS_COMPRESSION_LZ4 = 7,
+ TS_COMPRESSION_ZSTD = 8,
+ TS_COMPRESSION_LZMA2 = 9,
TS_COMPRESSION_INVALID = 255
} CompressionType;
@@ -83,15 +98,89 @@ typedef struct table_schema {
int column_num;
} TableSchema;
+typedef struct timeseries_schema {
+ char* timeseries_name;
+ TSDataType data_type;
+ TSEncoding encoding;
+ CompressionType compression;
+} TimeseriesSchema;
+
+typedef struct device_schema {
+ char* device_name;
+ TimeseriesSchema* timeseries_schema;
+ int timeseries_num;
+} DeviceSchema;
+
// ResultSetMetaData:结果集的元数据,包括列名和数据类型。
typedef struct result_set_meta_data {
char** column_names;
TSDataType* data_types;
int column_num;
} ResultSetMetaData;
+
+typedef struct arrow_schema ArrowSchema;
+typedef struct arrow_array ArrowArray;
+
+typedef struct DeviceID {
+ char* path;
+ char* table_name;
+ uint32_t segment_count;
+ char** segments;
+} DeviceID;
+
+typedef struct TsFileStatisticBase {
+ bool has_statistic;
+ TSDataType type;
+ int32_t row_count;
+ int64_t start_time;
+ int64_t end_time;
+} TsFileStatisticBase;
+
+typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic;
+typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic;
+typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic;
+typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic;
+typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic;
+
+typedef union TimeseriesStatisticUnion {
+ TsFileBoolStatistic bool_s;
+ TsFileIntStatistic int_s;
+ TsFileFloatStatistic float_s;
+ TsFileStringStatistic string_s;
+ TsFileTextStatistic text_s;
+} TimeseriesStatisticUnion;
+
+typedef struct TimeseriesStatistic {
+ TimeseriesStatisticUnion u;
+} TimeseriesStatistic;
+
+#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u)
+
+typedef struct TimeseriesMetadata {
+ char* measurement_name;
+ TSDataType data_type;
+ int32_t chunk_meta_count;
+ TimeseriesStatistic statistic;
+ TimeseriesStatistic timeline_statistic;
+} TimeseriesMetadata;
+
+typedef struct DeviceTimeseriesMetadataEntry {
+ DeviceID device;
+ TimeseriesMetadata* timeseries;
+ uint32_t timeseries_count;
+} DeviceTimeseriesMetadataEntry;
+
+typedef struct DeviceTimeseriesMetadataMap {
+ DeviceTimeseriesMetadataEntry* entries;
+ uint32_t device_count;
+} DeviceTimeseriesMetadataMap;
```
> `ColumnSchema` 不携带编码/压缩:写入时列遵循全局默认值(见[配置](#配置编码与压缩)),读取时按文件中的实际配置解码。
+>
+> `TimeseriesStatistic` 是 `tsfile_cwrapper.h` 中的带标签 union。可通过
+> `tsfile_statistic_base(&metadata.statistic)` 读取通用字段,再根据数据类型读取
+> `int_s`、`float_s`、`bool_s`、`string_s` 或 `text_s`。
## 写入接口
@@ -207,9 +296,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index,
*
* @param value [输入] 以 '\0' 结尾的字符串,调用者保留所有权。
*/
-ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index,
- const char* column_name,
- const char* value);
+ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ const char* column_name,
+ const char* value,
+ int value_len);
// 支持多种数据类型插入
ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index,
@@ -236,9 +327,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index,
*
* @param value [输入] 字符串会被内部复制。
*/
-ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index,
- uint32_t column_index,
- const char* value);
+ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ uint32_t column_index,
+ const char* value,
+ int value_len);
ERRNO tablet_add_value_by_index_int32_t(Tablet tablet, uint32_t row_index,
@@ -310,11 +403,12 @@ uint8_t get_global_time_compression();
| 数据类型 | 允许的编码 | 默认值 |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` |
+| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` |
| `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` |
-压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。
+压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`(默认 `LZ4`)。
```C
// 例如:所有列均以 LZ4 压缩写入
@@ -387,6 +481,31 @@ bool tsfile_result_set_next(ResultSet result_set, ERRNO* error_code);
void free_tsfile_result_set(ResultSet* result_set);
```
+### Tree 查询
+
+```C
+/**
+ * @brief 按测点名在时间范围内查询 tree 模型数据。
+ */
+ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns,
+ uint32_t column_num, Timestamp start_time,
+ Timestamp end_time, ERRNO* err_code);
+
+/**
+ * @brief 按行查询 tree 模型数据,支持 offset/limit。
+ *
+ * @param device_ids 设备标识数组。
+ * @param measurement_names 测点名数组。
+ * @param offset 需要跳过的起始行数(>= 0)。
+ * @param limit 最多返回行数;< 0 表示不限制。
+ */
+ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader,
+ char** device_ids, int device_ids_len,
+ char** measurement_names,
+ int measurement_names_len, int offset,
+ int limit, ERRNO* err_code);
+```
+
### 按标签过滤
@@ -409,6 +528,8 @@ typedef enum {
TAG_FILTER_GTEQ = 5, // 列 >= 值
TAG_FILTER_REGEXP = 6, // 列匹配正则 值
TAG_FILTER_NOT_REGEXP = 7, // 列不匹配正则 值
+ TAG_FILTER_IS_NULL = 8, // 列为空
+ TAG_FILTER_IS_NOT_NULL = 9, // 列不为空
} TagFilterOp;
/**
@@ -417,7 +538,7 @@ typedef enum {
* @param reader [输入] 有效的 TsFileReader 句柄。
* @param table_name [输入] 其 schema 定义了这些标签列的表名。
* @param column_name [输入] 要过滤的标签列名。
- * @param value [输入] 比较值(标签列为 STRING 类型)。
+ * @param value [输入] 比较值(IS NULL / IS NOT NULL 会忽略该参数)。
* @param op [输入] 比较运算符(TagFilterOp)。
* @param err_code [输出] 成功返回 RET_OK(0),否则返回 errno_define_c.h 中的错误码。
* @return 成功返回 TagFilterHandle,失败返回 NULL。
@@ -437,6 +558,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader,
const char* lower, const char* upper,
bool is_not, ERRNO* err_code);
+// 常用单列谓词的便捷构造函数。
+TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+
// 组合谓词。AND/OR/NOT 会接管其子节点的所有权,只需释放根节点。
TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right);
TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right);
@@ -493,6 +628,18 @@ ResultSet tsfile_query_table_with_tag_filter(
TagFilterHandle tag_filter, int batch_size, ERRNO* err_code);
```
+### 将批结果读取为 Arrow
+
+对 `batch_size > 0` 创建的批查询结果集,可以使用 Arrow C Data Interface 读取
+`ArrowArray` 和 `ArrowSchema`。调用者拥有返回的 Arrow 对象,使用完后必须调用
+它们的 `release` 回调。
+
+```C
+ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set,
+ ArrowArray* out_array,
+ ArrowSchema* out_schema);
+```
+
示例——只读取 `region` 标签等于 `shanghai` 的设备的 `temperature`:
```C
@@ -640,6 +787,17 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader,
TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader,
uint32_t* size);
+/**
+ * @brief 获取 TsFile 中所有 timeseries schema。
+ *
+ * @param size [输出] 返回的 DeviceSchema 数组中的元素数量。
+ * @return DeviceSchema* 设备 schema 数组指针。
+ * @note 调用者必须对数组中的每个元素调用 free_device_schema(),
+ * 并对整个数组指针调用 free() 进行释放。
+ */
+DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader,
+ uint32_t* size);
+
/**
* @brief 释放 TableSchema 占用的内存空间。
*
@@ -647,5 +805,40 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader,
*/
void free_table_schema(TableSchema schema);
+
+void free_device_schema(DeviceSchema schema);
```
+### 获取设备与 Timeseries Metadata
+
+```C
+/**
+ * @brief 列出文件中的所有设备。
+ *
+ * @param out_devices [输出] 分配出的设备数组;用 tsfile_free_device_id_array() 释放。
+ * @param out_length [输出] 返回数组中的设备数量。
+ */
+ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices,
+ uint32_t* out_length);
+
+void tsfile_free_device_id_array(DeviceID* devices, uint32_t length);
+void tsfile_device_id_free_contents(DeviceID* d);
+
+/**
+ * @brief 获取文件中所有设备的 timeseries metadata。
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap* out_map);
+
+/**
+ * @brief 获取指定设备的 timeseries metadata。
+ *
+ * length == 0 返回空 map。非空输入中每个 DeviceID.path 应包含规范设备路径。
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID* devices, uint32_t length,
+ DeviceTimeseriesMetadataMap* out_map);
+
+void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap* map);
+```
diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
index 50de78b47..f337c46ec 100644
--- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
+++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
@@ -31,10 +31,14 @@ enum TSDataType : uint8_t {
FLOAT = 3,
DOUBLE = 4,
TEXT = 5,
+ VECTOR = 6,
+ UNKNOWN = 7,
TIMESTAMP = 8,
DATE = 9,
BLOB = 10,
STRING = 11,
+ NULL_TYPE = 254,
+ INVALID_DATATYPE = 255,
};
// 值编码。各编码适用于哪些类型见下表。
@@ -42,19 +46,35 @@ enum TSEncoding : uint8_t {
PLAIN = 0,
DICTIONARY = 1,
RLE = 2,
+ DIFF = 3,
TS_2DIFF = 4,
+ BITMAP = 5,
+ GORILLA_V1 = 6,
+ REGULAR = 7,
GORILLA = 8,
ZIGZAG = 9,
+ FREQ = 10,
+ CHIMP = 11,
SPRINTZ = 12,
+ RLBE = 13,
+ CAMEL = 14,
+ INVALID_ENCODING = 255,
};
-// 压缩类型。SNAPPY/GZIP/LZO/LZ4 取决于构建选项;默认压缩为 LZ4。
+// 压缩类型。SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 取决于构建选项;
+// 默认压缩为 LZ4。
enum CompressionType : uint8_t {
UNCOMPRESSED = 0,
SNAPPY = 1,
GZIP = 2,
LZO = 3,
+ SDT = 4,
+ PAA = 5,
+ PLA = 6,
LZ4 = 7,
+ ZSTD = 8,
+ LZMA2 = 9,
+ INVALID_COMPRESSION = 255,
};
// 列在表 schema 内的角色。
@@ -72,6 +92,8 @@ enum class ColumnCategory { TAG = 0, FIELD = 1, ATTRIBUTE = 2, TIME = 3 };
| `GORILLA` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` |
| `ZIGZAG` | `INT32`、`INT64` |
| `SPRINTZ` | `INT32`、`INT64`、`FLOAT`、`DOUBLE` |
+| `CHIMP`、`RLBE` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` |
+| `CAMEL` | `DOUBLE` |
各类型的默认值编码:`BOOLEAN → PLAIN`、`INT32 / INT64 → TS_2DIFF`、
`FLOAT / DOUBLE → GORILLA`、`TEXT / STRING / BLOB → PLAIN`。默认压缩为 `LZ4`。
@@ -135,6 +157,64 @@ class TsFileTableWriter {
};
```
+### TsFileWriter
+
+`TsFileWriter` 是 `tsfile_writer.h` 中的低层写入接口,支持 tree 模型和 table
+模型写入。
+
+```cpp
+extern int libtsfile_init();
+extern void libtsfile_destroy();
+
+// 写入器配置。非法值返回 common::E_INVALID_ARG,并保持原配置不变。
+extern int set_page_max_point_count(uint32_t page_max_point_count);
+extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node);
+
+class TsFileWriter {
+ public:
+ TsFileWriter();
+ ~TsFileWriter();
+ void destroy();
+
+ int open(const std::string& file_path, int flags, mode_t mode);
+ int open(const std::string& file_path);
+ int init(storage::WriteFile* write_file);
+ int init(storage::RestorableTsFileIOWriter* rw);
+
+ void set_generate_table_schema(bool generate_table_schema);
+
+ int register_timeseries(const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_timeseries(
+ const std::string& device_path,
+ const std::vector<MeasurementSchema*>& measurement_schema_vec);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const std::vector<MeasurementSchema*>& measurement_schemas);
+ int register_table(const std::shared_ptr<TableSchema>& table_schema);
+
+ int write_record(const TsRecord& record);
+ int write_tablet(const Tablet& tablet);
+ int write_record_aligned(const TsRecord& record);
+ int write_tablet_aligned(const Tablet& tablet);
+ int write_tree(const Tablet& tablet);
+ int write_tree(const TsRecord& record);
+ int write_table(Tablet& tablet);
+
+ DeviceSchemasMap* get_schema_group_map();
+ std::shared_ptr<TableSchema> get_table_schema(
+ const std::string& table_name) const;
+ int64_t calculate_mem_size_for_all_group();
+ int64_t calculate_meta_mem_size() const;
+ int check_memory_size_and_may_flush_chunks();
+ int flush();
+ int close();
+};
+```
+
### TableSchema
描述表模式(schema)的数据结构。
@@ -297,8 +377,14 @@ uint8_t common::get_global_compression();
// 时间列的编码/压缩(数据类型固定为 INT64)。
int common::set_global_time_encoding(uint8_t encoding);
int common::set_global_time_compression(uint8_t compression);
+uint8_t common::get_global_time_encoding();
+uint8_t common::get_global_time_compression();
```
+全局压缩支持 `UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`
+和 `LZMA2`。压缩枚举中也包含 `SDT`、`PAA`、`PLA` 等历史值,但全局压缩
+setter 会拒绝这些值。
+
## 读取接口
### Tsfile Reader
```cpp
@@ -335,6 +421,16 @@ class TsFileReader {
* @return 成功时返回 0,失败时返回 errno_define.h 中的非零错误码。
*/
int query(storage::QueryExpression *qe, ResultSet *&ret_qds);
+ /**
+ * @brief 通过路径列表、起始时间和结束时间查询 tsfile。用于 tree 模型。
+ *
+ * @param [in] path_list 完整路径列表。
+ * @param [in] start_time 起始时间。
+ * @param [in] end_time 结束时间。
+ * @param [out] result_set 查询结果集。
+ */
+ int query(std::vector<std::string>& path_list, int64_t start_time,
+ int64_t end_time, ResultSet*& result_set);
/**
* @brief 通过表名、列名、起始时间和结束时间查询 tsfile。
*
@@ -346,7 +442,7 @@ class TsFileReader {
*/
int query(const std::string &table_name,
const std::vector<std::string> &columns_names, int64_t start_time,
- int64_t end_time, ResultSet *&result_set);
+ int64_t end_time, ResultSet *&result_set, int batch_size = -1);
/**
* @brief 通过表名、列名、开始时间、结束时间和标签过滤器查询 tsfile。
@@ -360,7 +456,14 @@ class TsFileReader {
*/
int query(const std::string& table_name,
const std::vector<std::string>& columns_names, int64_t start_time,
- int64_t end_time, ResultSet*& result_set, Filter* tag_filter);
+ int64_t end_time, ResultSet*& result_set, Filter* tag_filter,
+ int batch_size = 0);
+
+ /**
+ * @brief 按行查询 tree 模型序列,支持 offset/limit。
+ */
+ int queryByRow(std::vector<std::string>& path_list, int offset, int limit,
+ ResultSet*& result_set);
/**
* @brief 按行查询表,支持偏移量/行数限制下推与可选的标签过滤。
@@ -379,12 +482,39 @@ class TsFileReader {
int limit, ResultSet*& result_set,
Filter* tag_filter = nullptr, int batch_size = 0);
+ /**
+ * @brief 按测点名在时间范围内查询 tree 模型数据。
+ */
+ int query_table_on_tree(const std::vector<std::string>& measurement_names,
+ int64_t start_time, int64_t end_time,
+ ResultSet*& result_set);
+
/**
* @brief 销毁结果集,该方法应在查询完成并使用完 result_set 后调用。
*
* @param qds 查询结果集。
*/
void destroy_query_data_set(ResultSet *qds);
+
+ ResultSet* read_timeseries(
+ const std::shared_ptr<IDeviceID>& device_id,
+ const std::vector<std::string>& measurement_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices(
+ std::string table_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids();
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices();
+
+ int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id,
+ std::vector<MeasurementSchema>& result);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata(
+ const std::vector<std::shared_ptr<IDeviceID>>& device_ids);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata();
+
/**
* @brief 根据表名获取表的模式信息。
*
@@ -401,6 +531,11 @@ class TsFileReader {
std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas();
};
```
+
+`DeviceTimeseriesMetadataMap` 是 `get_timeseries_metadata()` 返回的元数据
+map:`std::map<std::shared_ptr<IDeviceID>,
+std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`。
+
### ResultSet
```cpp
/**
@@ -540,6 +675,8 @@ class TagFilterBuilder {
Filter* reg_exp(const std::string& columnName, const std::string& value);
Filter* not_reg_exp(const std::string& columnName,
const std::string& value);
+ Filter* is_null(const std::string& columnName);
+ Filter* is_not_null(const std::string& columnName);
Filter* between_and(const std::string& columnName, const std::string& lower,
const std::string& upper);
Filter* not_between_and(const std::string& columnName,
@@ -550,4 +687,4 @@ class TagFilterBuilder {
static Filter* or_filter(Filter* left, Filter* right);
static Filter* not_filter(Filter* filter);
};
-```
\ No newline at end of file
+```
diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
index c94b690c7..1d3e875b1 100644
--- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
+++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
@@ -45,10 +45,17 @@ class TSEncoding(IntEnum):
PLAIN = 0 # 所有类型
DICTIONARY = 1 # STRING、TEXT
RLE = 2 # INT32、INT64、TIMESTAMP、DATE
+ DIFF = 3
TS_2DIFF = 4 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
+ BITMAP = 5
+ GORILLA_V1 = 6
+ REGULAR = 7
GORILLA = 8 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
ZIGZAG = 9 # INT32、INT64
+ CHIMP = 11 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
SPRINTZ = 12 # INT32、INT64、FLOAT、DOUBLE
+ RLBE = 13 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
+ CAMEL = 14 # DOUBLE
class Compressor(IntEnum):
"""
@@ -58,7 +65,12 @@ class Compressor(IntEnum):
SNAPPY = 1
GZIP = 2
LZO = 3
+ SDT = 4
+ PAA = 5
+ PLA = 6
LZ4 = 7
+ ZSTD = 8
+ LZMA2 = 9
class ColumnCategory(IntEnum):
"""
@@ -99,6 +111,32 @@ class ResultSetMetaData:
def __init__(self, column_list: List[str], data_types: List[TSDataType])
+@dataclass(frozen=True)
+class DeviceID:
+ path: Optional[str]
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+
+@dataclass(frozen=True)
+class TimeseriesStatistic:
+ has_statistic: bool
+ row_count: int
+ start_time: int
+ end_time: int
+
+@dataclass(frozen=True)
+class TimeseriesMetadata:
+ measurement_name: str
+ data_type: TSDataType
+ chunk_meta_count: int
+ statistic: TimeseriesStatistic
+ timeline_statistic: TimeseriesStatistic
+
+@dataclass(frozen=True)
+class DeviceTimeseriesMetadataGroup:
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+ timeseries: list[TimeseriesMetadata]
```
@@ -106,7 +144,7 @@ class ResultSetMetaData:
## 写入接口
-### TsFileWriter
+### TsFileTableWriter
```python
class TsFileTableWriter:
@@ -157,6 +195,48 @@ class TsFileTableWriter:
```
+### TsFileWriter
+
+`TsFileWriter` 是 `writer.pyx` 暴露的低层写入接口。它可以写入 tree 模型数据
+(`register_timeseries`、`register_device`、`write_tablet`、`write_row_record`),
+也可以写入 table 模型数据(`register_table`、`write_table`、`write_dataframe`、
+`write_arrow_batch`)。
+
+```python
+class TsFileWriter:
+ """
+ :param pathname: 目标 TsFile 路径。
+ :param memory_threshold: 触发自动刷盘前缓冲的字节数(默认 128MB)。
+ """
+ def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024)
+
+ def register_timeseries(self, device_name: str,
+ timeseries_schema: TimeseriesSchema)
+
+ def register_device(self, device_schema: DeviceSchema)
+
+ def register_table(self, table_schema: TableSchema)
+
+ def write_tablet(self, tablet: Tablet)
+
+ def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame,
+ tableschema: TableSchema)
+
+ def write_row_record(self, record: RowRecord)
+
+ def write_table(self, tablet: Tablet)
+
+ def write_arrow_batch(self, table_name: str, data,
+ time_col_index: int = -1)
+
+ def flush(self)
+
+ def close(self)
+
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
+```
+
### Tablet definition
@@ -223,11 +303,12 @@ set_tsfile_config({
| 数据类型 | 允许的编码 | 默认值 |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` |
+| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` |
| `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` |
-压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。
+压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`(默认 `LZ4`)。枚举还暴露 `SDT`、`PAA`、`PLA` 等历史值,但全局配置 setter 会拒绝这些值。
## 读取接口
@@ -253,11 +334,39 @@ class TsFileReader:
:param column_names: 要检索的列名列表。
:param start_time: 查询范围的起始时间(默认:int64 最小值)。
:param end_time: 查询范围的结束时间(默认:int64 最大值)。
+ :param tag_filter: 可选的 table 模型 TAG 列谓词。
+ :param batch_size: <= 0 逐行返回;> 0 按该大小返回数据块。
:return: 查询结果集处理器。
"""
def query_table(self, table_name : str, column_names : List[str],
start_time : int = np.iinfo(np.int64).min,
- end_time: int = np.iinfo(np.int64).max) -> ResultSet
+ end_time: int = np.iinfo(np.int64).max,
+ tag_filter = None, batch_size : int = 0) -> ResultSet
+
+ """
+ 对 tree 模型测点列执行时间范围查询。
+
+ :param column_names: 要检索的测点名列表。
+ :param start_time: 查询范围的起始时间。
+ :param end_time: 查询范围的结束时间。
+ :return: 查询结果集处理器。
+ """
+ def query_table_on_tree(self, column_names : List[str],
+ start_time : int = np.iinfo(np.int64).min,
+ end_time : int = np.iinfo(np.int64).max) -> ResultSet
+
+ """
+ 按行查询 tree 模型数据,支持 offset/limit。
+
+ :param device_ids: 要查询的设备标识列表。
+ :param measurement_names: 要检索的测点名列表。
+ :param offset: 需要跳过的起始行数。
+ :param limit: 最多返回行数;< 0 表示不限制。
+ :return: 查询结果集处理器。
+ """
+ def query_tree_by_row(self, device_ids : List[str],
+ measurement_names : List[str],
+ offset : int = 0, limit : int = -1) -> ResultSet
"""
按行查询表,支持偏移量/行数限制下推与可选的标签过滤。标签谓词把查询限定到
@@ -277,6 +386,18 @@ class TsFileReader:
offset : int = 0, limit : int = -1,
tag_filter = None, batch_size : int = 0) -> ResultSet
+ """
+ 对单个设备执行 tree 模型时间范围查询。
+
+ :param device_name: 设备标识。
+ :param sensor_list: 要检索的测点名列表。
+ :param start_time: 查询起始时间。
+ :param end_time: 查询结束时间。
+ :return: 查询结果集处理器。
+ """
+ def query_timeseries(self, device_name : str, sensor_list : List[str],
+ start_time : int = 0, end_time : int = 0) -> ResultSet
+
"""
获取指定表的模式信息。
@@ -292,6 +413,31 @@ class TsFileReader:
"""
def get_all_table_schemas(self) -> dict[str, TableSchema]
+ """
+ 获取所有 tree 模型 timeseries schema,按设备分组。
+
+ :return: DeviceSchema 对象列表。
+ """
+ def get_all_timeseries_schemas(self) -> list[DeviceSchema]
+
+ """
+ 获取文件中的所有设备标识。
+
+ :return: DeviceID(path, table_name, segments) 列表。
+ """
+ def get_all_devices(self) -> List[DeviceID]
+
+ """
+ 获取所有设备或指定设备的 per-timeseries metadata。
+
+ :param device_ids: None 表示全部设备,[] 表示空结果,或传入 DeviceID /
+ 路径兼容的设备标识列表。
+ :return: dict,key 为设备 segment tuple,value 为 DeviceTimeseriesMetadataGroup。
+ """
+ def get_timeseries_metadata(
+ self, device_ids: Optional[List] = None
+ ) -> Dict[tuple, DeviceTimeseriesMetadataGroup]
+
"""
关闭 TsFile 读取器。如果读取器中有活动的结果集,它们将失效。
"""
@@ -332,6 +478,12 @@ class ResultSet:
"""
def read_data_frame(self, max_row_num : int = 1024) -> DataFrame
+ """
+ 将下一个批结果读取为 pyarrow.Table。没有更多 TsBlock 批时返回 None。
+ 仅适用于通过 batch_size > 0 创建的结果集。
+ """
+ def read_arrow_batch(self) -> Optional[pyarrow.Table]
+
"""
从查询结果集中按索引获取值。
@@ -376,6 +528,9 @@ class ResultSet:
"""
def close(self)
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
+
```
diff --git a/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md b/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md
index 816a8cca6..2a7bf9616 100644
--- a/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md
+++ b/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md
@@ -59,9 +59,11 @@ data.values # -> np.ndarray, shape (N, 2):N
| 示例 | 操作 | 返回类型 |
|---|---|---|
-| `TsFileDataFrame(paths)` | 加载文件 / 文件列表 / 目录 | `TsFileDataFrame` |
+| `TsFileDataFrame(paths, show_progress=True)` | 加载文件 / 文件列表 / 目录。设置 `show_progress=False` 可关闭 stderr 上的加载进度 | `TsFileDataFrame` |
| `len(df)` | 时间序列总数 | `int` |
+| `df.model` | 已加载文件模型:`"table"` 或 `"tree"` | `str` |
| `df.list_timeseries("weather")` | 获取序列名,可按前缀筛选 | `List[str]` |
+| `df.list_timeseries_metadata("weather")` | 获取序列元数据,可按前缀筛选 | `pandas.DataFrame` |
| `df["weather.Beijing.humidity"]`、`df[0]`、`df[-1]` | 获取单条序列 | `Timeseries` |
| `df["city"]` | 获取某元数据列(标签 / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` |
| `df[0:3]`、`df[[0, 2, 5]]` | 按整数位置取子集视图:连续区间(`0:3`)或所列位置(`[0, 2, 5]`);位置即打印的 `index` 列 | `TsFileDataFrame` |
@@ -132,6 +134,7 @@ from tsfile import TsFileDataFrame
df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"])
df = TsFileDataFrame("data/") # 递归查找目录下所有 .tsfile
+df = TsFileDataFrame("data/", show_progress=False)
print(df)
```
@@ -172,7 +175,21 @@ TsFileDataFrame(table model, 972 time series, 5 files)
['weather.Beijing.humidity', 'weather.Beijing.temperature']
```
-若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)。
+若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)——也可以调用
+`list_timeseries_metadata()`。
+
+## 查看元数据
+
+`list_timeseries_metadata(path_prefix="")` 返回以序列名为索引的 pandas DataFrame。
+其中包含 `field`、`start_time`、`end_time`、`count` 以及设备标签列。对于
+table 模型文件,还会包含 `table` 列。
+
+```python
+meta = df.list_timeseries_metadata()
+weather_meta = df.list_timeseries_metadata("weather")
+
+meta[["field", "start_time", "end_time", "count"]].head()
+```
## 选取序列
diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
index de6d39796..1edc34e54 100644
--- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
+++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md
@@ -32,10 +32,12 @@ typedef enum {
TS_DATATYPE_FLOAT = 3,
TS_DATATYPE_DOUBLE = 4,
TS_DATATYPE_TEXT = 5,
+ TS_DATATYPE_VECTOR = 6,
TS_DATATYPE_TIMESTAMP = 8,
TS_DATATYPE_DATE = 9,
TS_DATATYPE_BLOB = 10,
TS_DATATYPE_STRING = 11,
+ TS_DATATYPE_NULL_TYPE = 254,
TS_DATATYPE_INVALID = 255
} TSDataType;
@@ -44,10 +46,18 @@ typedef enum {
TS_ENCODING_PLAIN = 0,
TS_ENCODING_DICTIONARY = 1,
TS_ENCODING_RLE = 2,
+ TS_ENCODING_DIFF = 3,
TS_ENCODING_TS_2DIFF = 4,
+ TS_ENCODING_BITMAP = 5,
+ TS_ENCODING_GORILLA_V1 = 6,
+ TS_ENCODING_REGULAR = 7,
TS_ENCODING_GORILLA = 8,
TS_ENCODING_ZIGZAG = 9,
+ TS_ENCODING_FREQ = 10,
+ TS_ENCODING_CHIMP = 11,
TS_ENCODING_SPRINTZ = 12,
+ TS_ENCODING_RLBE = 13,
+ TS_ENCODING_CAMEL = 14,
TS_ENCODING_INVALID = 255
} TSEncoding;
@@ -57,7 +67,12 @@ typedef enum {
TS_COMPRESSION_SNAPPY = 1,
TS_COMPRESSION_GZIP = 2,
TS_COMPRESSION_LZO = 3,
+ TS_COMPRESSION_SDT = 4,
+ TS_COMPRESSION_PAA = 5,
+ TS_COMPRESSION_PLA = 6,
TS_COMPRESSION_LZ4 = 7,
+ TS_COMPRESSION_ZSTD = 8,
+ TS_COMPRESSION_LZMA2 = 9,
TS_COMPRESSION_INVALID = 255
} CompressionType;
@@ -83,15 +98,89 @@ typedef struct table_schema {
int column_num;
} TableSchema;
+typedef struct timeseries_schema {
+ char* timeseries_name;
+ TSDataType data_type;
+ TSEncoding encoding;
+ CompressionType compression;
+} TimeseriesSchema;
+
+typedef struct device_schema {
+ char* device_name;
+ TimeseriesSchema* timeseries_schema;
+ int timeseries_num;
+} DeviceSchema;
+
// ResultSetMetaData:结果集的元数据,包括列名和数据类型。
typedef struct result_set_meta_data {
char** column_names;
TSDataType* data_types;
int column_num;
} ResultSetMetaData;
+
+typedef struct arrow_schema ArrowSchema;
+typedef struct arrow_array ArrowArray;
+
+typedef struct DeviceID {
+ char* path;
+ char* table_name;
+ uint32_t segment_count;
+ char** segments;
+} DeviceID;
+
+typedef struct TsFileStatisticBase {
+ bool has_statistic;
+ TSDataType type;
+ int32_t row_count;
+ int64_t start_time;
+ int64_t end_time;
+} TsFileStatisticBase;
+
+typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic;
+typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic;
+typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic;
+typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic;
+typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic;
+
+typedef union TimeseriesStatisticUnion {
+ TsFileBoolStatistic bool_s;
+ TsFileIntStatistic int_s;
+ TsFileFloatStatistic float_s;
+ TsFileStringStatistic string_s;
+ TsFileTextStatistic text_s;
+} TimeseriesStatisticUnion;
+
+typedef struct TimeseriesStatistic {
+ TimeseriesStatisticUnion u;
+} TimeseriesStatistic;
+
+#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u)
+
+typedef struct TimeseriesMetadata {
+ char* measurement_name;
+ TSDataType data_type;
+ int32_t chunk_meta_count;
+ TimeseriesStatistic statistic;
+ TimeseriesStatistic timeline_statistic;
+} TimeseriesMetadata;
+
+typedef struct DeviceTimeseriesMetadataEntry {
+ DeviceID device;
+ TimeseriesMetadata* timeseries;
+ uint32_t timeseries_count;
+} DeviceTimeseriesMetadataEntry;
+
+typedef struct DeviceTimeseriesMetadataMap {
+ DeviceTimeseriesMetadataEntry* entries;
+ uint32_t device_count;
+} DeviceTimeseriesMetadataMap;
```
> `ColumnSchema` 不携带编码/压缩:写入时列遵循全局默认值(见[配置](#配置编码与压缩)),读取时按文件中的实际配置解码。
+>
+> `TimeseriesStatistic` 是 `tsfile_cwrapper.h` 中的带标签 union。可通过
+> `tsfile_statistic_base(&metadata.statistic)` 读取通用字段,再根据数据类型读取
+> `int_s`、`float_s`、`bool_s`、`string_s` 或 `text_s`。
## 写入接口
@@ -207,9 +296,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index,
*
* @param value [输入] 以 '\0' 结尾的字符串,调用者保留所有权。
*/
-ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index,
- const char* column_name,
- const char* value);
+ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ const char* column_name,
+ const char* value,
+ int value_len);
// 支持多种数据类型插入
ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index,
@@ -236,9 +327,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index,
*
* @param value [输入] 字符串会被内部复制。
*/
-ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index,
- uint32_t column_index,
- const char* value);
+ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet,
+ uint32_t row_index,
+ uint32_t column_index,
+ const char* value,
+ int value_len);
ERRNO tablet_add_value_by_index_int32_t(Tablet tablet, uint32_t row_index,
@@ -310,11 +403,12 @@ uint8_t get_global_time_compression();
| 数据类型 | 允许的编码 | 默认值 |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` |
+| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` |
| `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` |
-压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。
+压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`(默认 `LZ4`)。
```C
// 例如:所有列均以 LZ4 压缩写入
@@ -387,6 +481,31 @@ bool tsfile_result_set_next(ResultSet result_set, ERRNO* error_code);
void free_tsfile_result_set(ResultSet* result_set);
```
+### Tree 查询
+
+```C
+/**
+ * @brief 按测点名在时间范围内查询 tree 模型数据。
+ */
+ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns,
+ uint32_t column_num, Timestamp start_time,
+ Timestamp end_time, ERRNO* err_code);
+
+/**
+ * @brief 按行查询 tree 模型数据,支持 offset/limit。
+ *
+ * @param device_ids 设备标识数组。
+ * @param measurement_names 测点名数组。
+ * @param offset 需要跳过的起始行数(>= 0)。
+ * @param limit 最多返回行数;< 0 表示不限制。
+ */
+ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader,
+ char** device_ids, int device_ids_len,
+ char** measurement_names,
+ int measurement_names_len, int offset,
+ int limit, ERRNO* err_code);
+```
+
### 按标签过滤
@@ -409,6 +528,8 @@ typedef enum {
TAG_FILTER_GTEQ = 5, // 列 >= 值
TAG_FILTER_REGEXP = 6, // 列匹配正则 值
TAG_FILTER_NOT_REGEXP = 7, // 列不匹配正则 值
+ TAG_FILTER_IS_NULL = 8, // 列为空
+ TAG_FILTER_IS_NOT_NULL = 9, // 列不为空
} TagFilterOp;
/**
@@ -417,7 +538,7 @@ typedef enum {
* @param reader [输入] 有效的 TsFileReader 句柄。
* @param table_name [输入] 其 schema 定义了这些标签列的表名。
* @param column_name [输入] 要过滤的标签列名。
- * @param value [输入] 比较值(标签列为 STRING 类型)。
+ * @param value [输入] 比较值(IS NULL / IS NOT NULL 会忽略该参数)。
* @param op [输入] 比较运算符(TagFilterOp)。
* @param err_code [输出] 成功返回 RET_OK(0),否则返回 errno_define_c.h 中的错误码。
* @return 成功返回 TagFilterHandle,失败返回 NULL。
@@ -437,6 +558,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader,
const char* lower, const char* upper,
bool is_not, ERRNO* err_code);
+// 常用单列谓词的便捷构造函数。
+TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name,
+ const char* column_name, const char* value);
+
// 组合谓词。AND/OR/NOT 会接管其子节点的所有权,只需释放根节点。
TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right);
TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right);
@@ -493,6 +628,18 @@ ResultSet tsfile_query_table_with_tag_filter(
TagFilterHandle tag_filter, int batch_size, ERRNO* err_code);
```
+### 将批结果读取为 Arrow
+
+对 `batch_size > 0` 创建的批查询结果集,可以使用 Arrow C Data Interface 读取
+`ArrowArray` 和 `ArrowSchema`。调用者拥有返回的 Arrow 对象,使用完后必须调用
+它们的 `release` 回调。
+
+```C
+ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set,
+ ArrowArray* out_array,
+ ArrowSchema* out_schema);
+```
+
示例——只读取 `region` 标签等于 `shanghai` 的设备的 `temperature`:
```C
@@ -640,6 +787,17 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader,
TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader,
uint32_t* size);
+/**
+ * @brief 获取 TsFile 中所有 timeseries schema。
+ *
+ * @param size [输出] 返回的 DeviceSchema 数组中的元素数量。
+ * @return DeviceSchema* 设备 schema 数组指针。
+ * @note 调用者必须对数组中的每个元素调用 free_device_schema(),
+ * 并对整个数组指针调用 free() 进行释放。
+ */
+DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader,
+ uint32_t* size);
+
/**
* @brief 释放 TableSchema 占用的内存空间。
*
@@ -647,5 +805,40 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader,
*/
void free_table_schema(TableSchema schema);
+
+void free_device_schema(DeviceSchema schema);
```
+### 获取设备与 Timeseries Metadata
+
+```C
+/**
+ * @brief 列出文件中的所有设备。
+ *
+ * @param out_devices [输出] 分配出的设备数组;用 tsfile_free_device_id_array() 释放。
+ * @param out_length [输出] 返回数组中的设备数量。
+ */
+ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices,
+ uint32_t* out_length);
+
+void tsfile_free_device_id_array(DeviceID* devices, uint32_t length);
+void tsfile_device_id_free_contents(DeviceID* d);
+
+/**
+ * @brief 获取文件中所有设备的 timeseries metadata。
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap* out_map);
+
+/**
+ * @brief 获取指定设备的 timeseries metadata。
+ *
+ * length == 0 返回空 map。非空输入中每个 DeviceID.path 应包含规范设备路径。
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID* devices, uint32_t length,
+ DeviceTimeseriesMetadataMap* out_map);
+
+void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap* map);
+```
diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
index 50de78b47..f337c46ec 100644
--- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
+++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md
@@ -31,10 +31,14 @@ enum TSDataType : uint8_t {
FLOAT = 3,
DOUBLE = 4,
TEXT = 5,
+ VECTOR = 6,
+ UNKNOWN = 7,
TIMESTAMP = 8,
DATE = 9,
BLOB = 10,
STRING = 11,
+ NULL_TYPE = 254,
+ INVALID_DATATYPE = 255,
};
// 值编码。各编码适用于哪些类型见下表。
@@ -42,19 +46,35 @@ enum TSEncoding : uint8_t {
PLAIN = 0,
DICTIONARY = 1,
RLE = 2,
+ DIFF = 3,
TS_2DIFF = 4,
+ BITMAP = 5,
+ GORILLA_V1 = 6,
+ REGULAR = 7,
GORILLA = 8,
ZIGZAG = 9,
+ FREQ = 10,
+ CHIMP = 11,
SPRINTZ = 12,
+ RLBE = 13,
+ CAMEL = 14,
+ INVALID_ENCODING = 255,
};
-// 压缩类型。SNAPPY/GZIP/LZO/LZ4 取决于构建选项;默认压缩为 LZ4。
+// 压缩类型。SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 取决于构建选项;
+// 默认压缩为 LZ4。
enum CompressionType : uint8_t {
UNCOMPRESSED = 0,
SNAPPY = 1,
GZIP = 2,
LZO = 3,
+ SDT = 4,
+ PAA = 5,
+ PLA = 6,
LZ4 = 7,
+ ZSTD = 8,
+ LZMA2 = 9,
+ INVALID_COMPRESSION = 255,
};
// 列在表 schema 内的角色。
@@ -72,6 +92,8 @@ enum class ColumnCategory { TAG = 0, FIELD = 1, ATTRIBUTE = 2, TIME = 3 };
| `GORILLA` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` |
| `ZIGZAG` | `INT32`、`INT64` |
| `SPRINTZ` | `INT32`、`INT64`、`FLOAT`、`DOUBLE` |
+| `CHIMP`、`RLBE` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` |
+| `CAMEL` | `DOUBLE` |
各类型的默认值编码:`BOOLEAN → PLAIN`、`INT32 / INT64 → TS_2DIFF`、
`FLOAT / DOUBLE → GORILLA`、`TEXT / STRING / BLOB → PLAIN`。默认压缩为 `LZ4`。
@@ -135,6 +157,64 @@ class TsFileTableWriter {
};
```
+### TsFileWriter
+
+`TsFileWriter` 是 `tsfile_writer.h` 中的低层写入接口,支持 tree 模型和 table
+模型写入。
+
+```cpp
+extern int libtsfile_init();
+extern void libtsfile_destroy();
+
+// 写入器配置。非法值返回 common::E_INVALID_ARG,并保持原配置不变。
+extern int set_page_max_point_count(uint32_t page_max_point_count);
+extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node);
+
+class TsFileWriter {
+ public:
+ TsFileWriter();
+ ~TsFileWriter();
+ void destroy();
+
+ int open(const std::string& file_path, int flags, mode_t mode);
+ int open(const std::string& file_path);
+ int init(storage::WriteFile* write_file);
+ int init(storage::RestorableTsFileIOWriter* rw);
+
+ void set_generate_table_schema(bool generate_table_schema);
+
+ int register_timeseries(const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_timeseries(
+ const std::string& device_path,
+ const std::vector<MeasurementSchema*>& measurement_schema_vec);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const MeasurementSchema& measurement_schema);
+ int register_aligned_timeseries(
+ const std::string& device_id,
+ const std::vector<MeasurementSchema*>& measurement_schemas);
+ int register_table(const std::shared_ptr<TableSchema>& table_schema);
+
+ int write_record(const TsRecord& record);
+ int write_tablet(const Tablet& tablet);
+ int write_record_aligned(const TsRecord& record);
+ int write_tablet_aligned(const Tablet& tablet);
+ int write_tree(const Tablet& tablet);
+ int write_tree(const TsRecord& record);
+ int write_table(Tablet& tablet);
+
+ DeviceSchemasMap* get_schema_group_map();
+ std::shared_ptr<TableSchema> get_table_schema(
+ const std::string& table_name) const;
+ int64_t calculate_mem_size_for_all_group();
+ int64_t calculate_meta_mem_size() const;
+ int check_memory_size_and_may_flush_chunks();
+ int flush();
+ int close();
+};
+```
+
### TableSchema
描述表模式(schema)的数据结构。
@@ -297,8 +377,14 @@ uint8_t common::get_global_compression();
// 时间列的编码/压缩(数据类型固定为 INT64)。
int common::set_global_time_encoding(uint8_t encoding);
int common::set_global_time_compression(uint8_t compression);
+uint8_t common::get_global_time_encoding();
+uint8_t common::get_global_time_compression();
```
+全局压缩支持 `UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`
+和 `LZMA2`。压缩枚举中也包含 `SDT`、`PAA`、`PLA` 等历史值,但全局压缩
+setter 会拒绝这些值。
+
## 读取接口
### Tsfile Reader
```cpp
@@ -335,6 +421,16 @@ class TsFileReader {
* @return 成功时返回 0,失败时返回 errno_define.h 中的非零错误码。
*/
int query(storage::QueryExpression *qe, ResultSet *&ret_qds);
+ /**
+ * @brief 通过路径列表、起始时间和结束时间查询 tsfile。用于 tree 模型。
+ *
+ * @param [in] path_list 完整路径列表。
+ * @param [in] start_time 起始时间。
+ * @param [in] end_time 结束时间。
+ * @param [out] result_set 查询结果集。
+ */
+ int query(std::vector<std::string>& path_list, int64_t start_time,
+ int64_t end_time, ResultSet*& result_set);
/**
* @brief 通过表名、列名、起始时间和结束时间查询 tsfile。
*
@@ -346,7 +442,7 @@ class TsFileReader {
*/
int query(const std::string &table_name,
const std::vector<std::string> &columns_names, int64_t start_time,
- int64_t end_time, ResultSet *&result_set);
+ int64_t end_time, ResultSet *&result_set, int batch_size = -1);
/**
* @brief 通过表名、列名、开始时间、结束时间和标签过滤器查询 tsfile。
@@ -360,7 +456,14 @@ class TsFileReader {
*/
int query(const std::string& table_name,
const std::vector<std::string>& columns_names, int64_t start_time,
- int64_t end_time, ResultSet*& result_set, Filter* tag_filter);
+ int64_t end_time, ResultSet*& result_set, Filter* tag_filter,
+ int batch_size = 0);
+
+ /**
+ * @brief 按行查询 tree 模型序列,支持 offset/limit。
+ */
+ int queryByRow(std::vector<std::string>& path_list, int offset, int limit,
+ ResultSet*& result_set);
/**
* @brief 按行查询表,支持偏移量/行数限制下推与可选的标签过滤。
@@ -379,12 +482,39 @@ class TsFileReader {
int limit, ResultSet*& result_set,
Filter* tag_filter = nullptr, int batch_size = 0);
+ /**
+ * @brief 按测点名在时间范围内查询 tree 模型数据。
+ */
+ int query_table_on_tree(const std::vector<std::string>& measurement_names,
+ int64_t start_time, int64_t end_time,
+ ResultSet*& result_set);
+
/**
* @brief 销毁结果集,该方法应在查询完成并使用完 result_set 后调用。
*
* @param qds 查询结果集。
*/
void destroy_query_data_set(ResultSet *qds);
+
+ ResultSet* read_timeseries(
+ const std::shared_ptr<IDeviceID>& device_id,
+ const std::vector<std::string>& measurement_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices(
+ std::string table_name);
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids();
+
+ std::vector<std::shared_ptr<IDeviceID>> get_all_devices();
+
+ int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id,
+ std::vector<MeasurementSchema>& result);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata(
+ const std::vector<std::shared_ptr<IDeviceID>>& device_ids);
+
+ DeviceTimeseriesMetadataMap get_timeseries_metadata();
+
/**
* @brief 根据表名获取表的模式信息。
*
@@ -401,6 +531,11 @@ class TsFileReader {
std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas();
};
```
+
+`DeviceTimeseriesMetadataMap` 是 `get_timeseries_metadata()` 返回的元数据
+map:`std::map<std::shared_ptr<IDeviceID>,
+std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`。
+
### ResultSet
```cpp
/**
@@ -540,6 +675,8 @@ class TagFilterBuilder {
Filter* reg_exp(const std::string& columnName, const std::string& value);
Filter* not_reg_exp(const std::string& columnName,
const std::string& value);
+ Filter* is_null(const std::string& columnName);
+ Filter* is_not_null(const std::string& columnName);
Filter* between_and(const std::string& columnName, const std::string& lower,
const std::string& upper);
Filter* not_between_and(const std::string& columnName,
@@ -550,4 +687,4 @@ class TagFilterBuilder {
static Filter* or_filter(Filter* left, Filter* right);
static Filter* not_filter(Filter* filter);
};
-```
\ No newline at end of file
+```
diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
index c94b690c7..1d3e875b1 100644
--- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
+++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md
@@ -45,10 +45,17 @@ class TSEncoding(IntEnum):
PLAIN = 0 # 所有类型
DICTIONARY = 1 # STRING、TEXT
RLE = 2 # INT32、INT64、TIMESTAMP、DATE
+ DIFF = 3
TS_2DIFF = 4 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
+ BITMAP = 5
+ GORILLA_V1 = 6
+ REGULAR = 7
GORILLA = 8 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
ZIGZAG = 9 # INT32、INT64
+ CHIMP = 11 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
SPRINTZ = 12 # INT32、INT64、FLOAT、DOUBLE
+ RLBE = 13 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE
+ CAMEL = 14 # DOUBLE
class Compressor(IntEnum):
"""
@@ -58,7 +65,12 @@ class Compressor(IntEnum):
SNAPPY = 1
GZIP = 2
LZO = 3
+ SDT = 4
+ PAA = 5
+ PLA = 6
LZ4 = 7
+ ZSTD = 8
+ LZMA2 = 9
class ColumnCategory(IntEnum):
"""
@@ -99,6 +111,32 @@ class ResultSetMetaData:
def __init__(self, column_list: List[str], data_types: List[TSDataType])
+@dataclass(frozen=True)
+class DeviceID:
+ path: Optional[str]
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+
+@dataclass(frozen=True)
+class TimeseriesStatistic:
+ has_statistic: bool
+ row_count: int
+ start_time: int
+ end_time: int
+
+@dataclass(frozen=True)
+class TimeseriesMetadata:
+ measurement_name: str
+ data_type: TSDataType
+ chunk_meta_count: int
+ statistic: TimeseriesStatistic
+ timeline_statistic: TimeseriesStatistic
+
+@dataclass(frozen=True)
+class DeviceTimeseriesMetadataGroup:
+ table_name: Optional[str]
+ segments: tuple[Optional[str], ...]
+ timeseries: list[TimeseriesMetadata]
```
@@ -106,7 +144,7 @@ class ResultSetMetaData:
## 写入接口
-### TsFileWriter
+### TsFileTableWriter
```python
class TsFileTableWriter:
@@ -157,6 +195,48 @@ class TsFileTableWriter:
```
+### TsFileWriter
+
+`TsFileWriter` 是 `writer.pyx` 暴露的低层写入接口。它可以写入 tree 模型数据
+(`register_timeseries`、`register_device`、`write_tablet`、`write_row_record`),
+也可以写入 table 模型数据(`register_table`、`write_table`、`write_dataframe`、
+`write_arrow_batch`)。
+
+```python
+class TsFileWriter:
+ """
+ :param pathname: 目标 TsFile 路径。
+ :param memory_threshold: 触发自动刷盘前缓冲的字节数(默认 128MB)。
+ """
+ def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024)
+
+ def register_timeseries(self, device_name: str,
+ timeseries_schema: TimeseriesSchema)
+
+ def register_device(self, device_schema: DeviceSchema)
+
+ def register_table(self, table_schema: TableSchema)
+
+ def write_tablet(self, tablet: Tablet)
+
+ def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame,
+ tableschema: TableSchema)
+
+ def write_row_record(self, record: RowRecord)
+
+ def write_table(self, tablet: Tablet)
+
+ def write_arrow_batch(self, table_name: str, data,
+ time_col_index: int = -1)
+
+ def flush(self)
+
+ def close(self)
+
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
+```
+
### Tablet definition
@@ -223,11 +303,12 @@ set_tsfile_config({
| 数据类型 | 允许的编码 | 默认值 |
|---|---|---|
| `BOOLEAN` | `PLAIN` | `PLAIN` |
-| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` |
-| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` |
+| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` |
+| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` |
+| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` |
| `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` |
-压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。
+压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`(默认 `LZ4`)。枚举还暴露 `SDT`、`PAA`、`PLA` 等历史值,但全局配置 setter 会拒绝这些值。
## 读取接口
@@ -253,11 +334,39 @@ class TsFileReader:
:param column_names: 要检索的列名列表。
:param start_time: 查询范围的起始时间(默认:int64 最小值)。
:param end_time: 查询范围的结束时间(默认:int64 最大值)。
+ :param tag_filter: 可选的 table 模型 TAG 列谓词。
+ :param batch_size: <= 0 逐行返回;> 0 按该大小返回数据块。
:return: 查询结果集处理器。
"""
def query_table(self, table_name : str, column_names : List[str],
start_time : int = np.iinfo(np.int64).min,
- end_time: int = np.iinfo(np.int64).max) -> ResultSet
+ end_time: int = np.iinfo(np.int64).max,
+ tag_filter = None, batch_size : int = 0) -> ResultSet
+
+ """
+ 对 tree 模型测点列执行时间范围查询。
+
+ :param column_names: 要检索的测点名列表。
+ :param start_time: 查询范围的起始时间。
+ :param end_time: 查询范围的结束时间。
+ :return: 查询结果集处理器。
+ """
+ def query_table_on_tree(self, column_names : List[str],
+ start_time : int = np.iinfo(np.int64).min,
+ end_time : int = np.iinfo(np.int64).max) -> ResultSet
+
+ """
+ 按行查询 tree 模型数据,支持 offset/limit。
+
+ :param device_ids: 要查询的设备标识列表。
+ :param measurement_names: 要检索的测点名列表。
+ :param offset: 需要跳过的起始行数。
+ :param limit: 最多返回行数;< 0 表示不限制。
+ :return: 查询结果集处理器。
+ """
+ def query_tree_by_row(self, device_ids : List[str],
+ measurement_names : List[str],
+ offset : int = 0, limit : int = -1) -> ResultSet
"""
按行查询表,支持偏移量/行数限制下推与可选的标签过滤。标签谓词把查询限定到
@@ -277,6 +386,18 @@ class TsFileReader:
offset : int = 0, limit : int = -1,
tag_filter = None, batch_size : int = 0) -> ResultSet
+ """
+ 对单个设备执行 tree 模型时间范围查询。
+
+ :param device_name: 设备标识。
+ :param sensor_list: 要检索的测点名列表。
+ :param start_time: 查询起始时间。
+ :param end_time: 查询结束时间。
+ :return: 查询结果集处理器。
+ """
+ def query_timeseries(self, device_name : str, sensor_list : List[str],
+ start_time : int = 0, end_time : int = 0) -> ResultSet
+
"""
获取指定表的模式信息。
@@ -292,6 +413,31 @@ class TsFileReader:
"""
def get_all_table_schemas(self) -> dict[str, TableSchema]
+ """
+ 获取所有 tree 模型 timeseries schema,按设备分组。
+
+ :return: DeviceSchema 对象列表。
+ """
+ def get_all_timeseries_schemas(self) -> list[DeviceSchema]
+
+ """
+ 获取文件中的所有设备标识。
+
+ :return: DeviceID(path, table_name, segments) 列表。
+ """
+ def get_all_devices(self) -> List[DeviceID]
+
+ """
+ 获取所有设备或指定设备的 per-timeseries metadata。
+
+ :param device_ids: None 表示全部设备,[] 表示空结果,或传入 DeviceID /
+ 路径兼容的设备标识列表。
+ :return: dict,key 为设备 segment tuple,value 为 DeviceTimeseriesMetadataGroup。
+ """
+ def get_timeseries_metadata(
+ self, device_ids: Optional[List] = None
+ ) -> Dict[tuple, DeviceTimeseriesMetadataGroup]
+
"""
关闭 TsFile 读取器。如果读取器中有活动的结果集,它们将失效。
"""
@@ -332,6 +478,12 @@ class ResultSet:
"""
def read_data_frame(self, max_row_num : int = 1024) -> DataFrame
+ """
+ 将下一个批结果读取为 pyarrow.Table。没有更多 TsBlock 批时返回 None。
+ 仅适用于通过 batch_size > 0 创建的结果集。
+ """
+ def read_arrow_batch(self) -> Optional[pyarrow.Table]
+
"""
从查询结果集中按索引获取值。
@@ -376,6 +528,9 @@ class ResultSet:
"""
def close(self)
+ def __enter__(self)
+ def __exit__(self, exc_type, exc_val, exc_tb)
+
```