You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/06/24 09:17:19 UTC
(tsfile) 01/01: fix tag null segsegv.
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch fix_null_tag_filter
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 6413b96424059d73887fcefc8f3c101095c087f9
Author: ColinLee <sh...@163.com>
AuthorDate: Wed Jun 24 17:16:44 2026 +0800
fix tag null segsegv.
---
cpp/src/cwrapper/tsfile_cwrapper.cc | 15 +++-
cpp/src/cwrapper/tsfile_cwrapper.h | 5 +-
cpp/src/reader/filter/tag_filter.cc | 60 ++++++++++---
cpp/src/reader/filter/tag_filter.h | 17 ++++
cpp/test/reader/filter/tag_filter_test.cc | 83 ++++++++++++++++++
python/tests/test_tsfile_dataset.py | 139 ++++++++++++++++++++++++++----
python/tsfile/__init__.py | 2 +
python/tsfile/dataset/reader.py | 39 ++++++---
python/tsfile/tag_filter.py | 23 ++++-
python/tsfile/tsfile_cpp.pxd | 2 +
10 files changed, 342 insertions(+), 43 deletions(-)
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc
index 0934981f9..d9e19fb6b 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.cc
+++ b/cpp/src/cwrapper/tsfile_cwrapper.cc
@@ -1118,8 +1118,13 @@ int duplicate_ideviceid_to_device_fields(storage::IDeviceID* id,
for (int i = 0; i < n; i++) {
const std::string* ps =
(static_cast<size_t>(i) < segs.size()) ? segs[i] : nullptr;
- const char* lit = (ps != nullptr) ? ps->c_str() : "null";
- seg_arr[i] = strdup(lit);
+ // A null tag segment is exposed as a NULL pointer so callers can
+ // distinguish a missing/null tag from the literal string "null".
+ if (ps == nullptr) {
+ seg_arr[i] = nullptr;
+ continue;
+ }
+ seg_arr[i] = strdup(ps->c_str());
if (seg_arr[i] == nullptr) {
for (int j = 0; j < i; j++) {
free(seg_arr[j]);
@@ -1627,6 +1632,12 @@ TagFilterHandle tsfile_tag_filter_create(TsFileReader reader,
case TAG_FILTER_NOT_REGEXP:
filter = builder.not_reg_exp(column_name, value);
break;
+ case TAG_FILTER_IS_NULL:
+ filter = builder.is_null(column_name);
+ break;
+ case TAG_FILTER_IS_NOT_NULL:
+ filter = builder.is_not_null(column_name);
+ break;
default:
*err_code = common::E_INVALID_ARG;
return nullptr;
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h
index ae3e28eed..4471da89e 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.h
+++ b/cpp/src/cwrapper/tsfile_cwrapper.h
@@ -875,6 +875,8 @@ typedef enum {
TAG_FILTER_GTEQ = 5,
TAG_FILTER_REGEXP = 6,
TAG_FILTER_NOT_REGEXP = 7,
+ TAG_FILTER_IS_NULL = 8,
+ TAG_FILTER_IS_NOT_NULL = 9,
} TagFilterOp;
/**
@@ -884,7 +886,8 @@ typedef enum {
* index).
* @param table_name [in] Table name whose schema defines the TAG columns.
* @param column_name [in] Name of the TAG column to filter on.
- * @param value [in] Comparison value (string).
+ * @param value [in] Comparison value (string). Ignored for
+ * TAG_FILTER_IS_NULL / TAG_FILTER_IS_NOT_NULL (may be NULL).
* @param op [in] Comparison operator (TagFilterOp).
* @param err_code [out] Error code. E_OK(0) on success.
* @return TagFilterHandle on success; NULL on failure.
diff --git a/cpp/src/reader/filter/tag_filter.cc b/cpp/src/reader/filter/tag_filter.cc
index f92c9ef86..0115cc389 100644
--- a/cpp/src/reader/filter/tag_filter.cc
+++ b/cpp/src/reader/filter/tag_filter.cc
@@ -44,7 +44,8 @@ TagEq::TagEq(int col_idx, std::string tag_value)
: TagFilter(col_idx, std::move(tag_value)) {}
bool TagEq::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
return *segments[col_idx_] == value_;
}
@@ -53,7 +54,8 @@ TagNeq::TagNeq(int col_idx, std::string tag_value)
: TagFilter(col_idx, std::move(tag_value)) {}
bool TagNeq::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
return *segments[col_idx_] != value_;
}
@@ -62,7 +64,8 @@ TagLt::TagLt(int col_idx, std::string tag_value)
: TagFilter(col_idx, std::move(tag_value)) {}
bool TagLt::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
return *segments[col_idx_] < value_;
}
@@ -71,7 +74,8 @@ TagLteq::TagLteq(int col_idx, std::string tag_value)
: TagFilter(col_idx, std::move(tag_value)) {}
bool TagLteq::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
return *segments[col_idx_] <= value_;
}
@@ -80,7 +84,8 @@ TagGt::TagGt(int col_idx, std::string tag_value)
: TagFilter(col_idx, std::move(tag_value)) {}
bool TagGt::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
return *segments[col_idx_] > value_;
}
@@ -89,7 +94,8 @@ TagGteq::TagGteq(int col_idx, std::string tag_value)
: TagFilter(col_idx, std::move(tag_value)) {}
bool TagGteq::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
return *segments[col_idx_] >= value_;
}
@@ -105,7 +111,9 @@ TagRegExp::TagRegExp(int col_idx, std::string tag_value)
}
bool TagRegExp::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size() || !is_valid_pattern_) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr ||
+ !is_valid_pattern_)
+ return false;
try {
return std::regex_search(*segments[col_idx_], pattern_);
} catch (const std::regex_error&) {
@@ -125,7 +133,9 @@ TagNotRegExp::TagNotRegExp(int col_idx, std::string tag_value)
}
bool TagNotRegExp::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size() || !is_valid_pattern_) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr ||
+ !is_valid_pattern_)
+ return false;
try {
return !std::regex_search(*segments[col_idx_], pattern_);
} catch (const std::regex_error&) {
@@ -133,6 +143,22 @@ bool TagNotRegExp::satisfyRow(std::vector<std::string*> segments) const {
}
}
+// TagIsNull implementation
+TagIsNull::TagIsNull(int col_idx) : TagFilter(col_idx, "") {}
+
+bool TagIsNull::satisfyRow(std::vector<std::string*> segments) const {
+ // A tag is null when its segment is an explicit null pointer or when the
+ // device id omits the (trailing) segment entirely.
+ return col_idx_ >= segments.size() || segments[col_idx_] == nullptr;
+}
+
+// TagIsNotNull implementation
+TagIsNotNull::TagIsNotNull(int col_idx) : TagFilter(col_idx, "") {}
+
+bool TagIsNotNull::satisfyRow(std::vector<std::string*> segments) const {
+ return col_idx_ < segments.size() && segments[col_idx_] != nullptr;
+}
+
// TagBetween implementation
TagBetween::TagBetween(int col_idx, std::string lower_value,
std::string upper_value)
@@ -141,7 +167,8 @@ TagBetween::TagBetween(int col_idx, std::string lower_value,
}
bool TagBetween::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
const std::string& segment_value = *segments[col_idx_];
return segment_value >= value_ && segment_value <= value2_;
}
@@ -154,7 +181,8 @@ TagNotBetween::TagNotBetween(int col_idx, std::string lower_value,
}
bool TagNotBetween::satisfyRow(std::vector<std::string*> segments) const {
- if (col_idx_ >= segments.size()) return false;
+ if (col_idx_ >= segments.size() || segments[col_idx_] == nullptr)
+ return false;
const std::string& segment_value = *segments[col_idx_];
return segment_value < value_ || segment_value > value2_;
}
@@ -254,6 +282,18 @@ Filter* TagFilterBuilder::not_reg_exp(const std::string& columnName,
return new TagNotRegExp(idx, value);
}
+Filter* TagFilterBuilder::is_null(const std::string& columnName) {
+ auto idx = get_id_column_index(columnName);
+ if (idx < 0) return nullptr;
+ return new TagIsNull(idx);
+}
+
+Filter* TagFilterBuilder::is_not_null(const std::string& columnName) {
+ auto idx = get_id_column_index(columnName);
+ if (idx < 0) return nullptr;
+ return new TagIsNotNull(idx);
+}
+
Filter* TagFilterBuilder::between_and(const std::string& columnName,
const std::string& lower,
const std::string& upper) {
diff --git a/cpp/src/reader/filter/tag_filter.h b/cpp/src/reader/filter/tag_filter.h
index b858be8c9..6e84dcf50 100644
--- a/cpp/src/reader/filter/tag_filter.h
+++ b/cpp/src/reader/filter/tag_filter.h
@@ -106,6 +106,21 @@ class TagNotRegExp : public TagFilter {
bool satisfyRow(std::vector<std::string*> segments) const override;
};
+// IS NULL: tag column has no value for this device. An absent trailing
+// segment (col_idx_ beyond the device's segment count) is also treated as null.
+class TagIsNull : public TagFilter {
+ public:
+ explicit TagIsNull(int col_idx);
+ bool satisfyRow(std::vector<std::string*> segments) const override;
+};
+
+// IS NOT NULL: tag column has a concrete value for this device.
+class TagIsNotNull : public TagFilter {
+ public:
+ explicit TagIsNotNull(int col_idx);
+ bool satisfyRow(std::vector<std::string*> segments) const override;
+};
+
// Range query [value_, value2_]
class TagBetween : public TagFilter {
public:
@@ -171,6 +186,8 @@ class TagFilterBuilder {
Filter* reg_exp(const std::string& columnName, const std::string& value);
Filter* not_reg_exp(const std::string& columnName,
const std::string& value);
+ Filter* is_null(const std::string& columnName);
+ Filter* is_not_null(const std::string& columnName);
Filter* between_and(const std::string& columnName, const std::string& lower,
const std::string& upper);
Filter* not_between_and(const std::string& columnName,
diff --git a/cpp/test/reader/filter/tag_filter_test.cc b/cpp/test/reader/filter/tag_filter_test.cc
index 0274d2424..02ce64b85 100644
--- a/cpp/test/reader/filter/tag_filter_test.cc
+++ b/cpp/test/reader/filter/tag_filter_test.cc
@@ -414,4 +414,87 @@ TEST_F(TagFilterTest, TagRegExpEdgeCases) {
delete invalid_filter;
delete empty_filter;
+}
+
+// A null tag segment must not crash comparison filters; a null value never
+// satisfies a concrete-value predicate (SQL UNKNOWN -> not matched).
+TEST_F(TagFilterTest, ComparisonFiltersTreatNullSegmentAsNoMatch) {
+ // "name" (col 1) is an explicit null pointer.
+ std::vector<std::string*> segments = {nullptr, nullptr,
+ new std::string("25"),
+ new std::string("engineering")};
+
+ auto eq = builder_->eq("name", "john");
+ auto neq = builder_->neq("name", "john");
+ auto lt = builder_->lt("name", "z");
+ auto gt = builder_->gt("name", "a");
+ auto between = builder_->between_and("name", "a", "z");
+ auto reg = builder_->reg_exp("name", ".*");
+
+ EXPECT_FALSE(eq->satisfyRow(0, segments));
+ EXPECT_FALSE(neq->satisfyRow(0, segments));
+ EXPECT_FALSE(lt->satisfyRow(0, segments));
+ EXPECT_FALSE(gt->satisfyRow(0, segments));
+ EXPECT_FALSE(between->satisfyRow(0, segments));
+ EXPECT_FALSE(reg->satisfyRow(0, segments));
+
+ delete eq;
+ delete neq;
+ delete lt;
+ delete gt;
+ delete between;
+ delete reg;
+ delete segments[2];
+ delete segments[3];
+}
+
+// IS NULL filter
+TEST_F(TagFilterTest, TagIsNullFilter) {
+ auto filter = builder_->is_null("name");
+ ASSERT_NE(filter, nullptr);
+
+ // "name" is an explicit null pointer.
+ std::vector<std::string*> null_seg = {nullptr, nullptr,
+ new std::string("25"),
+ new std::string("engineering")};
+ EXPECT_TRUE(filter->satisfyRow(0, null_seg));
+ delete null_seg[2];
+ delete null_seg[3];
+
+ // "name" has a concrete value.
+ auto present = createSegments("john", "25", "engineering");
+ EXPECT_FALSE(filter->satisfyRow(0, present));
+ cleanupSegments(present);
+
+ // A trailing tag column omitted from the device id (segment count too
+ // small) is also treated as null.
+ auto trailing = builder_->is_null("score"); // col_idx 5
+ std::vector<std::string*> short_seg = {nullptr, new std::string("john")};
+ EXPECT_TRUE(trailing->satisfyRow(0, short_seg));
+ delete short_seg[1];
+
+ delete filter;
+ delete trailing;
+}
+
+// IS NOT NULL filter
+TEST_F(TagFilterTest, TagIsNotNullFilter) {
+ auto filter = builder_->is_not_null("name");
+ ASSERT_NE(filter, nullptr);
+
+ auto present = createSegments("john", "25", "engineering");
+ EXPECT_TRUE(filter->satisfyRow(0, present));
+ cleanupSegments(present);
+
+ std::vector<std::string*> null_seg = {nullptr, nullptr};
+ EXPECT_FALSE(filter->satisfyRow(0, null_seg));
+
+ // An omitted trailing tag is null, so IS NOT NULL is false.
+ auto trailing = builder_->is_not_null("score");
+ std::vector<std::string*> short_seg = {nullptr, new std::string("john")};
+ EXPECT_FALSE(trailing->satisfyRow(0, short_seg));
+ delete short_seg[1];
+
+ delete filter;
+ delete trailing;
}
\ No newline at end of file
diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py
index f79a6d466..c25ed76f3 100644
--- a/python/tests/test_tsfile_dataset.py
+++ b/python/tests/test_tsfile_dataset.py
@@ -290,6 +290,74 @@ def test_dataset_loc_aligns_timestamp_union_and_preserves_requested_order(tmp_pa
assert aligned.values[2, 1] == 30.0
+def test_dataset_reads_nullable_tag_devices_in_isolation(tmp_path):
+ path = tmp_path / "nullable_tags.tsfile"
+ schema = TableSchema(
+ "sensors",
+ [
+ ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG),
+ ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG),
+ ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD),
+ ],
+ )
+ # Non-trailing null: region IS NULL, device='alpha'.
+ null_region = pd.DataFrame(
+ {
+ "time": [0, 1, 2],
+ "region": [None, None, None],
+ "device": ["alpha", "alpha", "alpha"],
+ "temperature": [10.0, 11.0, 12.0],
+ }
+ )
+ # Trailing null: region='north', device IS NULL. Shares the region prefix
+ # with the fully specified device below to exercise device isolation.
+ null_device = pd.DataFrame(
+ {
+ "time": [0, 1, 2],
+ "region": ["north", "north", "north"],
+ "device": [None, None, None],
+ "temperature": [20.0, 21.0, 22.0],
+ }
+ )
+ full = pd.DataFrame(
+ {
+ "time": [0, 1, 2],
+ "region": ["north", "north", "north"],
+ "device": ["beta", "beta", "beta"],
+ "temperature": [30.0, 31.0, 32.0],
+ }
+ )
+ with TsFileTableWriter(str(path), schema) as writer:
+ writer.write_dataframe(null_region)
+ writer.write_dataframe(null_device)
+ writer.write_dataframe(full)
+
+ with TsFileDataFrame(str(path), show_progress=False) as tsdf:
+ series = tsdf.list_timeseries()
+ # Null tags are dropped from the compressed logical path.
+ assert set(series) == {
+ "sensors.alpha.temperature",
+ "sensors.north.temperature",
+ "sensors.north.beta.temperature",
+ }
+
+ ordered = sorted(series)
+ aligned = tsdf.loc[:, ordered]
+ by_name = dict(zip(aligned.series_names, aligned.values.T))
+
+ # Non-trailing null device reads its own data (previously crashed / NaN).
+ np.testing.assert_array_equal(
+ by_name["sensors.alpha.temperature"], np.array([10.0, 11.0, 12.0])
+ )
+ # Trailing-null device must NOT merge with the fully specified north.beta.
+ np.testing.assert_array_equal(
+ by_name["sensors.north.temperature"], np.array([20.0, 21.0, 22.0])
+ )
+ np.testing.assert_array_equal(
+ by_name["sensors.north.beta.temperature"], np.array([30.0, 31.0, 32.0])
+ )
+
+
def test_dataset_loc_supports_single_timestamp_and_mixed_series_specifiers(tmp_path):
path = tmp_path / "weather.tsfile"
_write_weather_file(path, 0)
@@ -826,24 +894,60 @@ def test_reader_metadata_tag_values_trim_trailing_none():
assert TsFileSeriesReader._metadata_tag_values(_Group(), 1) == ("device_a",)
-def test_exact_tag_filter_rejects_none_tag_values():
- with pytest.raises(NotImplementedError, match="IS NULL / IS NOT NULL"):
- _build_exact_tag_filter({"device": None})
- with pytest.raises(NotImplementedError, match="IS NULL / IS NOT NULL"):
- _build_exact_tag_filter({"city": "beijing", "device": None})
+def test_exact_tag_filter_uses_is_null_for_none_tag_values():
+ from tsfile.tag_filter import AndTagFilter, ComparisonTagFilter
+
+ only_null = _build_exact_tag_filter({"device": None})
+ assert isinstance(only_null, ComparisonTagFilter)
+ assert only_null.op == ComparisonTagFilter.IS_NULL
+ assert only_null.column_name == "device"
+
+ mixed = _build_exact_tag_filter({"city": "beijing", "device": None})
+ assert isinstance(mixed, AndTagFilter)
+ assert isinstance(mixed.left, ComparisonTagFilter)
+ assert mixed.left.op == ComparisonTagFilter.EQ
+ assert mixed.left.value == "beijing"
+ assert isinstance(mixed.right, ComparisonTagFilter)
+ assert mixed.right.op == ComparisonTagFilter.IS_NULL
+ assert mixed.right.column_name == "device"
+
+
+def _tag_filter_has_is_null(tag_filter) -> bool:
+ from tsfile.tag_filter import ComparisonTagFilter
+
+ if isinstance(tag_filter, ComparisonTagFilter):
+ return tag_filter.op == ComparisonTagFilter.IS_NULL
+ for attr in ("left", "right", "filter"):
+ child = getattr(tag_filter, attr, None)
+ if child is not None and _tag_filter_has_is_null(child):
+ return True
+ return False
+
+
+def test_reader_exact_match_with_none_tag_values_issues_is_null_query():
+ captured = {}
+ class _EmptyResultSet:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ return False
+
+ def read_arrow_batch(self):
+ return None
+
+ def next(self):
+ return False
-def test_reader_exact_match_with_none_tag_values_fails_fast():
class _FakeNativeReader:
def query_table(self, *args, **kwargs):
- raise AssertionError(
- "query should not be issued when None-tag exact matching is unsupported"
- )
+ captured["table"] = kwargs.get("tag_filter")
+ return _EmptyResultSet()
def query_table_by_row(self, *args, **kwargs):
- raise AssertionError(
- "row query should not be issued when None-tag exact matching is unsupported"
- )
+ captured["row"] = kwargs.get("tag_filter")
+ return _EmptyResultSet()
reader = object.__new__(TsFileSeriesReader)
reader._reader = _FakeNativeReader()
@@ -864,10 +968,13 @@ def test_reader_exact_match_with_none_tag_values_fails_fast():
"timeline_max_time": 1,
}
- with pytest.raises(NotImplementedError, match="IS NULL / IS NOT NULL"):
- reader.read_series_by_ref(device_id, 0, 0, 1)
- with pytest.raises(NotImplementedError, match="IS NULL / IS NOT NULL"):
- reader.read_series_by_row(device_id, 0, 0, 2)
+ # Both read paths now issue a native query that encodes the null tag as
+ # IS NULL instead of failing fast.
+ reader.read_series_by_ref(device_id, 0, 0, 1)
+ reader.read_series_by_row(device_id, 0, 0, 2)
+
+ assert _tag_filter_has_is_null(captured["table"])
+ assert _tag_filter_has_is_null(captured["row"])
def test_dataframe_resolves_named_sparse_tag_series_path():
diff --git a/python/tsfile/__init__.py b/python/tsfile/__init__.py
index ac8b6b853..783d9ef46 100644
--- a/python/tsfile/__init__.py
+++ b/python/tsfile/__init__.py
@@ -81,6 +81,8 @@ from .tag_filter import (
tag_gteq,
tag_regexp,
tag_not_regexp,
+ tag_is_null,
+ tag_is_not_null,
tag_between,
tag_not_between,
)
diff --git a/python/tsfile/dataset/reader.py b/python/tsfile/dataset/reader.py
index 4899b2bf9..831926324 100644
--- a/python/tsfile/dataset/reader.py
+++ b/python/tsfile/dataset/reader.py
@@ -25,7 +25,7 @@ from typing import Dict, Iterator, List, Tuple
import numpy as np
from ..constants import ColumnCategory, TSDataType
-from ..tag_filter import tag_eq
+from ..tag_filter import tag_eq, tag_is_null
from ..tsfile_reader import TsFileReaderPy
from .metadata import (
MetadataCatalog,
@@ -48,23 +48,36 @@ def _to_python_scalar(value):
return value.item() if hasattr(value, "item") else value
-def _ensure_supported_exact_tag_values(tag_values: Dict[str, object]) -> None:
- if any(tag_value is None for tag_value in tag_values.values()):
- raise NotImplementedError(
- "Exact tag matching with None tag values is not supported yet. "
- "Native tag filter support for IS NULL / IS NOT NULL is required."
- )
-
-
def _build_exact_tag_filter(tag_values: Dict[str, object]):
- _ensure_supported_exact_tag_values(tag_values)
+ """Build a conjunctive filter that isolates exactly one device.
+
+ A ``None`` tag value matches the device's null/missing tag via IS NULL so
+ that devices sharing the same non-null tags (for example a trailing-null
+ device versus a fully specified one) are not conflated.
+ """
tag_filter = None
for tag_column, tag_value in tag_values.items():
- expr = tag_eq(tag_column, str(tag_value))
+ if tag_value is None:
+ expr = tag_is_null(tag_column)
+ else:
+ expr = tag_eq(tag_column, str(tag_value))
tag_filter = expr if tag_filter is None else tag_filter & expr
return tag_filter
+def _device_exact_tag_values(table_entry, device_entry) -> Dict[str, object]:
+ """Map every declared tag column to this device's value (None when null/missing).
+
+ ``device_entry.tag_values`` drops trailing null tags, so columns beyond its
+ length are treated as null rather than omitted from the exact-match filter.
+ """
+ device_tag_values = device_entry.tag_values
+ return {
+ column: device_tag_values[idx] if idx < len(device_tag_values) else None
+ for idx, column in enumerate(table_entry.tag_columns)
+ }
+
+
class TsFileSeriesReader:
"""Wrap ``TsFileReaderPy`` with numeric dataset discovery and batch reads."""
@@ -362,7 +375,7 @@ class TsFileSeriesReader:
table_entry, device_entry, field_name = self._resolve_series_ref(
device_id, field_idx
)
- tag_values = dict(zip(table_entry.tag_columns, device_entry.tag_values))
+ tag_values = _device_exact_tag_values(table_entry, device_entry)
tag_filter = _build_exact_tag_filter(tag_values) if tag_values else None
# Some native row-query paths stop at an internal block boundary even
@@ -416,7 +429,7 @@ class TsFileSeriesReader:
table_entry.table_name,
requested_field_columns,
table_entry.tag_columns,
- dict(zip(table_entry.tag_columns, device_entry.tag_values)),
+ _device_exact_tag_values(table_entry, device_entry),
start_time,
end_time,
)
diff --git a/python/tsfile/tag_filter.py b/python/tsfile/tag_filter.py
index a40c0c47c..ac6e46c5c 100644
--- a/python/tsfile/tag_filter.py
+++ b/python/tsfile/tag_filter.py
@@ -42,10 +42,17 @@ class ComparisonTagFilter(TagFilter):
GTEQ = 5
REGEXP = 6
NOT_REGEXP = 7
+ IS_NULL = 8
+ IS_NOT_NULL = 9
+
+ # Operators that take no comparison value.
+ _NO_VALUE_OPS = (IS_NULL, IS_NOT_NULL)
def __init__(self, column_name: str, value: str, op: int):
self.column_name = column_name
- self.value = value
+ # IS NULL / IS NOT NULL carry no value; the native layer ignores it but
+ # still expects a (possibly empty) string.
+ self.value = "" if value is None else value
self.op = op
def __repr__(self):
@@ -58,7 +65,11 @@ class ComparisonTagFilter(TagFilter):
5: ">=",
6: "=~",
7: "!~",
+ 8: "IS NULL",
+ 9: "IS NOT NULL",
}
+ if self.op in self._NO_VALUE_OPS:
+ return f"TagFilter({self.column_name} {op_names.get(self.op, '?')})"
return (
f"TagFilter({self.column_name} {op_names.get(self.op, '?')} {self.value!r})"
)
@@ -151,6 +162,16 @@ def tag_not_regexp(column_name: str, pattern: str) -> TagFilter:
return ComparisonTagFilter(column_name, pattern, ComparisonTagFilter.NOT_REGEXP)
+def tag_is_null(column_name: str) -> TagFilter:
+ """Create a tag IS NULL filter: the device has no value for this tag column."""
+ return ComparisonTagFilter(column_name, "", ComparisonTagFilter.IS_NULL)
+
+
+def tag_is_not_null(column_name: str) -> TagFilter:
+ """Create a tag IS NOT NULL filter: the device has a value for this tag column."""
+ return ComparisonTagFilter(column_name, "", ComparisonTagFilter.IS_NOT_NULL)
+
+
def tag_between(column_name: str, lower: str, upper: str) -> TagFilter:
"""Create a tag BETWEEN filter: lower <= column <= upper."""
return BetweenTagFilter(column_name, lower, upper, is_not=False)
diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd
index 0fa570df2..4e90dd483 100644
--- a/python/tsfile/tsfile_cpp.pxd
+++ b/python/tsfile/tsfile_cpp.pxd
@@ -324,6 +324,8 @@ cdef extern from "cwrapper/tsfile_cwrapper.h":
TAG_FILTER_GTEQ = 5,
TAG_FILTER_REGEXP = 6,
TAG_FILTER_NOT_REGEXP = 7,
+ TAG_FILTER_IS_NULL = 8,
+ TAG_FILTER_IS_NOT_NULL = 9,
TagFilterHandle tsfile_tag_filter_create(TsFileReader reader,
const char* table_name,