You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/05/31 09:44:55 UTC
(tsfile) 02/03: add per-dataset manifest cache + LRU fd pool for TsFileDataFrame
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch colin_tsfile_dataframe_init
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit cd90641429051eee5221cdd87247397fbdd5defa
Author: ColinLee <sh...@163.com>
AuthorDate: Sun May 31 17:26:38 2026 +0800
add per-dataset manifest cache + LRU fd pool for TsFileDataFrame
Two related changes for fast repeated init over the same shards:
1. Per-dataset JSON manifest (cache_version=2) at
<common parent>/tsfile_dataset.metacache.json, keyed by absolute
shard path with each entry carrying its own (size, mtime_ns)
fingerprint. Hits inject MetadataCatalog into the reader and skip
the heavy get_all_table_schemas + get_timeseries_metadata native
path. Adds cache="auto|off|rebuild" to TsFileDataFrame.
2. LRU fd pool with max_open_files=1024 (default). Readers are still
opened eagerly at construction (fail-fast), but live C++ handles
beyond the cap are closed via _close_native(); Python wrapper and
catalog stay valid and the next read transparently reopens through
_ensure_open().
Existing per-shard sidecar API removed.
---
python/tests/test_dataset_cache.py | 177 +++++++++++++++-----
python/tests/test_tsfile_dataset.py | 6 +-
python/tsfile/dataset/README.md | 312 +++++++++++++++++++++++-------------
python/tsfile/dataset/cache.py | 133 +++++++++------
python/tsfile/dataset/dataframe.py | 71 +++++---
python/tsfile/dataset/pool.py | 75 +++++++++
python/tsfile/dataset/reader.py | 29 +++-
7 files changed, 576 insertions(+), 227 deletions(-)
diff --git a/python/tests/test_dataset_cache.py b/python/tests/test_dataset_cache.py
index 0bae42eda..82f4b3091 100644
--- a/python/tests/test_dataset_cache.py
+++ b/python/tests/test_dataset_cache.py
@@ -16,6 +16,7 @@
# under the License.
#
+import json
import os
import pandas as pd
@@ -31,11 +32,11 @@ from tsfile import (
)
from tsfile.dataset import reader as reader_module
from tsfile.dataset.cache import (
- cache_path,
catalog_from_dict,
catalog_to_dict,
- read_sidecar,
- write_sidecar,
+ extract_catalog,
+ manifest_path,
+ read_manifest,
)
@@ -67,8 +68,7 @@ def test_catalog_round_trip(tmp_path):
reader = next(iter(df._readers.values()))
catalog = reader.catalog
- data = catalog_to_dict(catalog)
- rebuilt = catalog_from_dict(data)
+ rebuilt = catalog_from_dict(catalog_to_dict(catalog))
assert [t.table_name for t in rebuilt.table_entries] == [
t.table_name for t in catalog.table_entries
@@ -93,16 +93,25 @@ def test_catalog_round_trip(tmp_path):
df.close()
+def test_manifest_path_uses_common_parent(tmp_path):
+ paths = [str(tmp_path / f"shard_{i}.tsfile") for i in range(3)]
+ assert manifest_path(paths) == str(tmp_path / "tsfile_dataset.metacache.json")
+
+
+def test_manifest_path_falls_back_for_single_file(tmp_path):
+ path = tmp_path / "single.tsfile"
+ _write_weather_file(path)
+ expected = str(tmp_path / "tsfile_dataset.metacache.json")
+ assert manifest_path([str(path)]) == expected
+
+
def test_cache_auto_hit_skips_native_metadata(tmp_path, monkeypatch):
path = tmp_path / "cache_hit.tsfile"
_write_weather_file(path)
- # First load: cold cache — populates the sidecar.
- df_cold = TsFileDataFrame(str(path), show_progress=False, cache="auto")
- df_cold.close()
- assert os.path.exists(cache_path(str(path)))
+ TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
+ assert os.path.exists(manifest_path([str(path)]))
- # Spy on the heavy native metadata fetch. A cache hit must not call it.
calls = {"count": 0}
original = reader_module.TsFileSeriesReader._cache_metadata_table_model
@@ -117,7 +126,6 @@ def test_cache_auto_hit_skips_native_metadata(tmp_path, monkeypatch):
df_hot = TsFileDataFrame(str(path), show_progress=False, cache="auto")
try:
assert calls["count"] == 0
- # The cached dataframe is functionally equivalent.
assert df_hot.list_timeseries() == [
"weather.device_a.temperature",
"weather.device_a.humidity",
@@ -131,12 +139,12 @@ def test_cache_invalidated_by_mtime_change(tmp_path):
_write_weather_file(path)
TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
- assert read_sidecar(str(path)) is not None
+ shards = read_manifest(manifest_path([str(path)]))
+ assert extract_catalog(shards, str(path)) is not None
- # Bump mtime so the cached fingerprint no longer matches.
stat = os.stat(str(path))
os.utime(str(path), ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))
- assert read_sidecar(str(path)) is None
+ assert extract_catalog(shards, str(path)) is None
def test_cache_invalidated_by_size_change(tmp_path):
@@ -144,46 +152,36 @@ def test_cache_invalidated_by_size_change(tmp_path):
_write_weather_file(path)
TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
- assert read_sidecar(str(path)) is not None
-
- # Pretend the sidecar was written for a different file size, preserving mtime.
- stat_before = os.stat(str(path))
- import json
-
- cp = cache_path(str(path))
- with open(cp, "r") as fh:
+ mp = manifest_path([str(path)])
+ with open(mp, "r") as fh:
payload = json.load(fh)
- payload["file"]["size"] = stat_before.st_size + 1
- with open(cp, "w") as fh:
+ payload["shards"][str(path)]["size"] += 1
+ with open(mp, "w") as fh:
json.dump(payload, fh)
- assert read_sidecar(str(path)) is None
+ assert extract_catalog(read_manifest(mp), str(path)) is None
def test_cache_off_does_not_write(tmp_path):
path = tmp_path / "cache_off.tsfile"
_write_weather_file(path)
-
df = TsFileDataFrame(str(path), show_progress=False, cache="off")
df.close()
-
- assert not os.path.exists(cache_path(str(path)))
+ assert not os.path.exists(manifest_path([str(path)]))
def test_cache_rebuild_overwrites_stale(tmp_path):
path = tmp_path / "rebuild.tsfile"
_write_weather_file(path)
-
- # Seed a corrupt sidecar to prove rebuild does not read it.
- with open(cache_path(str(path)), "w") as fh:
+ mp = manifest_path([str(path)])
+ with open(mp, "w") as fh:
fh.write("not json")
df = TsFileDataFrame(str(path), show_progress=False, cache="rebuild")
try:
- # After rebuild the sidecar must be valid again.
- rebuilt = read_sidecar(str(path))
- assert rebuilt is not None
- # And functionally the dataframe works.
+ shards = read_manifest(mp)
+ assert shards is not None
+ assert extract_catalog(shards, str(path)) is not None
assert df.list_timeseries()
finally:
df.close()
@@ -196,18 +194,115 @@ def test_invalid_cache_mode_rejected(tmp_path):
TsFileDataFrame(str(path), show_progress=False, cache="bogus")
-def test_multi_shard_cache(tmp_path):
+def test_invalid_max_open_files_rejected(tmp_path):
+ path = tmp_path / "bad_cap.tsfile"
+ _write_weather_file(path)
+ with pytest.raises(ValueError):
+ TsFileDataFrame(str(path), show_progress=False, max_open_files=0)
+
+
+def test_single_manifest_for_multiple_shards(tmp_path):
paths = [tmp_path / f"shard_{i}.tsfile" for i in range(3)]
for i, p in enumerate(paths):
_write_weather_file(p, start=i * 100)
+ str_paths = [str(p) for p in paths]
+ mp = manifest_path(str_paths)
- df = TsFileDataFrame([str(p) for p in paths], show_progress=False, cache="auto")
+ df = TsFileDataFrame(str_paths, show_progress=False, cache="auto")
df.close()
+ assert os.path.exists(mp)
+ shards = read_manifest(mp)
+ assert set(shards.keys()) == set(str_paths)
for p in paths:
- assert os.path.exists(cache_path(str(p)))
+ assert not os.path.exists(str(p) + ".metacache.json")
+
+
+def test_adding_shard_keeps_existing_entries(tmp_path):
+ initial = [tmp_path / "shard_0.tsfile", tmp_path / "shard_1.tsfile"]
+ for i, p in enumerate(initial):
+ _write_weather_file(p, start=i * 100)
+ initial_paths = [str(p) for p in initial]
+ TsFileDataFrame(initial_paths, show_progress=False, cache="auto").close()
+
+ new_shard = tmp_path / "shard_2.tsfile"
+ _write_weather_file(new_shard, start=200)
+ expanded_paths = initial_paths + [str(new_shard)]
+ TsFileDataFrame(expanded_paths, show_progress=False, cache="auto").close()
+
+ shards = read_manifest(manifest_path(expanded_paths))
+ assert set(shards.keys()) == set(expanded_paths)
+
+
+def test_manifest_skipped_on_full_hit(tmp_path):
+ path = tmp_path / "no_rewrite.tsfile"
+ _write_weather_file(path)
+
+ TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
+ mp = manifest_path([str(path)])
+ before = os.stat(mp).st_mtime_ns
+
+ TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
+ after = os.stat(mp).st_mtime_ns
+ assert before == after
+
+
+def test_lru_evicts_older_reader_when_over_cap(tmp_path):
+ paths = [tmp_path / f"lru_{i}.tsfile" for i in range(3)]
+ for i, p in enumerate(paths):
+ _write_weather_file(p, start=i * 100)
+ str_paths = [str(p) for p in paths]
- df_hot = TsFileDataFrame([str(p) for p in paths], show_progress=False, cache="auto")
+ df = TsFileDataFrame(str_paths, show_progress=False, cache="auto", max_open_files=2)
try:
- assert len(df_hot.list_timeseries()) == 2
+ # After eager init with cap=2, the LRU shard's native handle is closed
+ # but its Python wrapper + catalog stay valid.
+ first = df._readers[str_paths[0]]
+ second = df._readers[str_paths[1]]
+ third = df._readers[str_paths[2]]
+
+ assert first._reader is None
+ assert second._reader is not None
+ assert third._reader is not None
+ assert df._fd_pool.capacity == 2
+ assert len(df._fd_pool) == 2
+
+ # Touching the evicted reader reopens its native handle and pushes the
+ # current LRU (`second`) out.
+ first._ensure_open()
+
+ assert first._reader is not None
+ assert third._reader is not None
+ assert second._reader is None
+ assert len(df._fd_pool) == 2
finally:
- df_hot.close()
+ df.close()
+
+
+def test_lru_disabled_when_cap_exceeds_shard_count(tmp_path):
+ paths = [tmp_path / f"all_open_{i}.tsfile" for i in range(3)]
+ for i, p in enumerate(paths):
+ _write_weather_file(p, start=i * 100)
+
+ df = TsFileDataFrame(
+ [str(p) for p in paths],
+ show_progress=False,
+ cache="auto",
+ max_open_files=16,
+ )
+ try:
+ for p in paths:
+ assert df._readers[str(p)]._reader is not None
+ assert len(df._fd_pool) == 3
+ finally:
+ df.close()
+
+
+def test_lru_close_all_drains_pool(tmp_path):
+ path = tmp_path / "drain.tsfile"
+ _write_weather_file(path)
+
+ df = TsFileDataFrame(str(path), show_progress=False, cache="auto")
+ reader = next(iter(df._readers.values()))
+ assert reader._reader is not None
+ df.close()
+ assert reader._reader is None
diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py
index 74ad86952..4e52a1b5f 100644
--- a/python/tests/test_tsfile_dataset.py
+++ b/python/tests/test_tsfile_dataset.py
@@ -697,8 +697,10 @@ def test_reader_read_series_by_row_retries_across_native_row_query_boundaries():
class _FakeResultSet:
def __init__(self, times, values):
self._batch = pa.table(
- {"time": pa.array(times, type=pa.int64()),
- "totalcloudcover": pa.array(values, type=pa.float64())}
+ {
+ "time": pa.array(times, type=pa.int64()),
+ "totalcloudcover": pa.array(values, type=pa.float64()),
+ }
)
self._delivered = False
diff --git a/python/tsfile/dataset/README.md b/python/tsfile/dataset/README.md
index 825d2c6bf..4a14b6eb2 100644
--- a/python/tsfile/dataset/README.md
+++ b/python/tsfile/dataset/README.md
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
-# `TsFileDataFrame` Sidecar Metadata Cache
+# `TsFileDataFrame` Dataset Metadata Manifest
`TsFileDataFrame` opens one C++ reader per shard during construction. Opening
the reader itself is cheap (one `open(2)` syscall); the expensive part is
@@ -27,32 +27,42 @@ converting the result into Python objects. Repeating that work on every
process start is wasteful when the underlying `.tsfile` files have not
changed.
-To skip that work, every shard gets a JSON sidecar file next to it that
-records the catalog produced by the previous load. On subsequent loads the
-sidecar is validated against the live file's `stat()` fingerprint and, if it
-still matches, the catalog is rehydrated directly from JSON. The C++ reader
-is still opened — that cost is negligible — but `get_all_table_schemas` and
-`get_timeseries_metadata(None)` are skipped entirely.
+To skip that work, every dataset gets **one** JSON manifest file that
+records the catalog of every shard the dataframe has seen. On subsequent
+loads the manifest is consulted per shard: entries whose `(size, mtime_ns)`
+still match are rehydrated directly from JSON; entries that miss (modified,
+removed, or never recorded) trigger the normal native metadata load and are
+then merged back into the manifest.
+
+The C++ reader is still opened for every shard — that cost is negligible —
+but `get_all_table_schemas` and `get_timeseries_metadata(None)` are skipped
+on cache hits.
## File layout
-The sidecar lives next to the shard and reuses its full name plus a fixed
-suffix:
+One manifest covers an entire dataset. It sits in the common parent
+directory of all the shard paths passed to `TsFileDataFrame`:
```
-data/2024-01-01.tsfile
-data/2024-01-01.tsfile.metacache.json
+data/
+├── part_0.tsfile
+├── part_1.tsfile
+├── part_2.tsfile
+├── part_3.tsfile
+└── tsfile_dataset.metacache.json ← the manifest
```
-Defined in `cache.py`:
+When the shards span multiple subdirectories, the manifest lives in their
+deepest common ancestor. When the inputs share no common path (rare cross-
+filesystem case), the manifest falls back to the first shard's directory so
+there is always a writable target.
-| Constant | Value |
-|--------------------------|----------------------|
-| `_CACHE_SUFFIX` | `".metacache.json"` |
-| `_CACHE_FORMAT_VERSION` | `1` |
+Defined in `cache.py`:
-One shard, one sidecar. There is no cross-shard manifest: each sidecar is
-independent and can be deleted or rebuilt in isolation.
+| Constant | Value |
+|-----------------------|--------------------------------------|
+| `_MANIFEST_FILENAME` | `"tsfile_dataset.metacache.json"` |
+| `_MANIFEST_VERSION` | `2` |
## JSON schema
@@ -60,29 +70,37 @@ The on-disk payload is a single JSON object:
```json
{
- "cache_version": 1,
- "file": {
- "size": 1234567,
- "mtime_ns": 1717050000000000000
- },
- "catalog": {
- "tables": [ ... ],
- "devices": [ ... ],
- "series_stats": [ ... ]
+ "cache_version": 2,
+ "shards": {
+ "/abs/path/data/part_0.tsfile": {
+ "size": 1234567,
+ "mtime_ns": 1717050000000000000,
+ "catalog": { "tables": [...], "devices": [...], "series_stats": [...] }
+ },
+ "/abs/path/data/part_1.tsfile": { ... }
}
}
```
### Top-level fields
-| Field | Type | Purpose |
-|-----------------|--------|--------------------------------------------------------------------------------------|
-| `cache_version` | int | Sidecar schema version. Mismatch → invalidate. |
-| `file.size` | int | `os.stat(tsfile).st_size` at the time the sidecar was written. Mismatch → invalidate.|
-| `file.mtime_ns` | int | `os.stat(tsfile).st_mtime_ns` at the time the sidecar was written. Mismatch → invalidate. |
-| `catalog` | object | Serialized form of one shard's `MetadataCatalog`. |
+| Field | Type | Purpose |
+|-----------------|---------------------|----------------------------------------------------------------|
+| `cache_version` | int | Manifest schema version. Mismatch → invalidate the entire manifest. |
+| `shards` | object | Map from each shard's absolute path to its cache entry. |
+
+### `shards[<abs_path>]`
-### `catalog.tables[]`
+Each shard entry is independent. A stale fingerprint on one shard only
+invalidates that shard; the other shards still hit.
+
+| Field | Type | Notes |
+|-------------|--------|------------------------------------------------------------------------------------------------|
+| `size` | int | `os.stat(shard).st_size` at the time the entry was written. Mismatch → that shard misses. |
+| `mtime_ns` | int | `os.stat(shard).st_mtime_ns` at the time the entry was written. Mismatch → that shard misses. |
+| `catalog` | object | Serialized form of the shard's `MetadataCatalog`. See below. |
+
+### `shards[<abs_path>].catalog.tables[]`
Each entry mirrors a `TableEntry`:
@@ -96,38 +114,38 @@ Each entry mirrors a `TableEntry`:
The position of a table in this array is its `table_index`, referenced by
`devices[].table_index` below.
-### `catalog.devices[]`
+### `shards[<abs_path>].catalog.devices[]`
Each entry mirrors a `DeviceEntry`:
| Field | Type | Notes |
|---------------|---------|----------------------------------------------------------------------------------------------|
-| `table_index` | int | Index into `catalog.tables[]`. |
+| `table_index` | int | Index into the same shard's `catalog.tables[]`. |
| `tag_values` | any[] | Tag values in the order of the table's `tag_columns`. Types follow the table's `tag_types`. |
| `min_time` | int | Minimum timestamp observed across the device's contributing measurements. |
| `max_time` | int | Maximum timestamp observed across the device's contributing measurements. |
Tag values are stored using their natural JSON representation:
-| `TSDataType` | JSON form |
-|-------------------------|-----------|
-| `BOOLEAN` | bool |
-| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | int |
-| `FLOAT`, `DOUBLE` | number |
-| `TEXT`, `STRING`, `BLOB`| string |
-| (missing trailing tag) | `null` |
+| `TSDataType` | JSON form |
+|-------------------------------------|-----------|
+| `BOOLEAN` | bool |
+| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | int |
+| `FLOAT`, `DOUBLE` | number |
+| `TEXT`, `STRING`, `BLOB` | string |
+| (missing trailing tag) | `null` |
The position of a device in this array is its `device_index`, referenced by
`series_stats[].device_index` below.
-### `catalog.series_stats[]`
+### `shards[<abs_path>].catalog.series_stats[]`
Each entry mirrors one `(device_id, field_idx) → stats` entry in
`MetadataCatalog.series_stats_by_ref`:
| Field | Type | Notes |
|----------------------|--------------|------------------------------------------------------------------------------------|
-| `device_index` | int | Index into `catalog.devices[]`. |
+| `device_index` | int | Index into the same shard's `catalog.devices[]`. |
| `field_index` | int | Index into the device's table's `field_columns`. |
| `length` | int | Row count derived from the value column's statistic. |
| `min_time` | int \| null | Minimum timestamp from the value column statistic. `null` when no statistic. |
@@ -141,84 +159,150 @@ present so that `(device_index, field_index)` resolution always succeeds.
## Validation rules
-When `cache_from_dict` and `read_sidecar` decide whether to trust a sidecar,
-the checks below run in order. Any failure returns `None` and the dataframe
-falls back to a full native metadata load.
+Validation happens at two levels.
+
+**Manifest-level**, by `read_manifest`:
-1. The shard's `os.stat()` succeeds.
-2. The sidecar exists and parses as JSON.
-3. The top-level value is an object.
-4. `cache_version` equals `_CACHE_FORMAT_VERSION` (currently `1`).
-5. `file.size` equals the shard's current `st_size`.
-6. `file.mtime_ns` equals the shard's current `st_mtime_ns`.
-7. `catalog_from_dict` succeeds on `catalog` (well-formed tables, devices,
- and stats).
+1. The manifest file exists and parses as JSON.
+2. The top-level value is an object.
+3. `cache_version` equals `_MANIFEST_VERSION` (currently `2`).
+4. `shards` is an object.
+
+A manifest-level failure throws away the entire cache; every shard becomes
+a miss and the manifest is rewritten from scratch.
+
+**Per-shard**, by `extract_catalog`:
+
+1. The shard has an entry in `shards`.
+2. `os.stat(shard)` succeeds.
+3. The entry's `size` equals the live `st_size`.
+4. The entry's `mtime_ns` equals the live `st_mtime_ns`.
+5. `catalog_from_dict` succeeds on the entry's `catalog`.
+
+A per-shard failure only invalidates that one shard; other shards in the
+same manifest continue to hit.
## Write semantics
-`write_sidecar` writes through a temporary file in the same directory and
-calls `os.replace` for an atomic swap. The `size` and `mtime_ns` recorded in
-the payload come from a fresh `os.stat()` taken immediately before writing.
+After a load, `write_manifest` is called only if at least one shard missed
+(or the user passed `cache="rebuild"`). It does:
+
+1. Re-read the existing manifest from disk so concurrent writers' new
+ entries are not lost.
+2. Overwrite the entries for the shards we just loaded with fresh
+ `(size, mtime_ns, catalog)` triples.
+3. Write through a temporary file in the same directory and `os.replace`
+ atomically.
+
+Entries for shards not in the current dataframe are preserved verbatim:
+loading a subset of a dataset does not prune the others.
-All disk-level errors (permission denied, read-only filesystem, no space) are
-swallowed silently; the dataframe is still constructed successfully, the
-sidecar is simply absent and the next load pays the full metadata cost.
+All disk-level errors (permission denied, read-only filesystem, no space)
+are swallowed silently; the dataframe is still constructed successfully,
+the manifest is simply not updated.
+
+When every shard hits, the manifest is **not** rewritten — pure-hit loads
+do not touch the filesystem.
## Cache modes
`TsFileDataFrame(paths, cache=...)` accepts:
-| Mode | Read sidecar? | Write sidecar after load? |
-|-----------|---------------|---------------------------|
-| `"auto"` (default) | yes | yes, only when the read missed |
-| `"off"` | no | no |
-| `"rebuild"` | no | yes, always (overwrites any existing sidecar) |
+| Mode | Read manifest? | Write manifest after load? |
+|-----------|----------------|-----------------------------|
+| `"auto"` (default) | yes | yes, only if some shard missed |
+| `"off"` | no | no |
+| `"rebuild"` | no | yes — every shard counts as a miss |
+
+`"rebuild"` exists for the case where the manifest is suspected of being
+stale or corrupt despite passing the size/mtime checks — for example after
+a manual edit or an interrupted previous write.
+
+## File descriptor budget (`max_open_files`)
+
+Every shard's `TsFileSeriesReader` opens a C++ reader eagerly during
+construction — failures surface immediately rather than at the first
+query. With many shards this could exhaust the process's fd budget
+(`ulimit -n`), so the dataframe wraps the live readers in an LRU pool.
+
+`TsFileDataFrame(paths, max_open_files=N)`:
+
+| Behavior | When `N >= len(paths)` | When `N < len(paths)` |
+|-------------------------------------------|------------------------|------------------------|
+| Native C++ reader opened at construction? | every shard, all stay | every shard, but only N stay live |
+| Catalog kept in memory? | every shard | every shard |
+| Eviction trigger | never | each new touch pushes the LRU out |
+| Reopen on eviction | n/a | first read on the evicted shard calls `_ensure_open()` |
+
+Default is `1024`. Tune up when a single query routinely touches more
+shards than that — every miss costs one `open(2)` plus a footer reload
+on the C++ side, so a too-small pool causes thrashing. Tune down when
+the host has a tight fd cap and you accept the reopen cost.
-`"rebuild"` exists for the case where the sidecar is suspected of being
-stale or corrupt despite passing the size/mtime checks — for example after a
-manual edit or an interrupted previous write.
+Eviction only closes the native handle. The Python wrapper and its
+catalog (from the manifest or from native metadata) remain valid, so
+nothing about query semantics changes — the next data access on the
+evicted shard reopens transparently.
+
+## Concurrency
+
+Multiple processes loading the same dataset concurrently both end up
+writing the manifest. Writes are atomic (temp file + `os.replace`) and
+content is keyed by absolute shard path, so:
+
+- If they cover the same shards: last writer wins; the manifest is
+ identical either way.
+- If they cover different overlapping shards: each writer re-reads the
+ current manifest before merging, so neither side wipes the other's
+ contributions.
+
+A pathological race (both writers see an empty manifest, then both write
+their disjoint subsets) can drop the earlier writer's entries. The next
+load reproduces them. No explicit lock is taken.
## Worked example
-The smallest possible sidecar — one table, one device, one field, three
-rows:
+The smallest possible manifest — one shard, one table, one device, one
+field, three rows:
```json
{
- "cache_version": 1,
- "file": {
- "size": 4096,
- "mtime_ns": 1717050000000000000
- },
- "catalog": {
- "tables": [
- {
- "name": "weather",
- "tag_columns": ["device"],
- "tag_types": [11],
- "field_columns": ["temperature"]
- }
- ],
- "devices": [
- {
- "table_index": 0,
- "tag_values": ["device_a"],
- "min_time": 0,
- "max_time": 2
- }
- ],
- "series_stats": [
- {
- "device_index": 0,
- "field_index": 0,
- "length": 3,
- "min_time": 0,
- "max_time": 2,
- "timeline_length": 3,
- "timeline_min_time": 0,
- "timeline_max_time": 2
+ "cache_version": 2,
+ "shards": {
+ "/abs/path/data/weather.tsfile": {
+ "size": 4096,
+ "mtime_ns": 1717050000000000000,
+ "catalog": {
+ "tables": [
+ {
+ "name": "weather",
+ "tag_columns": ["device"],
+ "tag_types": [11],
+ "field_columns": ["temperature"]
+ }
+ ],
+ "devices": [
+ {
+ "table_index": 0,
+ "tag_values": ["device_a"],
+ "min_time": 0,
+ "max_time": 2
+ }
+ ],
+ "series_stats": [
+ {
+ "device_index": 0,
+ "field_index": 0,
+ "length": 3,
+ "min_time": 0,
+ "max_time": 2,
+ "timeline_length": 3,
+ "timeline_min_time": 0,
+ "timeline_max_time": 2
+ }
+ ]
}
- ]
+ }
}
}
```
@@ -227,12 +311,12 @@ rows:
## Operational notes
-- The sidecar is **not** a portable export. It encodes positional indices
- into the same shard's footer ordering. Moving a sidecar between hosts is
- fine as long as the corresponding shard file moves with it; copying it to
- a different shard is not supported.
-- Cleaning up sidecars is safe at any time. The next load will pay the
- metadata cost once and rewrite them.
-- Bumping `_CACHE_FORMAT_VERSION` is the migration path when the schema
- needs to change. All older sidecars become invalidated automatically by
- the version check.
+- The manifest is **not** a portable export. Shard keys are absolute
+ paths, and the inner `catalog` encodes positional indices into the
+ same shard's footer. Moving the dataset to a different absolute path
+ invalidates every entry; the next load rebuilds them.
+- Deleting the manifest is safe at any time. The next load pays the full
+ metadata cost once and rewrites it.
+- Bumping `_MANIFEST_VERSION` is the migration path when the schema needs
+ to change. All older manifests are discarded automatically by the
+ version check.
diff --git a/python/tsfile/dataset/cache.py b/python/tsfile/dataset/cache.py
index 40b8d8d58..a2269910d 100644
--- a/python/tsfile/dataset/cache.py
+++ b/python/tsfile/dataset/cache.py
@@ -16,28 +16,43 @@
# under the License.
#
-"""Per-shard JSON sidecar cache for :class:`MetadataCatalog`.
+"""Per-dataset JSON manifest cache for :class:`MetadataCatalog`.
-The dataframe's expensive init step is reading TsFile footer + materializing
-every device's TimeseriesIndex into Python objects. Opening the C++ reader
-itself is just one ``open(2)``. So we cache the materialized catalog next to
-each .tsfile and skip the metadata path on subsequent loads.
+One JSON file describes the catalogs of every shard the dataframe was
+constructed from. The manifest lives in the common parent directory of the
+input paths and is keyed by each shard's absolute path. Each shard entry
+carries its own ``(size, mtime_ns)`` fingerprint so that a single stale
+shard does not invalidate the rest of the manifest.
"""
import json
import os
import tempfile
-from typing import Optional
+from typing import Dict, Iterable, List, Optional, Tuple
from ..constants import TSDataType
from .metadata import MetadataCatalog
-_CACHE_SUFFIX = ".metacache.json"
-_CACHE_FORMAT_VERSION = 1
+_MANIFEST_FILENAME = "tsfile_dataset.metacache.json"
+_MANIFEST_VERSION = 2
-def cache_path(tsfile_path: str) -> str:
- return tsfile_path + _CACHE_SUFFIX
+def manifest_path(paths: List[str]) -> str:
+ """Return the dataset manifest path for a list of absolute shard paths.
+
+ The manifest sits in the common parent directory of all shards. When the
+ paths span unrelated trees (no common parent exists), we fall back to the
+ first path's directory so we always have a writable location.
+ """
+ if not paths:
+ raise ValueError("Cannot compute manifest path for empty paths.")
+ try:
+ common = os.path.commonpath(paths)
+ except ValueError:
+ common = os.path.dirname(paths[0])
+ if os.path.isfile(common):
+ common = os.path.dirname(common)
+ return os.path.join(common, _MANIFEST_FILENAME)
def catalog_to_dict(catalog: MetadataCatalog) -> dict:
@@ -104,55 +119,84 @@ def catalog_from_dict(data: dict) -> MetadataCatalog:
return catalog
-def read_sidecar(tsfile_path: str) -> Optional[MetadataCatalog]:
- """Read and validate the sidecar cache for one shard.
-
- Returns ``None`` for any of: missing/unreadable cache file, schema
- mismatch, size/mtime mismatch, or corrupt JSON.
- """
+def _shard_entry(file_path: str, catalog: MetadataCatalog) -> Optional[dict]:
+ """Build a shard manifest entry by stat'ing the file right before write."""
try:
- tsfile_stat = os.stat(tsfile_path)
+ stat = os.stat(file_path)
except OSError:
return None
+ return {
+ "size": stat.st_size,
+ "mtime_ns": stat.st_mtime_ns,
+ "catalog": catalog_to_dict(catalog),
+ }
+
+
+def _entry_matches_file(entry: dict, file_path: str) -> bool:
+ try:
+ stat = os.stat(file_path)
+ except OSError:
+ return False
+ return (
+ entry.get("size") == stat.st_size and entry.get("mtime_ns") == stat.st_mtime_ns
+ )
+
+
+def read_manifest(path: str) -> Optional[Dict[str, dict]]:
+ """Load and shallow-validate the manifest. Returns the shards dict, or None."""
try:
- with open(cache_path(tsfile_path), "r") as fh:
+ with open(path, "r") as fh:
data = json.load(fh)
except (OSError, json.JSONDecodeError):
return None
if not isinstance(data, dict):
return None
- if data.get("cache_version") != _CACHE_FORMAT_VERSION:
+ if data.get("cache_version") != _MANIFEST_VERSION:
return None
- file_meta = data.get("file") or {}
- if file_meta.get("size") != tsfile_stat.st_size:
+ shards = data.get("shards")
+ if not isinstance(shards, dict):
return None
- if file_meta.get("mtime_ns") != tsfile_stat.st_mtime_ns:
+ return shards
+
+
+def extract_catalog(
+ shards: Dict[str, dict], file_path: str
+) -> Optional[MetadataCatalog]:
+ """Return the cached catalog for one shard iff its fingerprint still matches."""
+ entry = shards.get(file_path)
+ if entry is None or not isinstance(entry, dict):
+ return None
+ if not _entry_matches_file(entry, file_path):
return None
try:
- return catalog_from_dict(data["catalog"])
+ return catalog_from_dict(entry["catalog"])
except (KeyError, ValueError, TypeError):
return None
-def write_sidecar(tsfile_path: str, catalog: MetadataCatalog) -> None:
- """Atomically write the sidecar cache. Disk errors are swallowed."""
- try:
- tsfile_stat = os.stat(tsfile_path)
- except OSError:
- return
- payload = {
- "cache_version": _CACHE_FORMAT_VERSION,
- "file": {
- "size": tsfile_stat.st_size,
- "mtime_ns": tsfile_stat.st_mtime_ns,
- },
- "catalog": catalog_to_dict(catalog),
- }
- target = cache_path(tsfile_path)
- target_dir = os.path.dirname(target) or "."
+def write_manifest(
+ path: str,
+ fresh_catalogs: Iterable[Tuple[str, MetadataCatalog]],
+) -> None:
+ """Atomically merge fresh catalogs into the manifest at ``path``.
+
+ Reads the existing manifest first (if any), overwrites the entries for
+ the shards in ``fresh_catalogs``, and writes the result back via temp
+ file + ``os.replace``. Other shards' entries are preserved. OS errors
+ (read-only filesystem, no space) are swallowed silently.
+ """
+ existing = read_manifest(path) or {}
+ merged = dict(existing)
+ for file_path, catalog in fresh_catalogs:
+ entry = _shard_entry(file_path, catalog)
+ if entry is not None:
+ merged[file_path] = entry
+
+ payload = {"cache_version": _MANIFEST_VERSION, "shards": merged}
+ target_dir = os.path.dirname(path) or "."
try:
fd, tmp_path = tempfile.mkstemp(
- prefix=os.path.basename(target) + ".",
+ prefix=_MANIFEST_FILENAME + ".",
suffix=".tmp",
dir=target_dir,
)
@@ -161,16 +205,9 @@ def write_sidecar(tsfile_path: str, catalog: MetadataCatalog) -> None:
try:
with os.fdopen(fd, "w") as fh:
json.dump(payload, fh)
- os.replace(tmp_path, target)
+ os.replace(tmp_path, path)
except OSError:
try:
os.unlink(tmp_path)
except OSError:
pass
-
-
-def remove_sidecar(tsfile_path: str) -> None:
- try:
- os.unlink(cache_path(tsfile_path))
- except OSError:
- pass
diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py
index c83439d81..ba1d8124a 100644
--- a/python/tsfile/dataset/dataframe.py
+++ b/python/tsfile/dataset/dataframe.py
@@ -28,7 +28,7 @@ import warnings
import numpy as np
-from .cache import read_sidecar, write_sidecar
+from .cache import extract_catalog, manifest_path, read_manifest, write_manifest
from .formatting import format_dataframe_table
from .metadata import (
TableEntry,
@@ -37,9 +37,11 @@ from .metadata import (
split_logical_series_path,
)
from .merge import build_aligned_matrix, merge_time_value_parts, merge_timestamp_parts
+from .pool import _FdPool
from .timeseries import AlignedTimeseries, Timeseries
_CACHE_MODES = ("auto", "off", "rebuild")
+_DEFAULT_MAX_OPEN_FILES = 1024
DeviceKey = Tuple[str, tuple]
SeriesRefKey = Tuple[int, int]
@@ -583,12 +585,16 @@ class TsFileDataFrame:
show_progress: bool = True,
*,
cache: str = "auto",
+ max_open_files: int = _DEFAULT_MAX_OPEN_FILES,
):
if cache not in _CACHE_MODES:
raise ValueError(f"cache must be one of {_CACHE_MODES}, got {cache!r}")
+ if max_open_files < 1:
+ raise ValueError(f"max_open_files must be >= 1, got {max_open_files}")
self._paths = _expand_paths(paths)
self._show_progress = show_progress
self._cache_mode = cache
+ self._fd_pool = _FdPool(max_open_files)
self._readers: Dict[str, object] = {}
self._index = _LogicalIndex()
self._cache = _DerivedCache()
@@ -608,6 +614,7 @@ class TsFileDataFrame:
obj._paths = parent._paths
obj._show_progress = parent._show_progress
obj._cache_mode = parent._cache_mode
+ obj._fd_pool = parent._fd_pool
obj._readers = parent._readers
obj._index = _LogicalIndex(
table_entries=parent._index.table_entries,
@@ -635,10 +642,20 @@ class TsFileDataFrame:
"""Build the logical cross-file index and the derived per-series caches."""
from .reader import TsFileSeriesReader
+ shards: Dict[str, dict] = {}
+ if self._cache_mode == "auto":
+ shards = read_manifest(manifest_path(self._paths)) or {}
+
if len(self._paths) >= 2:
- self._load_metadata_parallel(TsFileSeriesReader)
+ misses = self._load_metadata_parallel(TsFileSeriesReader, shards)
else:
- self._load_metadata_serial(TsFileSeriesReader)
+ misses = self._load_metadata_serial(TsFileSeriesReader, shards)
+
+ if self._cache_mode != "off" and misses:
+ write_manifest(
+ manifest_path(self._paths),
+ [(file_path, self._readers[file_path].catalog) for file_path in misses],
+ )
self._cache.devices = [
_build_device_entry(refs) for refs in self._index.device_refs
@@ -664,49 +681,61 @@ class TsFileDataFrame:
)
sys.stderr.flush()
- def _open_with_cache(self, reader_class, file_path: str, show_progress: bool):
- """Open one shard reader, injecting a sidecar catalog when available.
+ def _open_with_cache(
+ self,
+ reader_class,
+ file_path: str,
+ show_progress: bool,
+ shards: Dict[str, dict],
+ ):
+ """Open one shard reader, injecting a cached catalog when available.
Returns ``(reader, was_cache_miss)``. ``was_cache_miss`` is True when
- the catalog had to be materialized from the file rather than loaded
- from the JSON sidecar.
+ the catalog had to be materialized from the file rather than rehydrated
+ from the dataset manifest. The reader is registered in ``self._fd_pool``
+ so eager opens beyond ``max_open_files`` evict older live handles.
"""
catalog = None
if self._cache_mode == "auto":
- catalog = read_sidecar(file_path)
- reader = reader_class(file_path, show_progress=show_progress, catalog=catalog)
+ catalog = extract_catalog(shards, file_path)
+ reader = reader_class(
+ file_path,
+ show_progress=show_progress,
+ catalog=catalog,
+ pool=self._fd_pool,
+ )
return reader, catalog is None
- def _maybe_write_sidecar(self, file_path: str, reader) -> None:
- if self._cache_mode == "off":
- return
- write_sidecar(file_path, reader.catalog)
-
- def _load_metadata_serial(self, reader_class):
+ def _load_metadata_serial(self, reader_class, shards: Dict[str, dict]):
total = len(self._paths)
self._show_loading_progress(0, total)
+ misses: List[str] = []
for index, file_path in enumerate(self._paths, start=1):
reader, was_miss = self._open_with_cache(
reader_class,
file_path,
self._show_progress and total == 1,
+ shards,
)
_register_reader(self._readers, self._index, file_path, reader)
if was_miss:
- self._maybe_write_sidecar(file_path, reader)
+ misses.append(file_path)
if total > 1:
self._show_loading_progress(index, total)
self._show_loading_progress(
total, total, sum(reader.series_count for reader in self._readers.values())
)
+ return misses
- def _load_metadata_parallel(self, reader_class):
+ def _load_metadata_parallel(self, reader_class, shards: Dict[str, dict]):
from concurrent.futures import ThreadPoolExecutor, as_completed
def open_file(file_path):
- reader, was_miss = self._open_with_cache(reader_class, file_path, False)
+ reader, was_miss = self._open_with_cache(
+ reader_class, file_path, False, shards
+ )
return file_path, reader, was_miss
total = len(self._paths)
@@ -716,7 +745,7 @@ class TsFileDataFrame:
) as executor:
futures = {executor.submit(open_file, path): path for path in self._paths}
results = {}
- misses = []
+ misses: List[str] = []
done = 0
for future in as_completed(futures):
file_path, reader, was_miss = future.result()
@@ -738,8 +767,7 @@ class TsFileDataFrame:
results[file_path],
)
- for file_path in misses:
- self._maybe_write_sidecar(file_path, results[file_path])
+ return misses
def _get_series_components(
self, series_ref: SeriesRefKey
@@ -1029,6 +1057,7 @@ class TsFileDataFrame:
for reader in self._readers.values():
reader.close()
self._readers.clear()
+ self._fd_pool.close_all()
self._closed = True
def __del__(self):
diff --git a/python/tsfile/dataset/pool.py b/python/tsfile/dataset/pool.py
new file mode 100644
index 000000000..106eb95d5
--- /dev/null
+++ b/python/tsfile/dataset/pool.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+"""LRU pool of live C++ TsFile readers.
+
+Per-shard ``TsFileSeriesReader`` instances always carry their materialized
+catalog, but the underlying C++ ``TsFileReaderPy`` (one fd + footer state)
+is expensive enough that we want to bound how many stay live at once.
+
+The pool tracks readers in MRU order. When ``touch`` would push the size
+above ``capacity``, the least-recently-used reader's native handle is
+closed via ``_close_native()``; its Python wrapper and catalog stay valid
+and the next read on that shard will reopen lazily through
+``_ensure_open()``.
+
+This is single-threaded by design — the dataframe's read paths are
+expected to be called from one thread at a time.
+"""
+
+from collections import OrderedDict
+
+
+class _FdPool:
+ def __init__(self, capacity: int):
+ if capacity < 1:
+ raise ValueError(f"max_open_files must be >= 1, got {capacity}")
+ self._capacity = capacity
+ self._order: "OrderedDict[str, object]" = OrderedDict()
+
+ @property
+ def capacity(self) -> int:
+ return self._capacity
+
+ def __len__(self) -> int:
+ return len(self._order)
+
+ def touch(self, reader) -> None:
+ """Mark ``reader`` as most-recently-used. Evict LRU if over capacity."""
+ key = reader.file_path
+ if key in self._order:
+ self._order.move_to_end(key)
+ return
+ self._order[key] = reader
+ while len(self._order) > self._capacity:
+ evict_key, victim = next(iter(self._order.items()))
+ if victim is reader:
+ # Never evict the reader we just inserted.
+ break
+ self._order.pop(evict_key, None)
+ victim._close_native()
+
+ def discard(self, reader) -> None:
+ """Remove ``reader`` from the pool without closing anything."""
+ self._order.pop(reader.file_path, None)
+
+ def close_all(self) -> None:
+ """Close native handles for every reader still in the pool."""
+ for reader in list(self._order.values()):
+ reader._close_native()
+ self._order.clear()
diff --git a/python/tsfile/dataset/reader.py b/python/tsfile/dataset/reader.py
index bcf977831..fa83dff7f 100644
--- a/python/tsfile/dataset/reader.py
+++ b/python/tsfile/dataset/reader.py
@@ -75,12 +75,14 @@ class TsFileSeriesReader:
show_progress: bool = True,
*,
catalog: "MetadataCatalog | None" = None,
+ pool: "object | None" = None,
):
if not os.path.exists(file_path):
raise FileNotFoundError(f"TsFile not found: {file_path}")
self.file_path = file_path
self.show_progress = show_progress
+ self._pool = pool
try:
self._reader = TsFileReaderPy(file_path)
@@ -93,9 +95,29 @@ class TsFileSeriesReader:
self._catalog = MetadataCatalog()
self._cache_metadata()
+ if self._pool is not None:
+ self._pool.touch(self)
+
def __del__(self):
self.close()
+ def _ensure_open(self) -> None:
+ """Reopen the native reader if it was evicted; mark MRU in the pool."""
+ if getattr(self, "_reader", None) is None:
+ self._reader = TsFileReaderPy(self.file_path)
+ pool = getattr(self, "_pool", None)
+ if pool is not None:
+ pool.touch(self)
+
+ def _close_native(self) -> None:
+ """Close the native handle but keep the Python wrapper + catalog usable."""
+ if getattr(self, "_reader", None) is not None:
+ try:
+ self._reader.close()
+ except Exception:
+ pass
+ self._reader = None
+
@property
def catalog(self) -> MetadataCatalog:
return self._catalog
@@ -119,11 +141,14 @@ class TsFileSeriesReader:
), device_id, field_idx
def close(self):
- if hasattr(self, "_reader"):
+ if getattr(self, "_pool", None) is not None:
+ self._pool.discard(self)
+ if getattr(self, "_reader", None) is not None:
try:
self._reader.close()
except Exception:
pass
+ self._reader = None
def _cache_metadata(self):
"""Wrap metadata discovery so reader construction surfaces one stable error shape."""
@@ -368,6 +393,7 @@ class TsFileSeriesReader:
if limit <= 0:
return np.array([], dtype=np.int64), np.array([], dtype=np.float64)
+ self._ensure_open()
table_entry, device_entry, field_name = self._resolve_series_ref(
device_id, field_idx
)
@@ -448,6 +474,7 @@ class TsFileSeriesReader:
end_time: int,
) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:
"""Execute the underlying table query with exact tag filter pushdown."""
+ self._ensure_open()
tag_columns = list(tag_columns)
field_columns = list(field_columns)
query_columns = list(field_columns)