You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/05/31 09:44:55 UTC

(tsfile) 02/03: add per-dataset manifest cache + LRU fd pool for TsFileDataFrame

This is an automated email from the ASF dual-hosted git repository.

ColinLeeo pushed a commit to branch colin_tsfile_dataframe_init
in repository https://gitbox.apache.org/repos/asf/tsfile.git

commit cd90641429051eee5221cdd87247397fbdd5defa
Author: ColinLee <sh...@163.com>
AuthorDate: Sun May 31 17:26:38 2026 +0800

    add per-dataset manifest cache + LRU fd pool for TsFileDataFrame
    
    Two related changes for fast repeated init over the same shards:
    
    1. Per-dataset JSON manifest (cache_version=2) at
       <common parent>/tsfile_dataset.metacache.json, keyed by absolute
       shard path with each entry carrying its own (size, mtime_ns)
       fingerprint. Hits inject MetadataCatalog into the reader and skip
       the heavy get_all_table_schemas + get_timeseries_metadata native
       path. Adds cache="auto|off|rebuild" to TsFileDataFrame.
    
    2. LRU fd pool with max_open_files=1024 (default). Readers are still
       opened eagerly at construction (fail-fast), but live C++ handles
       beyond the cap are closed via _close_native(); Python wrapper and
       catalog stay valid and the next read transparently reopens through
       _ensure_open().
    
    Existing per-shard sidecar API removed.
---
 python/tests/test_dataset_cache.py  | 177 +++++++++++++++-----
 python/tests/test_tsfile_dataset.py |   6 +-
 python/tsfile/dataset/README.md     | 312 +++++++++++++++++++++++-------------
 python/tsfile/dataset/cache.py      | 133 +++++++++------
 python/tsfile/dataset/dataframe.py  |  71 +++++---
 python/tsfile/dataset/pool.py       |  75 +++++++++
 python/tsfile/dataset/reader.py     |  29 +++-
 7 files changed, 576 insertions(+), 227 deletions(-)

diff --git a/python/tests/test_dataset_cache.py b/python/tests/test_dataset_cache.py
index 0bae42eda..82f4b3091 100644
--- a/python/tests/test_dataset_cache.py
+++ b/python/tests/test_dataset_cache.py
@@ -16,6 +16,7 @@
 # under the License.
 #
 
+import json
 import os
 
 import pandas as pd
@@ -31,11 +32,11 @@ from tsfile import (
 )
 from tsfile.dataset import reader as reader_module
 from tsfile.dataset.cache import (
-    cache_path,
     catalog_from_dict,
     catalog_to_dict,
-    read_sidecar,
-    write_sidecar,
+    extract_catalog,
+    manifest_path,
+    read_manifest,
 )
 
 
@@ -67,8 +68,7 @@ def test_catalog_round_trip(tmp_path):
     reader = next(iter(df._readers.values()))
     catalog = reader.catalog
 
-    data = catalog_to_dict(catalog)
-    rebuilt = catalog_from_dict(data)
+    rebuilt = catalog_from_dict(catalog_to_dict(catalog))
 
     assert [t.table_name for t in rebuilt.table_entries] == [
         t.table_name for t in catalog.table_entries
@@ -93,16 +93,25 @@ def test_catalog_round_trip(tmp_path):
     df.close()
 
 
+def test_manifest_path_uses_common_parent(tmp_path):
+    paths = [str(tmp_path / f"shard_{i}.tsfile") for i in range(3)]
+    assert manifest_path(paths) == str(tmp_path / "tsfile_dataset.metacache.json")
+
+
+def test_manifest_path_falls_back_for_single_file(tmp_path):
+    path = tmp_path / "single.tsfile"
+    _write_weather_file(path)
+    expected = str(tmp_path / "tsfile_dataset.metacache.json")
+    assert manifest_path([str(path)]) == expected
+
+
 def test_cache_auto_hit_skips_native_metadata(tmp_path, monkeypatch):
     path = tmp_path / "cache_hit.tsfile"
     _write_weather_file(path)
 
-    # First load: cold cache — populates the sidecar.
-    df_cold = TsFileDataFrame(str(path), show_progress=False, cache="auto")
-    df_cold.close()
-    assert os.path.exists(cache_path(str(path)))
+    TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
+    assert os.path.exists(manifest_path([str(path)]))
 
-    # Spy on the heavy native metadata fetch. A cache hit must not call it.
     calls = {"count": 0}
     original = reader_module.TsFileSeriesReader._cache_metadata_table_model
 
@@ -117,7 +126,6 @@ def test_cache_auto_hit_skips_native_metadata(tmp_path, monkeypatch):
     df_hot = TsFileDataFrame(str(path), show_progress=False, cache="auto")
     try:
         assert calls["count"] == 0
-        # The cached dataframe is functionally equivalent.
         assert df_hot.list_timeseries() == [
             "weather.device_a.temperature",
             "weather.device_a.humidity",
@@ -131,12 +139,12 @@ def test_cache_invalidated_by_mtime_change(tmp_path):
     _write_weather_file(path)
 
     TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
-    assert read_sidecar(str(path)) is not None
+    shards = read_manifest(manifest_path([str(path)]))
+    assert extract_catalog(shards, str(path)) is not None
 
-    # Bump mtime so the cached fingerprint no longer matches.
     stat = os.stat(str(path))
     os.utime(str(path), ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))
-    assert read_sidecar(str(path)) is None
+    assert extract_catalog(shards, str(path)) is None
 
 
 def test_cache_invalidated_by_size_change(tmp_path):
@@ -144,46 +152,36 @@ def test_cache_invalidated_by_size_change(tmp_path):
     _write_weather_file(path)
 
     TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
-    assert read_sidecar(str(path)) is not None
-
-    # Pretend the sidecar was written for a different file size, preserving mtime.
-    stat_before = os.stat(str(path))
-    import json
-
-    cp = cache_path(str(path))
-    with open(cp, "r") as fh:
+    mp = manifest_path([str(path)])
+    with open(mp, "r") as fh:
         payload = json.load(fh)
-    payload["file"]["size"] = stat_before.st_size + 1
-    with open(cp, "w") as fh:
+    payload["shards"][str(path)]["size"] += 1
+    with open(mp, "w") as fh:
         json.dump(payload, fh)
 
-    assert read_sidecar(str(path)) is None
+    assert extract_catalog(read_manifest(mp), str(path)) is None
 
 
 def test_cache_off_does_not_write(tmp_path):
     path = tmp_path / "cache_off.tsfile"
     _write_weather_file(path)
-
     df = TsFileDataFrame(str(path), show_progress=False, cache="off")
     df.close()
-
-    assert not os.path.exists(cache_path(str(path)))
+    assert not os.path.exists(manifest_path([str(path)]))
 
 
 def test_cache_rebuild_overwrites_stale(tmp_path):
     path = tmp_path / "rebuild.tsfile"
     _write_weather_file(path)
-
-    # Seed a corrupt sidecar to prove rebuild does not read it.
-    with open(cache_path(str(path)), "w") as fh:
+    mp = manifest_path([str(path)])
+    with open(mp, "w") as fh:
         fh.write("not json")
 
     df = TsFileDataFrame(str(path), show_progress=False, cache="rebuild")
     try:
-        # After rebuild the sidecar must be valid again.
-        rebuilt = read_sidecar(str(path))
-        assert rebuilt is not None
-        # And functionally the dataframe works.
+        shards = read_manifest(mp)
+        assert shards is not None
+        assert extract_catalog(shards, str(path)) is not None
         assert df.list_timeseries()
     finally:
         df.close()
@@ -196,18 +194,115 @@ def test_invalid_cache_mode_rejected(tmp_path):
         TsFileDataFrame(str(path), show_progress=False, cache="bogus")
 
 
-def test_multi_shard_cache(tmp_path):
+def test_invalid_max_open_files_rejected(tmp_path):
+    path = tmp_path / "bad_cap.tsfile"
+    _write_weather_file(path)
+    with pytest.raises(ValueError):
+        TsFileDataFrame(str(path), show_progress=False, max_open_files=0)
+
+
+def test_single_manifest_for_multiple_shards(tmp_path):
     paths = [tmp_path / f"shard_{i}.tsfile" for i in range(3)]
     for i, p in enumerate(paths):
         _write_weather_file(p, start=i * 100)
+    str_paths = [str(p) for p in paths]
+    mp = manifest_path(str_paths)
 
-    df = TsFileDataFrame([str(p) for p in paths], show_progress=False, cache="auto")
+    df = TsFileDataFrame(str_paths, show_progress=False, cache="auto")
     df.close()
+    assert os.path.exists(mp)
+    shards = read_manifest(mp)
+    assert set(shards.keys()) == set(str_paths)
     for p in paths:
-        assert os.path.exists(cache_path(str(p)))
+        assert not os.path.exists(str(p) + ".metacache.json")
+
+
+def test_adding_shard_keeps_existing_entries(tmp_path):
+    initial = [tmp_path / "shard_0.tsfile", tmp_path / "shard_1.tsfile"]
+    for i, p in enumerate(initial):
+        _write_weather_file(p, start=i * 100)
+    initial_paths = [str(p) for p in initial]
+    TsFileDataFrame(initial_paths, show_progress=False, cache="auto").close()
+
+    new_shard = tmp_path / "shard_2.tsfile"
+    _write_weather_file(new_shard, start=200)
+    expanded_paths = initial_paths + [str(new_shard)]
+    TsFileDataFrame(expanded_paths, show_progress=False, cache="auto").close()
+
+    shards = read_manifest(manifest_path(expanded_paths))
+    assert set(shards.keys()) == set(expanded_paths)
+
+
+def test_manifest_skipped_on_full_hit(tmp_path):
+    path = tmp_path / "no_rewrite.tsfile"
+    _write_weather_file(path)
+
+    TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
+    mp = manifest_path([str(path)])
+    before = os.stat(mp).st_mtime_ns
+
+    TsFileDataFrame(str(path), show_progress=False, cache="auto").close()
+    after = os.stat(mp).st_mtime_ns
+    assert before == after
+
+
+def test_lru_evicts_older_reader_when_over_cap(tmp_path):
+    paths = [tmp_path / f"lru_{i}.tsfile" for i in range(3)]
+    for i, p in enumerate(paths):
+        _write_weather_file(p, start=i * 100)
+    str_paths = [str(p) for p in paths]
 
-    df_hot = TsFileDataFrame([str(p) for p in paths], show_progress=False, cache="auto")
+    df = TsFileDataFrame(str_paths, show_progress=False, cache="auto", max_open_files=2)
     try:
-        assert len(df_hot.list_timeseries()) == 2
+        # After eager init with cap=2, the LRU shard's native handle is closed
+        # but its Python wrapper + catalog stay valid.
+        first = df._readers[str_paths[0]]
+        second = df._readers[str_paths[1]]
+        third = df._readers[str_paths[2]]
+
+        assert first._reader is None
+        assert second._reader is not None
+        assert third._reader is not None
+        assert df._fd_pool.capacity == 2
+        assert len(df._fd_pool) == 2
+
+        # Touching the evicted reader reopens its native handle and pushes the
+        # current LRU (`second`) out.
+        first._ensure_open()
+
+        assert first._reader is not None
+        assert third._reader is not None
+        assert second._reader is None
+        assert len(df._fd_pool) == 2
     finally:
-        df_hot.close()
+        df.close()
+
+
+def test_lru_disabled_when_cap_exceeds_shard_count(tmp_path):
+    paths = [tmp_path / f"all_open_{i}.tsfile" for i in range(3)]
+    for i, p in enumerate(paths):
+        _write_weather_file(p, start=i * 100)
+
+    df = TsFileDataFrame(
+        [str(p) for p in paths],
+        show_progress=False,
+        cache="auto",
+        max_open_files=16,
+    )
+    try:
+        for p in paths:
+            assert df._readers[str(p)]._reader is not None
+        assert len(df._fd_pool) == 3
+    finally:
+        df.close()
+
+
+def test_lru_close_all_drains_pool(tmp_path):
+    path = tmp_path / "drain.tsfile"
+    _write_weather_file(path)
+
+    df = TsFileDataFrame(str(path), show_progress=False, cache="auto")
+    reader = next(iter(df._readers.values()))
+    assert reader._reader is not None
+    df.close()
+    assert reader._reader is None
diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py
index 74ad86952..4e52a1b5f 100644
--- a/python/tests/test_tsfile_dataset.py
+++ b/python/tests/test_tsfile_dataset.py
@@ -697,8 +697,10 @@ def test_reader_read_series_by_row_retries_across_native_row_query_boundaries():
     class _FakeResultSet:
         def __init__(self, times, values):
             self._batch = pa.table(
-                {"time": pa.array(times, type=pa.int64()),
-                 "totalcloudcover": pa.array(values, type=pa.float64())}
+                {
+                    "time": pa.array(times, type=pa.int64()),
+                    "totalcloudcover": pa.array(values, type=pa.float64()),
+                }
             )
             self._delivered = False
 
diff --git a/python/tsfile/dataset/README.md b/python/tsfile/dataset/README.md
index 825d2c6bf..4a14b6eb2 100644
--- a/python/tsfile/dataset/README.md
+++ b/python/tsfile/dataset/README.md
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-# `TsFileDataFrame` Sidecar Metadata Cache
+# `TsFileDataFrame` Dataset Metadata Manifest
 
 `TsFileDataFrame` opens one C++ reader per shard during construction. Opening
 the reader itself is cheap (one `open(2)` syscall); the expensive part is
@@ -27,32 +27,42 @@ converting the result into Python objects. Repeating that work on every
 process start is wasteful when the underlying `.tsfile` files have not
 changed.
 
-To skip that work, every shard gets a JSON sidecar file next to it that
-records the catalog produced by the previous load. On subsequent loads the
-sidecar is validated against the live file's `stat()` fingerprint and, if it
-still matches, the catalog is rehydrated directly from JSON. The C++ reader
-is still opened — that cost is negligible — but `get_all_table_schemas` and
-`get_timeseries_metadata(None)` are skipped entirely.
+To skip that work, every dataset gets **one** JSON manifest file that
+records the catalog of every shard the dataframe has seen. On subsequent
+loads the manifest is consulted per shard: entries whose `(size, mtime_ns)`
+still match are rehydrated directly from JSON; entries that miss (modified,
+removed, or never recorded) trigger the normal native metadata load and are
+then merged back into the manifest.
+
+The C++ reader is still opened for every shard — that cost is negligible —
+but `get_all_table_schemas` and `get_timeseries_metadata(None)` are skipped
+on cache hits.
 
 ## File layout
 
-The sidecar lives next to the shard and reuses its full name plus a fixed
-suffix:
+One manifest covers an entire dataset. It sits in the common parent
+directory of all the shard paths passed to `TsFileDataFrame`:
 
 ```
-data/2024-01-01.tsfile
-data/2024-01-01.tsfile.metacache.json
+data/
+├── part_0.tsfile
+├── part_1.tsfile
+├── part_2.tsfile
+├── part_3.tsfile
+└── tsfile_dataset.metacache.json   ← the manifest
 ```
 
-Defined in `cache.py`:
+When the shards span multiple subdirectories, the manifest lives in their
+deepest common ancestor. When the inputs share no common path (rare cross-
+filesystem case), the manifest falls back to the first shard's directory so
+there is always a writable target.
 
-| Constant                 | Value                |
-|--------------------------|----------------------|
-| `_CACHE_SUFFIX`          | `".metacache.json"`  |
-| `_CACHE_FORMAT_VERSION`  | `1`                  |
+Defined in `cache.py`:
 
-One shard, one sidecar. There is no cross-shard manifest: each sidecar is
-independent and can be deleted or rebuilt in isolation.
+| Constant              | Value                                |
+|-----------------------|--------------------------------------|
+| `_MANIFEST_FILENAME`  | `"tsfile_dataset.metacache.json"`    |
+| `_MANIFEST_VERSION`   | `2`                                  |
 
 ## JSON schema
 
@@ -60,29 +70,37 @@ The on-disk payload is a single JSON object:
 
 ```json
 {
-  "cache_version": 1,
-  "file": {
-    "size": 1234567,
-    "mtime_ns": 1717050000000000000
-  },
-  "catalog": {
-    "tables": [ ... ],
-    "devices": [ ... ],
-    "series_stats": [ ... ]
+  "cache_version": 2,
+  "shards": {
+    "/abs/path/data/part_0.tsfile": {
+      "size": 1234567,
+      "mtime_ns": 1717050000000000000,
+      "catalog": { "tables": [...], "devices": [...], "series_stats": [...] }
+    },
+    "/abs/path/data/part_1.tsfile": { ... }
   }
 }
 ```
 
 ### Top-level fields
 
-| Field           | Type   | Purpose                                                                              |
-|-----------------|--------|--------------------------------------------------------------------------------------|
-| `cache_version` | int    | Sidecar schema version. Mismatch → invalidate.                                       |
-| `file.size`     | int    | `os.stat(tsfile).st_size` at the time the sidecar was written. Mismatch → invalidate.|
-| `file.mtime_ns` | int    | `os.stat(tsfile).st_mtime_ns` at the time the sidecar was written. Mismatch → invalidate. |
-| `catalog`       | object | Serialized form of one shard's `MetadataCatalog`.                                    |
+| Field           | Type                | Purpose                                                        |
+|-----------------|---------------------|----------------------------------------------------------------|
+| `cache_version` | int                 | Manifest schema version. Mismatch → invalidate the entire manifest. |
+| `shards`        | object              | Map from each shard's absolute path to its cache entry.        |
+
+### `shards[<abs_path>]`
 
-### `catalog.tables[]`
+Each shard entry is independent. A stale fingerprint on one shard only
+invalidates that shard; the other shards still hit.
+
+| Field       | Type   | Notes                                                                                          |
+|-------------|--------|------------------------------------------------------------------------------------------------|
+| `size`      | int    | `os.stat(shard).st_size` at the time the entry was written. Mismatch → that shard misses.      |
+| `mtime_ns`  | int    | `os.stat(shard).st_mtime_ns` at the time the entry was written. Mismatch → that shard misses.  |
+| `catalog`   | object | Serialized form of the shard's `MetadataCatalog`. See below.                                   |
+
+### `shards[<abs_path>].catalog.tables[]`
 
 Each entry mirrors a `TableEntry`:
 
@@ -96,38 +114,38 @@ Each entry mirrors a `TableEntry`:
 The position of a table in this array is its `table_index`, referenced by
 `devices[].table_index` below.
 
-### `catalog.devices[]`
+### `shards[<abs_path>].catalog.devices[]`
 
 Each entry mirrors a `DeviceEntry`:
 
 | Field         | Type    | Notes                                                                                        |
 |---------------|---------|----------------------------------------------------------------------------------------------|
-| `table_index` | int     | Index into `catalog.tables[]`.                                                               |
+| `table_index` | int     | Index into the same shard's `catalog.tables[]`.                                              |
 | `tag_values`  | any[]   | Tag values in the order of the table's `tag_columns`. Types follow the table's `tag_types`.  |
 | `min_time`    | int     | Minimum timestamp observed across the device's contributing measurements.                    |
 | `max_time`    | int     | Maximum timestamp observed across the device's contributing measurements.                    |
 
 Tag values are stored using their natural JSON representation:
 
-| `TSDataType`            | JSON form |
-|-------------------------|-----------|
-| `BOOLEAN`               | bool      |
-| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | int |
-| `FLOAT`, `DOUBLE`       | number    |
-| `TEXT`, `STRING`, `BLOB`| string    |
-| (missing trailing tag)  | `null`    |
+| `TSDataType`                        | JSON form |
+|-------------------------------------|-----------|
+| `BOOLEAN`                           | bool      |
+| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | int     |
+| `FLOAT`, `DOUBLE`                   | number    |
+| `TEXT`, `STRING`, `BLOB`            | string    |
+| (missing trailing tag)              | `null`    |
 
 The position of a device in this array is its `device_index`, referenced by
 `series_stats[].device_index` below.
 
-### `catalog.series_stats[]`
+### `shards[<abs_path>].catalog.series_stats[]`
 
 Each entry mirrors one `(device_id, field_idx) → stats` entry in
 `MetadataCatalog.series_stats_by_ref`:
 
 | Field                | Type         | Notes                                                                              |
 |----------------------|--------------|------------------------------------------------------------------------------------|
-| `device_index`       | int          | Index into `catalog.devices[]`.                                                    |
+| `device_index`       | int          | Index into the same shard's `catalog.devices[]`.                                   |
 | `field_index`        | int          | Index into the device's table's `field_columns`.                                   |
 | `length`             | int          | Row count derived from the value column's statistic.                               |
 | `min_time`           | int \| null  | Minimum timestamp from the value column statistic. `null` when no statistic.       |
@@ -141,84 +159,150 @@ present so that `(device_index, field_index)` resolution always succeeds.
 
 ## Validation rules
 
-When `cache_from_dict` and `read_sidecar` decide whether to trust a sidecar,
-the checks below run in order. Any failure returns `None` and the dataframe
-falls back to a full native metadata load.
+Validation happens at two levels.
+
+**Manifest-level**, by `read_manifest`:
 
-1. The shard's `os.stat()` succeeds.
-2. The sidecar exists and parses as JSON.
-3. The top-level value is an object.
-4. `cache_version` equals `_CACHE_FORMAT_VERSION` (currently `1`).
-5. `file.size` equals the shard's current `st_size`.
-6. `file.mtime_ns` equals the shard's current `st_mtime_ns`.
-7. `catalog_from_dict` succeeds on `catalog` (well-formed tables, devices,
-   and stats).
+1. The manifest file exists and parses as JSON.
+2. The top-level value is an object.
+3. `cache_version` equals `_MANIFEST_VERSION` (currently `2`).
+4. `shards` is an object.
+
+A manifest-level failure throws away the entire cache; every shard becomes
+a miss and the manifest is rewritten from scratch.
+
+**Per-shard**, by `extract_catalog`:
+
+1. The shard has an entry in `shards`.
+2. `os.stat(shard)` succeeds.
+3. The entry's `size` equals the live `st_size`.
+4. The entry's `mtime_ns` equals the live `st_mtime_ns`.
+5. `catalog_from_dict` succeeds on the entry's `catalog`.
+
+A per-shard failure only invalidates that one shard; other shards in the
+same manifest continue to hit.
 
 ## Write semantics
 
-`write_sidecar` writes through a temporary file in the same directory and
-calls `os.replace` for an atomic swap. The `size` and `mtime_ns` recorded in
-the payload come from a fresh `os.stat()` taken immediately before writing.
+After a load, `write_manifest` is called only if at least one shard missed
+(or the user passed `cache="rebuild"`). It does:
+
+1. Re-read the existing manifest from disk so concurrent writers' new
+   entries are not lost.
+2. Overwrite the entries for the shards we just loaded with fresh
+   `(size, mtime_ns, catalog)` triples.
+3. Write through a temporary file in the same directory and `os.replace`
+   atomically.
+
+Entries for shards not in the current dataframe are preserved verbatim:
+loading a subset of a dataset does not prune the others.
 
-All disk-level errors (permission denied, read-only filesystem, no space) are
-swallowed silently; the dataframe is still constructed successfully, the
-sidecar is simply absent and the next load pays the full metadata cost.
+All disk-level errors (permission denied, read-only filesystem, no space)
+are swallowed silently; the dataframe is still constructed successfully,
+the manifest is simply not updated.
+
+When every shard hits, the manifest is **not** rewritten — pure-hit loads
+do not touch the filesystem.
 
 ## Cache modes
 
 `TsFileDataFrame(paths, cache=...)` accepts:
 
-| Mode      | Read sidecar? | Write sidecar after load? |
-|-----------|---------------|---------------------------|
-| `"auto"` (default) | yes  | yes, only when the read missed |
-| `"off"`            | no   | no                              |
-| `"rebuild"`        | no   | yes, always (overwrites any existing sidecar) |
+| Mode      | Read manifest? | Write manifest after load? |
+|-----------|----------------|-----------------------------|
+| `"auto"` (default) | yes  | yes, only if some shard missed    |
+| `"off"`            | no   | no                                 |
+| `"rebuild"`        | no   | yes — every shard counts as a miss |
+
+`"rebuild"` exists for the case where the manifest is suspected of being
+stale or corrupt despite passing the size/mtime checks — for example after
+a manual edit or an interrupted previous write.
+
+## File descriptor budget (`max_open_files`)
+
+Every shard's `TsFileSeriesReader` opens a C++ reader eagerly during
+construction — failures surface immediately rather than at the first
+query. With many shards this could exhaust the process's fd budget
+(`ulimit -n`), so the dataframe wraps the live readers in an LRU pool.
+
+`TsFileDataFrame(paths, max_open_files=N)`:
+
+| Behavior                                  | When `N >= len(paths)` | When `N < len(paths)` |
+|-------------------------------------------|------------------------|------------------------|
+| Native C++ reader opened at construction? | every shard, all stay  | every shard, but only N stay live |
+| Catalog kept in memory?                   | every shard            | every shard            |
+| Eviction trigger                          | never                  | each new touch pushes the LRU out |
+| Reopen on eviction                        | n/a                    | first read on the evicted shard calls `_ensure_open()` |
+
+Default is `1024`. Tune up when a single query routinely touches more
+shards than that — every miss costs one `open(2)` plus a footer reload
+on the C++ side, so a too-small pool causes thrashing. Tune down when
+the host has a tight fd cap and you accept the reopen cost.
 
-`"rebuild"` exists for the case where the sidecar is suspected of being
-stale or corrupt despite passing the size/mtime checks — for example after a
-manual edit or an interrupted previous write.
+Eviction only closes the native handle. The Python wrapper and its
+catalog (from the manifest or from native metadata) remain valid, so
+nothing about query semantics changes — the next data access on the
+evicted shard reopens transparently.
+
+## Concurrency
+
+Multiple processes loading the same dataset concurrently both end up
+writing the manifest. Writes are atomic (temp file + `os.replace`) and
+content is keyed by absolute shard path, so:
+
+- If they cover the same shards: last writer wins; the manifest is
+  identical either way.
+- If they cover different overlapping shards: each writer re-reads the
+  current manifest before merging, so neither side wipes the other's
+  contributions.
+
+A pathological race (both writers see an empty manifest, then both write
+their disjoint subsets) can drop the earlier writer's entries. The next
+load reproduces them. No explicit lock is taken.
 
 ## Worked example
 
-The smallest possible sidecar — one table, one device, one field, three
-rows:
+The smallest possible manifest — one shard, one table, one device, one
+field, three rows:
 
 ```json
 {
-  "cache_version": 1,
-  "file": {
-    "size": 4096,
-    "mtime_ns": 1717050000000000000
-  },
-  "catalog": {
-    "tables": [
-      {
-        "name": "weather",
-        "tag_columns": ["device"],
-        "tag_types": [11],
-        "field_columns": ["temperature"]
-      }
-    ],
-    "devices": [
-      {
-        "table_index": 0,
-        "tag_values": ["device_a"],
-        "min_time": 0,
-        "max_time": 2
-      }
-    ],
-    "series_stats": [
-      {
-        "device_index": 0,
-        "field_index": 0,
-        "length": 3,
-        "min_time": 0,
-        "max_time": 2,
-        "timeline_length": 3,
-        "timeline_min_time": 0,
-        "timeline_max_time": 2
+  "cache_version": 2,
+  "shards": {
+    "/abs/path/data/weather.tsfile": {
+      "size": 4096,
+      "mtime_ns": 1717050000000000000,
+      "catalog": {
+        "tables": [
+          {
+            "name": "weather",
+            "tag_columns": ["device"],
+            "tag_types": [11],
+            "field_columns": ["temperature"]
+          }
+        ],
+        "devices": [
+          {
+            "table_index": 0,
+            "tag_values": ["device_a"],
+            "min_time": 0,
+            "max_time": 2
+          }
+        ],
+        "series_stats": [
+          {
+            "device_index": 0,
+            "field_index": 0,
+            "length": 3,
+            "min_time": 0,
+            "max_time": 2,
+            "timeline_length": 3,
+            "timeline_min_time": 0,
+            "timeline_max_time": 2
+          }
+        ]
       }
-    ]
+    }
   }
 }
 ```
@@ -227,12 +311,12 @@ rows:
 
 ## Operational notes
 
-- The sidecar is **not** a portable export. It encodes positional indices
-  into the same shard's footer ordering. Moving a sidecar between hosts is
-  fine as long as the corresponding shard file moves with it; copying it to
-  a different shard is not supported.
-- Cleaning up sidecars is safe at any time. The next load will pay the
-  metadata cost once and rewrite them.
-- Bumping `_CACHE_FORMAT_VERSION` is the migration path when the schema
-  needs to change. All older sidecars become invalidated automatically by
-  the version check.
+- The manifest is **not** a portable export. Shard keys are absolute
+  paths, and the inner `catalog` encodes positional indices into the
+  same shard's footer. Moving the dataset to a different absolute path
+  invalidates every entry; the next load rebuilds them.
+- Deleting the manifest is safe at any time. The next load pays the full
+  metadata cost once and rewrites it.
+- Bumping `_MANIFEST_VERSION` is the migration path when the schema needs
+  to change. All older manifests are discarded automatically by the
+  version check.
diff --git a/python/tsfile/dataset/cache.py b/python/tsfile/dataset/cache.py
index 40b8d8d58..a2269910d 100644
--- a/python/tsfile/dataset/cache.py
+++ b/python/tsfile/dataset/cache.py
@@ -16,28 +16,43 @@
 # under the License.
 #
 
-"""Per-shard JSON sidecar cache for :class:`MetadataCatalog`.
+"""Per-dataset JSON manifest cache for :class:`MetadataCatalog`.
 
-The dataframe's expensive init step is reading TsFile footer + materializing
-every device's TimeseriesIndex into Python objects. Opening the C++ reader
-itself is just one ``open(2)``. So we cache the materialized catalog next to
-each .tsfile and skip the metadata path on subsequent loads.
+One JSON file describes the catalogs of every shard the dataframe was
+constructed from. The manifest lives in the common parent directory of the
+input paths and is keyed by each shard's absolute path. Each shard entry
+carries its own ``(size, mtime_ns)`` fingerprint so that a single stale
+shard does not invalidate the rest of the manifest.
 """
 
 import json
 import os
 import tempfile
-from typing import Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from ..constants import TSDataType
 from .metadata import MetadataCatalog
 
-_CACHE_SUFFIX = ".metacache.json"
-_CACHE_FORMAT_VERSION = 1
+_MANIFEST_FILENAME = "tsfile_dataset.metacache.json"
+_MANIFEST_VERSION = 2
 
 
-def cache_path(tsfile_path: str) -> str:
-    return tsfile_path + _CACHE_SUFFIX
+def manifest_path(paths: List[str]) -> str:
+    """Return the dataset manifest path for a list of absolute shard paths.
+
+    The manifest sits in the common parent directory of all shards. When the
+    paths span unrelated trees (no common parent exists), we fall back to the
+    first path's directory so we always have a writable location.
+    """
+    if not paths:
+        raise ValueError("Cannot compute manifest path for empty paths.")
+    try:
+        common = os.path.commonpath(paths)
+    except ValueError:
+        common = os.path.dirname(paths[0])
+    if os.path.isfile(common):
+        common = os.path.dirname(common)
+    return os.path.join(common, _MANIFEST_FILENAME)
 
 
 def catalog_to_dict(catalog: MetadataCatalog) -> dict:
@@ -104,55 +119,84 @@ def catalog_from_dict(data: dict) -> MetadataCatalog:
     return catalog
 
 
-def read_sidecar(tsfile_path: str) -> Optional[MetadataCatalog]:
-    """Read and validate the sidecar cache for one shard.
-
-    Returns ``None`` for any of: missing/unreadable cache file, schema
-    mismatch, size/mtime mismatch, or corrupt JSON.
-    """
+def _shard_entry(file_path: str, catalog: MetadataCatalog) -> Optional[dict]:
+    """Build a shard manifest entry by stat'ing the file right before write."""
     try:
-        tsfile_stat = os.stat(tsfile_path)
+        stat = os.stat(file_path)
     except OSError:
         return None
+    return {
+        "size": stat.st_size,
+        "mtime_ns": stat.st_mtime_ns,
+        "catalog": catalog_to_dict(catalog),
+    }
+
+
+def _entry_matches_file(entry: dict, file_path: str) -> bool:
+    try:
+        stat = os.stat(file_path)
+    except OSError:
+        return False
+    return (
+        entry.get("size") == stat.st_size and entry.get("mtime_ns") == stat.st_mtime_ns
+    )
+
+
+def read_manifest(path: str) -> Optional[Dict[str, dict]]:
+    """Load and shallow-validate the manifest. Returns the shards dict, or None."""
     try:
-        with open(cache_path(tsfile_path), "r") as fh:
+        with open(path, "r") as fh:
             data = json.load(fh)
     except (OSError, json.JSONDecodeError):
         return None
     if not isinstance(data, dict):
         return None
-    if data.get("cache_version") != _CACHE_FORMAT_VERSION:
+    if data.get("cache_version") != _MANIFEST_VERSION:
         return None
-    file_meta = data.get("file") or {}
-    if file_meta.get("size") != tsfile_stat.st_size:
+    shards = data.get("shards")
+    if not isinstance(shards, dict):
         return None
-    if file_meta.get("mtime_ns") != tsfile_stat.st_mtime_ns:
+    return shards
+
+
+def extract_catalog(
+    shards: Dict[str, dict], file_path: str
+) -> Optional[MetadataCatalog]:
+    """Return the cached catalog for one shard iff its fingerprint still matches."""
+    entry = shards.get(file_path)
+    if entry is None or not isinstance(entry, dict):
+        return None
+    if not _entry_matches_file(entry, file_path):
         return None
     try:
-        return catalog_from_dict(data["catalog"])
+        return catalog_from_dict(entry["catalog"])
     except (KeyError, ValueError, TypeError):
         return None
 
 
-def write_sidecar(tsfile_path: str, catalog: MetadataCatalog) -> None:
-    """Atomically write the sidecar cache. Disk errors are swallowed."""
-    try:
-        tsfile_stat = os.stat(tsfile_path)
-    except OSError:
-        return
-    payload = {
-        "cache_version": _CACHE_FORMAT_VERSION,
-        "file": {
-            "size": tsfile_stat.st_size,
-            "mtime_ns": tsfile_stat.st_mtime_ns,
-        },
-        "catalog": catalog_to_dict(catalog),
-    }
-    target = cache_path(tsfile_path)
-    target_dir = os.path.dirname(target) or "."
+def write_manifest(
+    path: str,
+    fresh_catalogs: Iterable[Tuple[str, MetadataCatalog]],
+) -> None:
+    """Atomically merge fresh catalogs into the manifest at ``path``.
+
+    Reads the existing manifest first (if any), overwrites the entries for
+    the shards in ``fresh_catalogs``, and writes the result back via temp
+    file + ``os.replace``. Other shards' entries are preserved. OS errors
+    (read-only filesystem, no space) are swallowed silently.
+    """
+    existing = read_manifest(path) or {}
+    merged = dict(existing)
+    for file_path, catalog in fresh_catalogs:
+        entry = _shard_entry(file_path, catalog)
+        if entry is not None:
+            merged[file_path] = entry
+
+    payload = {"cache_version": _MANIFEST_VERSION, "shards": merged}
+    target_dir = os.path.dirname(path) or "."
     try:
         fd, tmp_path = tempfile.mkstemp(
-            prefix=os.path.basename(target) + ".",
+            prefix=_MANIFEST_FILENAME + ".",
             suffix=".tmp",
             dir=target_dir,
         )
@@ -161,16 +205,9 @@ def write_sidecar(tsfile_path: str, catalog: MetadataCatalog) -> None:
     try:
         with os.fdopen(fd, "w") as fh:
             json.dump(payload, fh)
-        os.replace(tmp_path, target)
+        os.replace(tmp_path, path)
     except OSError:
         try:
             os.unlink(tmp_path)
         except OSError:
             pass
-
-
-def remove_sidecar(tsfile_path: str) -> None:
-    try:
-        os.unlink(cache_path(tsfile_path))
-    except OSError:
-        pass
diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py
index c83439d81..ba1d8124a 100644
--- a/python/tsfile/dataset/dataframe.py
+++ b/python/tsfile/dataset/dataframe.py
@@ -28,7 +28,7 @@ import warnings
 
 import numpy as np
 
-from .cache import read_sidecar, write_sidecar
+from .cache import extract_catalog, manifest_path, read_manifest, write_manifest
 from .formatting import format_dataframe_table
 from .metadata import (
     TableEntry,
@@ -37,9 +37,11 @@ from .metadata import (
     split_logical_series_path,
 )
 from .merge import build_aligned_matrix, merge_time_value_parts, merge_timestamp_parts
+from .pool import _FdPool
 from .timeseries import AlignedTimeseries, Timeseries
 
 _CACHE_MODES = ("auto", "off", "rebuild")
+_DEFAULT_MAX_OPEN_FILES = 1024
 
 DeviceKey = Tuple[str, tuple]
 SeriesRefKey = Tuple[int, int]
@@ -583,12 +585,16 @@ class TsFileDataFrame:
         show_progress: bool = True,
         *,
         cache: str = "auto",
+        max_open_files: int = _DEFAULT_MAX_OPEN_FILES,
     ):
         if cache not in _CACHE_MODES:
             raise ValueError(f"cache must be one of {_CACHE_MODES}, got {cache!r}")
+        if max_open_files < 1:
+            raise ValueError(f"max_open_files must be >= 1, got {max_open_files}")
         self._paths = _expand_paths(paths)
         self._show_progress = show_progress
         self._cache_mode = cache
+        self._fd_pool = _FdPool(max_open_files)
         self._readers: Dict[str, object] = {}
         self._index = _LogicalIndex()
         self._cache = _DerivedCache()
@@ -608,6 +614,7 @@ class TsFileDataFrame:
         obj._paths = parent._paths
         obj._show_progress = parent._show_progress
         obj._cache_mode = parent._cache_mode
+        obj._fd_pool = parent._fd_pool
         obj._readers = parent._readers
         obj._index = _LogicalIndex(
             table_entries=parent._index.table_entries,
@@ -635,10 +642,20 @@ class TsFileDataFrame:
         """Build the logical cross-file index and the derived per-series caches."""
         from .reader import TsFileSeriesReader
 
+        shards: Dict[str, dict] = {}
+        if self._cache_mode == "auto":
+            shards = read_manifest(manifest_path(self._paths)) or {}
+
         if len(self._paths) >= 2:
-            self._load_metadata_parallel(TsFileSeriesReader)
+            misses = self._load_metadata_parallel(TsFileSeriesReader, shards)
         else:
-            self._load_metadata_serial(TsFileSeriesReader)
+            misses = self._load_metadata_serial(TsFileSeriesReader, shards)
+
+        if self._cache_mode != "off" and misses:
+            write_manifest(
+                manifest_path(self._paths),
+                [(file_path, self._readers[file_path].catalog) for file_path in misses],
+            )
 
         self._cache.devices = [
             _build_device_entry(refs) for refs in self._index.device_refs
@@ -664,49 +681,61 @@ class TsFileDataFrame:
             )
         sys.stderr.flush()
 
-    def _open_with_cache(self, reader_class, file_path: str, show_progress: bool):
-        """Open one shard reader, injecting a sidecar catalog when available.
+    def _open_with_cache(
+        self,
+        reader_class,
+        file_path: str,
+        show_progress: bool,
+        shards: Dict[str, dict],
+    ):
+        """Open one shard reader, injecting a cached catalog when available.
 
         Returns ``(reader, was_cache_miss)``. ``was_cache_miss`` is True when
-        the catalog had to be materialized from the file rather than loaded
-        from the JSON sidecar.
+        the catalog had to be materialized from the file rather than rehydrated
+        from the dataset manifest. The reader is registered in ``self._fd_pool``
+        so eager opens beyond ``max_open_files`` evict older live handles.
         """
         catalog = None
         if self._cache_mode == "auto":
-            catalog = read_sidecar(file_path)
-        reader = reader_class(file_path, show_progress=show_progress, catalog=catalog)
+            catalog = extract_catalog(shards, file_path)
+        reader = reader_class(
+            file_path,
+            show_progress=show_progress,
+            catalog=catalog,
+            pool=self._fd_pool,
+        )
         return reader, catalog is None
 
-    def _maybe_write_sidecar(self, file_path: str, reader) -> None:
-        if self._cache_mode == "off":
-            return
-        write_sidecar(file_path, reader.catalog)
-
-    def _load_metadata_serial(self, reader_class):
+    def _load_metadata_serial(self, reader_class, shards: Dict[str, dict]):
         total = len(self._paths)
         self._show_loading_progress(0, total)
+        misses: List[str] = []
 
         for index, file_path in enumerate(self._paths, start=1):
             reader, was_miss = self._open_with_cache(
                 reader_class,
                 file_path,
                 self._show_progress and total == 1,
+                shards,
             )
             _register_reader(self._readers, self._index, file_path, reader)
             if was_miss:
-                self._maybe_write_sidecar(file_path, reader)
+                misses.append(file_path)
             if total > 1:
                 self._show_loading_progress(index, total)
 
         self._show_loading_progress(
             total, total, sum(reader.series_count for reader in self._readers.values())
         )
+        return misses
 
-    def _load_metadata_parallel(self, reader_class):
+    def _load_metadata_parallel(self, reader_class, shards: Dict[str, dict]):
         from concurrent.futures import ThreadPoolExecutor, as_completed
 
         def open_file(file_path):
-            reader, was_miss = self._open_with_cache(reader_class, file_path, False)
+            reader, was_miss = self._open_with_cache(
+                reader_class, file_path, False, shards
+            )
             return file_path, reader, was_miss
 
         total = len(self._paths)
@@ -716,7 +745,7 @@ class TsFileDataFrame:
         ) as executor:
             futures = {executor.submit(open_file, path): path for path in self._paths}
             results = {}
-            misses = []
+            misses: List[str] = []
             done = 0
             for future in as_completed(futures):
                 file_path, reader, was_miss = future.result()
@@ -738,8 +767,7 @@ class TsFileDataFrame:
                 results[file_path],
             )
 
-        for file_path in misses:
-            self._maybe_write_sidecar(file_path, results[file_path])
+        return misses
 
     def _get_series_components(
         self, series_ref: SeriesRefKey
@@ -1029,6 +1057,7 @@ class TsFileDataFrame:
         for reader in self._readers.values():
             reader.close()
         self._readers.clear()
+        self._fd_pool.close_all()
         self._closed = True
 
     def __del__(self):
diff --git a/python/tsfile/dataset/pool.py b/python/tsfile/dataset/pool.py
new file mode 100644
index 000000000..106eb95d5
--- /dev/null
+++ b/python/tsfile/dataset/pool.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+"""LRU pool of live C++ TsFile readers.
+
+Per-shard ``TsFileSeriesReader`` instances always carry their materialized
+catalog, but the underlying C++ ``TsFileReaderPy`` (one fd + footer state)
+is expensive enough that we want to bound how many stay live at once.
+
+The pool tracks readers in MRU order. When ``touch`` would push the size
+above ``capacity``, the least-recently-used reader's native handle is
+closed via ``_close_native()``; its Python wrapper and catalog stay valid
+and the next read on that shard will reopen lazily through
+``_ensure_open()``.
+
+This is single-threaded by design — the dataframe's read paths are
+expected to be called from one thread at a time.
+"""
+
+from collections import OrderedDict
+
+
+class _FdPool:
+    def __init__(self, capacity: int):
+        if capacity < 1:
+            raise ValueError(f"max_open_files must be >= 1, got {capacity}")
+        self._capacity = capacity
+        self._order: "OrderedDict[str, object]" = OrderedDict()
+
+    @property
+    def capacity(self) -> int:
+        return self._capacity
+
+    def __len__(self) -> int:
+        return len(self._order)
+
+    def touch(self, reader) -> None:
+        """Mark ``reader`` as most-recently-used. Evict LRU if over capacity."""
+        key = reader.file_path
+        if key in self._order:
+            self._order.move_to_end(key)
+            return
+        self._order[key] = reader
+        while len(self._order) > self._capacity:
+            evict_key, victim = next(iter(self._order.items()))
+            if victim is reader:
+                # Never evict the reader we just inserted.
+                break
+            self._order.pop(evict_key, None)
+            victim._close_native()
+
+    def discard(self, reader) -> None:
+        """Remove ``reader`` from the pool without closing anything."""
+        self._order.pop(reader.file_path, None)
+
+    def close_all(self) -> None:
+        """Close native handles for every reader still in the pool."""
+        for reader in list(self._order.values()):
+            reader._close_native()
+        self._order.clear()
diff --git a/python/tsfile/dataset/reader.py b/python/tsfile/dataset/reader.py
index bcf977831..fa83dff7f 100644
--- a/python/tsfile/dataset/reader.py
+++ b/python/tsfile/dataset/reader.py
@@ -75,12 +75,14 @@ class TsFileSeriesReader:
         show_progress: bool = True,
         *,
         catalog: "MetadataCatalog | None" = None,
+        pool: "object | None" = None,
     ):
         if not os.path.exists(file_path):
             raise FileNotFoundError(f"TsFile not found: {file_path}")
 
         self.file_path = file_path
         self.show_progress = show_progress
+        self._pool = pool
 
         try:
             self._reader = TsFileReaderPy(file_path)
@@ -93,9 +95,29 @@ class TsFileSeriesReader:
             self._catalog = MetadataCatalog()
             self._cache_metadata()
 
+        if self._pool is not None:
+            self._pool.touch(self)
+
     def __del__(self):
         self.close()
 
+    def _ensure_open(self) -> None:
+        """Reopen the native reader if it was evicted; mark MRU in the pool."""
+        if getattr(self, "_reader", None) is None:
+            self._reader = TsFileReaderPy(self.file_path)
+        pool = getattr(self, "_pool", None)
+        if pool is not None:
+            pool.touch(self)
+
+    def _close_native(self) -> None:
+        """Close the native handle but keep the Python wrapper + catalog usable."""
+        if getattr(self, "_reader", None) is not None:
+            try:
+                self._reader.close()
+            except Exception:
+                pass
+            self._reader = None
+
     @property
     def catalog(self) -> MetadataCatalog:
         return self._catalog
@@ -119,11 +141,14 @@ class TsFileSeriesReader:
             ), device_id, field_idx
 
     def close(self):
-        if hasattr(self, "_reader"):
+        if getattr(self, "_pool", None) is not None:
+            self._pool.discard(self)
+        if getattr(self, "_reader", None) is not None:
             try:
                 self._reader.close()
             except Exception:
                 pass
+            self._reader = None
 
     def _cache_metadata(self):
         """Wrap metadata discovery so reader construction surfaces one stable error shape."""
@@ -368,6 +393,7 @@ class TsFileSeriesReader:
         if limit <= 0:
             return np.array([], dtype=np.int64), np.array([], dtype=np.float64)
 
+        self._ensure_open()
         table_entry, device_entry, field_name = self._resolve_series_ref(
             device_id, field_idx
         )
@@ -448,6 +474,7 @@ class TsFileSeriesReader:
         end_time: int,
     ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:
         """Execute the underlying table query with exact tag filter pushdown."""
+        self._ensure_open()
         tag_columns = list(tag_columns)
         field_columns = list(field_columns)
         query_columns = list(field_columns)