Update CTable docs, examples, and benchmarks for TreeStore-backed persistence

Author: FrancescAlted

bench/ctable/bench_persistency.py

@@ -59,11 +59,11 @@ def clean() -> None:
 # Section 1: bulk creation — extend()
 # ---------------------------------------------------------------------------
 
-sep("1. extend() — bulk insert: in-memory vs file-backed")
+sep("1. extend() — bulk insert: in-memory vs TreeStore-backed")
 
 SIZES = [1_000, 10_000, 100_000, 1_000_000]
 
-print(f"{'rows':>12}  {'in-memory (s)':>16}  {'file-backed (s)':>16}  {'overhead':>10}")
+print(f"{'rows':>12}  {'in-memory (s)':>16}  {'store-backed (s)':>16}  {'overhead':>10}")
 print(f"{'----':>12}  {'-------------':>16}  {'---------------':>16}  {'--------':>10}")
 
 for N in SIZES:
@@ -77,6 +77,7 @@ def bench_file(N=N, data=data):
         clean()
         t = blosc2.CTable(Row, urlpath=TABLE_DIR + "/ext", mode="w", expected_size=N)
         t.extend(data, validate=False)
+        t.close()
 
     t_mem = tmin(bench_mem)
     t_file = tmin(bench_file)
@@ -89,16 +90,20 @@ def bench_file(N=N, data=data):
 
 sep("2. open() — time to reopen a persistent table")
 
-print(f"{'rows':>12}  {'CTable.open() (s)':>20}  {'CTable(..., mode=a) (s)':>24}")
-print(f"{'----':>12}  {'------------------':>20}  {'------------------------':>24}")
+print(f"{'rows':>12}  {'blosc2.open() (s)':>18}  {'CTable.open() (s)':>20}  {'CTable(..., mode=a) (s)':>24}")
+print(f"{'----':>12}  {'----------------':>18}  {'------------------':>20}  {'------------------------':>24}")
 
 for N in SIZES:
     data = [(i, float(i % 100), i % 2 == 0) for i in range(N)]
     clean()
     path = TABLE_DIR + "/reopen"
     t = blosc2.CTable(Row, urlpath=path, mode="w", expected_size=N)
     t.extend(data, validate=False)
-    del t
+    t.close()
+
+    def bench_blosc2_open(path=path):
+        t2 = blosc2.open(path, mode="r")
+        _ = len(t2)
 
     def bench_open(path=path):
         t2 = blosc2.CTable.open(path, mode="r")
@@ -108,15 +113,16 @@ def bench_ctor(path=path):
         t2 = blosc2.CTable(Row, urlpath=path, mode="a")
         _ = len(t2)
 
+    t_b2_open = tmin(bench_blosc2_open)
     t_open = tmin(bench_open)
     t_ctor = tmin(bench_ctor)
-    print(f"{N:>12,}  {t_open:>20.4f}  {t_ctor:>24.4f}")
+    print(f"{N:>12,}  {t_b2_open:>18.4f}  {t_open:>20.4f}  {t_ctor:>24.4f}")
 
 # ---------------------------------------------------------------------------
 # Section 3: append() — single-row inserts after reopen
 # ---------------------------------------------------------------------------
 
-sep("3. append() — 1 000 single-row inserts: in-memory vs file-backed")
+sep("3. append() — 1 000 single-row inserts: in-memory vs TreeStore-backed")
 
 APPEND_N = 1_000
 PREALLOCATE = 10_000  # avoid resize noise
@@ -146,7 +152,8 @@ def bench_append_file():
     # Reset file table before each run
     if label == "file-backed":
         clean()
-        blosc2.CTable(Row, urlpath=path, mode="w", expected_size=PREALLOCATE)
+        t = blosc2.CTable(Row, urlpath=path, mode="w", expected_size=PREALLOCATE)
+        t.close()
     elapsed = tmin(fn)
     us_per_row = elapsed / APPEND_N * 1e6
     print(f"{label:>14}  {elapsed:>12.4f}  {us_per_row:>12.1f}")
@@ -155,9 +162,9 @@ def bench_append_file():
 # Section 4: column read — to_numpy() after reopen
 # ---------------------------------------------------------------------------
 
-sep("4. column read — to_numpy() on 'id': in-memory vs file-backed")
+sep("4. column read — to_numpy() on 'id': in-memory vs TreeStore-backed")
 
-print(f"{'rows':>12}  {'in-memory (s)':>16}  {'file-backed (s)':>16}  {'ratio':>8}")
+print(f"{'rows':>12}  {'in-memory (s)':>16}  {'store-backed (s)':>16}  {'ratio':>8}")
 print(f"{'----':>12}  {'-------------':>16}  {'---------------':>16}  {'-----':>8}")
 
 for N in SIZES:
@@ -170,6 +177,7 @@ def bench_append_file():
     path = TABLE_DIR + "/read"
     t_file_table = blosc2.CTable(Row, urlpath=path, mode="w", expected_size=N)
     t_file_table.extend(data, validate=False)
+    t_file_table.close()
     # Reopen read-only (simulates a real read workload)
     t_ro = blosc2.CTable.open(path, mode="r")
 

examples/ctable/persistence.py

@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #######################################################################
 
-# Persistence: write to disk, open read-only/read-write, save, load.
+# Persistence: write to disk, open read-only/read-write, generic open(), save, load.
 
 import shutil
 import tempfile
@@ -35,9 +35,11 @@ class Measurement:
 
 try:
     # -- Create directly on disk (mode="w") ---------------------------------
+    # Extensionless paths default to a directory-backed TreeStore.
     t = blosc2.CTable(Measurement, new_data=data, urlpath=disk_path, mode="w")
     print(f"Created on disk: {len(t):,} rows at '{disk_path}'")
     t.info()
+    t.close()
 
     # -- Open read-only (default) -------------------------------------------
     ro = blosc2.CTable.open(disk_path)  # mode="r" by default
@@ -48,11 +50,18 @@ class Measurement:
         ro.append(Measurement(sensor_id=0, temperature=20.0, day=1))
     except ValueError as e:
         print(f"  Write blocked (read-only): {e}")
+    ro.close()
+
+    # -- Generic open() materializes the CTable -----------------------------
+    opened = blosc2.open(disk_path, mode="r")
+    print(f"Generic open(): {type(opened).__name__} with {len(opened):,} rows")
+    opened.close()
 
     # -- Open read-write and mutate -----------------------------------------
     rw = blosc2.CTable.open(disk_path, mode="a")
     rw.append(Measurement(sensor_id=99, temperature=99.0, day=100))
     print(f"\nAfter append (read-write): {len(rw):,} rows")
+    rw.close()
 
     # -- save(): copy in-memory table to disk -------------------------------
     mem = blosc2.CTable(Measurement, new_data=data[:100])
@@ -62,7 +71,8 @@ class Measurement:
     # -- load(): pull a disk table fully into RAM ---------------------------
     ram = blosc2.CTable.load(disk_path)
     print(f"Loaded into RAM: {len(ram):,} rows  (cbytes={ram.cbytes:,})")
-    assert len(ram) == len(rw)
+    with blosc2.CTable.open(disk_path) as check:
+        assert len(ram) == len(check)
 
 finally:
     shutil.rmtree(tmpdir)

plans/ctable-user-guide.md

@@ -98,8 +98,9 @@ t = b2.CTable(Row, cparams={"codec": b2.Codec.ZSTD, "clevel": 5})
 
 ### Persistent tables
 
-Pass `urlpath` to store the table on disk. The table root is a directory containing
-compressed array files — everything is handled automatically.
+Pass `urlpath` to store the table on disk. Persistent `CTable` is backed by a
+`TreeStore`, and `blosc2.open(urlpath)` can materialize it directly from the
+root `/_meta` manifest.
 
 ```python
 # Create a new persistent table (overwrites any existing table at that path)
@@ -116,6 +117,9 @@ t = b2.CTable.open("people", mode="r")  # explicit
 
 # Open read/write via the classmethod
 t = b2.CTable.open("people", mode="a")
+
+# Generic open() also materializes the richer object
+t = b2.open("people")
 ```
 
 `mode` values:
@@ -129,12 +133,18 @@ t = b2.CTable.open("people", mode="a")
 In-memory tables (`urlpath=None`, the default) behave exactly as before — no
 `mode` or path handling is involved.
 
-### Disk layout
+Recommended conventions:
+
+- extensionless paths default to directory-backed stores
+- `.b2d` and `.b2z` are still valid and useful conventions, but no longer required
+
+### Store layout
 
 ```
-people/
-    _meta.b2frame       ← schema JSON, kind marker, version (in vlmeta)
-    _valid_rows.b2nd    ← tombstone mask
+people/                  ← TreeStore root (extensionless directory-backed example)
+    embed.b2e            ← internal store metadata
+    _meta.b2f            ← SChunk manifest with kind/version/schema in vlmeta
+    _valid_rows.b2nd     ← tombstone mask
     _cols/
         id.b2nd
         score.b2nd
@@ -146,7 +156,8 @@ You can inspect the raw metadata:
 ```python
 import blosc2, json
 
-meta = blosc2.open("people/_meta.b2frame")
+store = blosc2.TreeStore("people", mode="r")
+meta = store["/_meta"]
 print(meta.vlmeta["kind"])  # "ctable"
 print(meta.vlmeta["version"])  # 1
 schema = json.loads(meta.vlmeta["schema"])