"""Chunked, version-keyed cache for million-path PCA universes."""
from __future__ import annotations
import json
import os
from collections.abc import Iterator, Mapping
from dataclasses import asdict
from pathlib import Path
from typing import Any
import numpy as np
from cdxcore.err import verify_inp
from .experiment import ImpliedPCAExperiment
from .universe import PCAUniverseBlock
_UNIVERSE_CACHE_SCHEMA_VERSION = "0.2"
[docs]
class PCAUniverseBlockCache:
"""Persist compact universe arrays one simulation block at a time.
The directory name is derived from the complete versioned experiment, not
from path count or a caller-selected filename. Blocks are never combined
into a second in-memory universe.
"""
def __init__(
self,
root: str | Path,
*,
experiment: ImpliedPCAExperiment,
) -> None:
self.experiment = experiment
self.directory = Path(root) / experiment.filename
self.manifest_path = self.directory / "manifest.json"
self._blocks: list[dict[str, Any]] = []
@property
def _configuration(self) -> dict[str, Any]:
return json.loads(json.dumps(asdict(self.experiment)))
@property
def is_complete(self) -> bool:
if not self.manifest_path.exists():
return False
try:
manifest = json.loads(self.manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return False
if (
manifest.get("status") != "complete"
or manifest.get("cache_uid") != self.experiment.filename
or manifest.get("schema_version") != _UNIVERSE_CACHE_SCHEMA_VERSION
or manifest.get("experiment_code_version")
!= ImpliedPCAExperiment.version.unique_id48
or manifest.get("configuration") != self._configuration
):
return False
blocks = manifest.get("blocks", [])
return (
bool(blocks)
and blocks[0]["start"] == 0
and blocks[-1]["stop"] == self.experiment.num_samples
and all((self.directory / block["file"]).exists() for block in blocks)
)
[docs]
def begin(self) -> None:
self.directory.mkdir(parents=True, exist_ok=True)
self._blocks = []
self._write_manifest(status="writing")
[docs]
def write(self, block: PCAUniverseBlock) -> None:
verify_inp(
0 <= block.start < block.stop <= self.experiment.num_samples,
lambda: f"invalid cached block interval {block.start}:{block.stop}",
)
expected_start = self._blocks[-1]["stop"] if self._blocks else 0
verify_inp(
block.start == expected_start,
lambda: (
f"cached blocks must be contiguous; expected {expected_start}, "
f"got {block.start}"
),
)
rows = block.stop - block.start
for name, value in block.universe.items():
verify_inp(
isinstance(value, np.ndarray)
and np.issubdtype(value.dtype, np.number)
and np.all(np.isfinite(value)),
lambda name=name: (
f"cached universe field {name!r} must be a finite numeric array"
),
)
verify_inp(
name == "expiries" or value.shape[0] == rows,
lambda name=name, value=value: (
f"cached field {name!r} has {value.shape[0]} rows, expected {rows}"
),
)
filename = f"block-{block.start:09d}-{block.stop:09d}.npz"
path = self.directory / filename
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
with temporary.open("wb") as handle:
np.savez_compressed(handle, **block.universe)
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
self._blocks.append(
{
"start": block.start,
"stop": block.stop,
"file": filename,
"fields": sorted(block.universe),
}
)
self._write_manifest(status="writing")
[docs]
def complete(self, *, diagnostics: Mapping[str, Any]) -> None:
verify_inp(
bool(self._blocks)
and self._blocks[-1]["stop"] == self.experiment.num_samples,
"cannot complete an incomplete PCA universe cache",
)
self._write_manifest(status="complete", diagnostics=diagnostics)
def _write_manifest(
self,
*,
status: str,
diagnostics: Mapping[str, Any] | None = None,
) -> None:
value: dict[str, Any] = {
"status": status,
"schema_version": _UNIVERSE_CACHE_SCHEMA_VERSION,
"cache_uid": self.experiment.filename,
"experiment_code_version": (
ImpliedPCAExperiment.version.unique_id48
),
"configuration": self._configuration,
"blocks": self._blocks,
"market_scope": {
"ticker": "SPX",
"start": "2020-01-01",
"end": "2025-08-30",
"PCA_input": "mid_vols",
},
}
if diagnostics is not None:
value["diagnostics"] = diagnostics
temporary = self.manifest_path.with_name(
f".{self.manifest_path.name}.{os.getpid()}.tmp"
)
try:
temporary.write_text(
json.dumps(value, indent=2, allow_nan=False), encoding="utf-8"
)
os.replace(temporary, self.manifest_path)
finally:
temporary.unlink(missing_ok=True)
[docs]
def manifest(self) -> dict[str, Any]:
verify_inp(self.is_complete, "PCA universe cache is not complete")
return json.loads(self.manifest_path.read_text(encoding="utf-8"))
[docs]
def update_diagnostics(self, diagnostics: Mapping[str, Any]) -> None:
"""Atomically attach completed downstream diagnostics to this cache."""
manifest = self.manifest()
manifest["diagnostics"] = diagnostics
temporary = self.manifest_path.with_name(
f".{self.manifest_path.name}.{os.getpid()}.tmp"
)
try:
temporary.write_text(
json.dumps(manifest, indent=2, allow_nan=False), encoding="utf-8"
)
os.replace(temporary, self.manifest_path)
finally:
temporary.unlink(missing_ok=True)
[docs]
def iter_blocks(self) -> Iterator[PCAUniverseBlock]:
"""Load one compact cached block at a time with pickle disabled."""
manifest = self.manifest()
for block in manifest["blocks"]:
with np.load(
self.directory / block["file"], allow_pickle=False
) as state:
universe = {name: state[name] for name in state.files}
yield PCAUniverseBlock(
start=int(block["start"]),
stop=int(block["stop"]),
universe=universe,
)
__all__ = ["PCAUniverseBlockCache"]