"""Checkpoint writer/reader for pipeline state persistence. Each stage writes a checkpoint after completion. The orchestrator uses checkpoints to resume pipelines and to present state at human checkpoints. """ from __future__ import annotations import json from functools import lru_cache from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional import jsonschema from schemas.artifacts import ARTIFACT_NAMES, validate_artifact STAGES = ["research", "proposal", "idea", "script", "scene_plan", "assets", "edit", "compose", "publish"] CANONICAL_STAGE_ARTIFACTS = { "research": "research_brief", "proposal": "proposal_packet", "idea": "brief", "script": "script", "scene_plan": "scene_plan", "assets": "asset_manifest", "edit": "edit_decisions", "compose": "render_report", "publish": "publish_log", } CHECKPOINT_SCHEMA_PATH = ( Path(__file__).resolve().parent.parent / "schemas" / "checkpoints" / "checkpoint.schema.json" ) class CheckpointValidationError(ValueError): """Raised when a checkpoint or its canonical artifacts are invalid.""" @lru_cache(maxsize=1) def _load_checkpoint_schema() -> dict[str, Any]: with open(CHECKPOINT_SCHEMA_PATH) as f: return json.load(f) def _validate_artifacts_for_stage( stage: str, status: str, artifacts: dict[str, Any], ) -> None: required_artifact = CANONICAL_STAGE_ARTIFACTS[stage] if status in {"completed", "awaiting_human"} and required_artifact not in artifacts: raise CheckpointValidationError( f"Stage {stage!r} with status {status!r} must include " f"canonical artifact {required_artifact!r}" ) for artifact_name, artifact_data in artifacts.items(): if artifact_name not in ARTIFACT_NAMES: continue if not isinstance(artifact_data, dict): raise CheckpointValidationError( f"Artifact {artifact_name!r} must be a JSON object matching its schema" ) try: validate_artifact(artifact_name, artifact_data) except Exception as exc: raise CheckpointValidationError( f"Artifact {artifact_name!r} failed schema validation: {exc}" ) from exc def validate_checkpoint(checkpoint: dict[str, Any]) -> None: """Validate checkpoint structure and canonical artifact payloads.""" stage = checkpoint.get("stage") status = checkpoint.get("status") artifacts = checkpoint.get("artifacts") if not isinstance(stage, str) or stage not in STAGES: raise CheckpointValidationError(f"Invalid stage: {stage!r}") if not isinstance(status, str): raise CheckpointValidationError(f"Invalid status: {status!r}") if not isinstance(artifacts, dict): raise CheckpointValidationError("Checkpoint artifacts must be a dictionary") _validate_artifacts_for_stage(stage, status, artifacts) try: jsonschema.validate(instance=checkpoint, schema=_load_checkpoint_schema()) except jsonschema.ValidationError as exc: raise CheckpointValidationError(f"Checkpoint failed schema validation: {exc.message}") from exc def _checkpoint_path(pipeline_dir: Path, project_id: str, stage: str) -> Path: return pipeline_dir / project_id / f"checkpoint_{stage}.json" def write_checkpoint( pipeline_dir: Path, project_id: str, stage: str, status: str, artifacts: dict[str, Any], *, pipeline_type: Optional[str] = None, style_playbook: Optional[str] = None, checkpoint_policy: str = "guided", human_approval_required: bool = False, human_approved: bool = False, review: Optional[dict] = None, cost_snapshot: Optional[dict] = None, error: Optional[str] = None, metadata: Optional[dict] = None, ) -> Path: """Write a checkpoint file for a pipeline stage.""" if stage not in STAGES: raise ValueError(f"Invalid stage: {stage!r}. Must be one of {STAGES}") checkpoint = { "version": "1.0", "project_id": project_id, "stage": stage, "status": status, "timestamp": datetime.now(timezone.utc).isoformat(), "checkpoint_policy": checkpoint_policy, "human_approval_required": human_approval_required, "human_approved": human_approved, "artifacts": artifacts, } if pipeline_type is not None: checkpoint["pipeline_type"] = pipeline_type if style_playbook is not None: checkpoint["style_playbook"] = style_playbook if review is not None: checkpoint["review"] = review if cost_snapshot is not None: checkpoint["cost_snapshot"] = cost_snapshot if error is not None: checkpoint["error"] = error if metadata is not None: checkpoint["metadata"] = metadata validate_checkpoint(checkpoint) path = _checkpoint_path(pipeline_dir, project_id, stage) path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: json.dump(checkpoint, f, indent=2) return path def read_checkpoint( pipeline_dir: Path, project_id: str, stage: str ) -> Optional[dict[str, Any]]: """Read a checkpoint file. Returns None if not found.""" path = _checkpoint_path(pipeline_dir, project_id, stage) if not path.exists(): return None with open(path) as f: checkpoint = json.load(f) validate_checkpoint(checkpoint) return checkpoint def get_latest_checkpoint( pipeline_dir: Path, project_id: str ) -> Optional[dict[str, Any]]: """Find the most recent checkpoint for a project (by file mtime).""" project_dir = pipeline_dir / project_id if not project_dir.exists(): return None checkpoints = sorted( project_dir.glob("checkpoint_*.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) if not checkpoints: return None with open(checkpoints[0]) as f: checkpoint = json.load(f) validate_checkpoint(checkpoint) return checkpoint def get_completed_stages(pipeline_dir: Path, project_id: str) -> list[str]: """Return list of stages that have a completed checkpoint.""" completed = [] for stage in STAGES: cp = read_checkpoint(pipeline_dir, project_id, stage) if cp and cp.get("status") == "completed": completed.append(stage) return completed def get_next_stage(pipeline_dir: Path, project_id: str) -> Optional[str]: """Determine the next stage to run based on completed checkpoints.""" completed = set(get_completed_stages(pipeline_dir, project_id)) for stage in STAGES: if stage not in completed: return stage return None