karenina.benchmark

benchmark

Benchmark module for Karenina verification system.

Classes

Benchmark

Main class for managing Karenina benchmarks in JSON-LD format.

This class provides a high-level API for: - Creating benchmarks manually or automatically - Loading/saving JSON-LD benchmark files - Running verification with existing execution system - Full compatibility with frontend GUI exports

This is a facade that delegates to specialized manager classes for better maintainability.

Source code in src/karenina/benchmark/benchmark.py

class Benchmark:
    """
    Main class for managing Karenina benchmarks in JSON-LD format.

    This class provides a high-level API for:
    - Creating benchmarks manually or automatically
    - Loading/saving JSON-LD benchmark files
    - Running verification with existing execution system
    - Full compatibility with frontend GUI exports

    This is a facade that delegates to specialized manager classes for better maintainability.
    """

    def __init__(
        self,
        name: str,
        description: str = "",
        version: str = "0.1.0",
        creator: str = "Karenina Benchmarking System",
        workspace_root: Path | None = None,
    ):
        """
        Initialize a new benchmark.

        Args:
            name: Name of the benchmark
            description: Description of the benchmark
            version: Version of the benchmark content
            creator: Creator name or organization
            workspace_root: Root directory containing task workspaces.
                Question workspace paths are resolved relative to this root.
                Not persisted in the checkpoint (it is a local filesystem path).
        """
        self._base = BenchmarkBase(name, description, version, creator)
        self._workspace_root = workspace_root
        self._scenarios: dict[str, ScenarioDefinition] = {}
        self._metadata_manager = MetadataManager(self._base)
        self._question_manager = QuestionManager(self._base)
        self._rubric_manager = RubricManager(self._base)
        self._template_manager = TemplateManager(self._base)
        self._results_manager = ResultsManager(self._base)
        self._verification_manager = VerificationManager(self._base, self._rubric_manager)
        self._export_manager = ExportManager(self._base, self._template_manager, self._rubric_manager)

    def _init_managers(self) -> None:
        """Initialize all managers from self._base (used by load/clone)."""
        if not hasattr(self, "_scenarios"):
            self._scenarios = {}
        self._metadata_manager = MetadataManager(self._base)
        self._question_manager = QuestionManager(self._base)
        self._rubric_manager = RubricManager(self._base)
        self._template_manager = TemplateManager(self._base)
        self._results_manager = ResultsManager(self._base)
        self._verification_manager = VerificationManager(self._base, self._rubric_manager)
        self._export_manager = ExportManager(self._base, self._template_manager, self._rubric_manager)
        self._rebuild_scenarios()

    def _rebuild_scenarios(self) -> None:
        """Rebuild _scenarios cache from checkpoint hasPart data."""
        from ..scenario.checkpoint import schema_org_to_scenario

        has_part = self._base._checkpoint.hasPart
        if not has_part:
            return

        # Validate homogeneity: cannot have both questions and scenarios
        if self._base._questions_cache:
            raise ValueError(
                "Checkpoint contains both questions and scenarios; this is not supported. "
                "A benchmark must contain either standalone questions or scenarios, not both."
            )

        for schema_org in has_part:
            defn = schema_org_to_scenario(schema_org)
            self._scenarios[defn.name] = defn

    @property
    def workspace_root(self) -> Path | None:
        """Root directory for task workspaces (not persisted in checkpoint)."""
        return self._workspace_root

    def set_workspace_root(self, path: Path) -> None:
        """Set the root directory for task workspaces.

        Args:
            path: Directory containing task workspace subdirectories.
                Question workspace paths are resolved relative to this root.
        """
        self._workspace_root = path

    @classmethod
    def create(
        cls,
        name: str,
        description: str = "",
        version: str = "0.1.0",
        creator: str = "Karenina Benchmarking System",
        workspace_root: Path | None = None,
    ) -> "Benchmark":
        """Create a new benchmark (alias for constructor)."""
        return cls(name, description, version, creator, workspace_root=workspace_root)

    @classmethod
    def load(cls, path: Path, workspace_root: Path | None = None) -> "Benchmark":
        """Load a benchmark from a JSON-LD file.

        Args:
            path: Path to the JSON-LD benchmark file.
            workspace_root: Optional root directory for task workspaces.
        """
        base = BenchmarkBase.load(path)
        instance = cls.__new__(cls)
        instance._base = base
        instance._workspace_root = workspace_root
        instance._init_managers()
        return instance

    def save(self, path: Path, save_deep_judgment_config: bool = False) -> None:
        """Save the benchmark to a JSON-LD file.

        Args:
            path: Path where to save the benchmark.
            save_deep_judgment_config: If True, include deep judgment
                configuration in LLM rubric traits. If False (default),
                deep judgment settings are stripped before saving.
        """
        self._base.save(path, save_deep_judgment_config=save_deep_judgment_config)

    def save_to_db(self, storage: str, checkpoint_path: Path | None = None) -> "Benchmark":
        """Save this benchmark to a database."""
        from typing import cast

        from ..storage import save_benchmark

        result = save_benchmark(self, storage, checkpoint_path)
        return cast("Benchmark", result)

    @classmethod
    def load_from_db(cls, benchmark_name: str, storage: str) -> "Benchmark":
        """Load a benchmark from a database."""
        from ..storage import load_benchmark

        result = load_benchmark(benchmark_name, storage, load_config=False)
        return result  # type: ignore[return-value]

    # ── Question management ──────────────────────────────────────────────

    def add_question(
        self,
        question: Union[str, dict[str, Any], "Question"],
        raw_answer: str | None = None,
        answer_template: str | type | None = None,
        question_id: str | None = None,
        finished: bool | object = _NOT_PROVIDED,
        author: dict[str, Any] | None = None,
        sources: list[dict[str, Any]] | None = None,
        custom_metadata: dict[str, Any] | None = None,
        few_shot_examples: list[dict[str, str]] | None = None,
        answer_notes: str | None = None,
    ) -> str:
        """Add a question to the benchmark.

        Accepts a question string, a Question object, or a dict with keys
        ``question`` and ``raw_answer`` (plus any optional kwargs).

        Raises:
            ValueError: If scenarios already exist (homogeneous enforcement).
        """
        if self._scenarios:
            raise ValueError(
                "Cannot add standalone questions to a scenario benchmark. "
                "Scenarios and standalone questions cannot coexist in the same benchmark."
            )
        return self._question_manager.add_question(
            question,
            raw_answer,
            answer_template,
            question_id,
            finished,
            author,
            sources,
            custom_metadata,
            few_shot_examples,
            answer_notes=answer_notes,
        )

    def add_questions(
        self,
        questions_data: "Sequence[dict[str, Any] | Question]",
        finished: bool | object = _NOT_PROVIDED,
    ) -> list[str]:
        """Add multiple questions at once.

        Accepts a list of dicts, Question objects, or a mix of both.

        Args:
            questions_data: List of dicts or Question objects.
            finished: Default finished state for items that do not define one.
                Pass False when importing items for template drafting and
                curator review. Omission preserves the existing default.

        Returns:
            List of question IDs that were created.

        Raises:
            ValueError: If scenarios already exist (homogeneous enforcement).
        """
        if self._scenarios:
            raise ValueError(
                "Cannot add standalone questions to a scenario benchmark. "
                "Scenarios and standalone questions cannot coexist in the same benchmark."
            )
        return self._question_manager.add_questions(questions_data, finished=finished)

    def get_question_ids(self) -> list[str]:
        """Get all question IDs in the benchmark."""
        return self._question_manager.get_question_ids()

    def get_question(self, question_id: str) -> dict[str, Any]:
        """Get a question by ID."""
        return self._question_manager.get_question(question_id)

    def get_all_questions(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
        """Get all questions in the benchmark."""
        return self._question_manager.get_all_questions(ids_only)

    def get_question_as_object(self, question_id: str) -> "Question":
        """Get a question as a Question object."""
        return self._question_manager.get_question_as_object(question_id)

    def get_all_questions_as_objects(self) -> list["Question"]:
        """Get all questions as Question objects."""
        return self._question_manager.get_all_questions_as_objects()

    def add_question_from_object(self, question_obj: "Question", **metadata: Any) -> str:
        """Add a question to the benchmark from a Question object."""
        return self._question_manager.add_question_from_object(question_obj, **metadata)

    def update_question_metadata(self, question_id: str, **metadata: Any) -> None:
        """Update question metadata fields."""
        self._question_manager.update_question_metadata(question_id, **metadata)

    def get_question_metadata(self, question_id: str) -> dict[str, Any]:
        """Get all metadata for a specific question."""
        return self._question_manager.get_question_metadata(question_id)

    def get_question_custom_property(self, question_id: str, name: str) -> Any:
        """Get a custom property from question metadata."""
        return self._question_manager.get_question_custom_property(question_id, name)

    def set_question_custom_property(self, question_id: str, name: str, value: Any) -> None:
        """Set a custom property on question metadata."""
        self._question_manager.set_question_custom_property(question_id, name, value)

    def remove_question_custom_property(self, question_id: str, name: str) -> bool:
        """Remove a custom property from question metadata."""
        return self._question_manager.remove_question_custom_property(question_id, name)

    def get_question_author(self, question_id: str) -> dict[str, Any] | None:
        """Get author information for a question."""
        return self._question_manager.get_question_author(question_id)

    def set_question_author(self, question_id: str, author: dict[str, Any] | None) -> None:
        """Set author information for a question."""
        self._question_manager.set_question_author(question_id, author)

    def get_question_sources(self, question_id: str) -> list[dict[str, Any]] | None:
        """Get source documents for a question."""
        return self._question_manager.get_question_sources(question_id)

    def set_question_sources(self, question_id: str, sources: list[dict[str, Any]] | None) -> None:
        """Set source documents for a question."""
        self._question_manager.set_question_sources(question_id, sources)

    def get_question_timestamps(self, question_id: str) -> dict[str, str]:
        """Get creation and modification timestamps for a question."""
        return self._question_manager.get_question_timestamps(question_id)

    def remove_question(self, question_id: str) -> bool:
        """Remove a specific question from the benchmark."""
        return self._question_manager.remove_question(question_id)

    def clear_questions(self) -> int:
        """Remove all questions from the benchmark."""
        return self._question_manager.clear_questions()

    def add_questions_batch(
        self,
        questions_data: "Sequence[dict[str, Any] | Question]",
        finished: bool | object = _NOT_PROVIDED,
    ) -> list[str]:
        """Add multiple questions at once with an optional finished state."""
        return self._question_manager.add_questions_batch(questions_data, finished=finished)

    def mark_finished(self, question_id: str) -> None:
        """Mark a question as finished."""
        self._question_manager.mark_finished(question_id)

    def mark_unfinished(self, question_id: str) -> None:
        """Mark a question as unfinished."""
        self._question_manager.mark_unfinished(question_id)

    def mark_finished_batch(self, question_ids: list[str]) -> None:
        """Mark multiple questions as finished."""
        self._question_manager.mark_finished_batch(question_ids)

    def mark_unfinished_batch(self, question_ids: list[str]) -> None:
        """Mark multiple questions as unfinished."""
        self._question_manager.mark_unfinished_batch(question_ids)

    def toggle_finished(self, question_id: str) -> bool:
        """Toggle finished status of a question."""
        return self._question_manager.toggle_finished(question_id)

    def get_unfinished_questions(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
        """Get questions that are not marked as finished."""
        return self._question_manager.get_unfinished_questions(ids_only)

    def get_finished_questions(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
        """Get questions that are marked as finished."""
        return self._question_manager.get_finished_questions(ids_only)

    def filter_questions(
        self,
        finished: bool | None = None,
        has_template: bool | None = None,
        has_rubric: bool | None = None,
        author: str | None = None,
        custom_filter: Any = None,
    ) -> list[dict[str, Any]]:
        """Filter questions based on criteria."""
        return self._question_manager.filter_questions(finished, has_template, has_rubric, author, custom_filter)

    def filter_by_metadata(self, field_path: str, value: Any, match_mode: str = "exact") -> list[dict[str, Any]]:
        """Filter questions by a metadata field using dot notation."""
        return self._question_manager.filter_by_metadata(field_path, value, match_mode)

    def filter_by_custom_metadata(self, match_all: bool = True, **criteria: Any) -> list[dict[str, Any]]:
        """Filter questions by custom metadata fields with AND/OR logic."""
        return self._question_manager.filter_by_custom_metadata(match_all, **criteria)

    def search_questions(
        self,
        query: str | list[str],
        match_all: bool = True,
        fields: list[str] | None = None,
        case_sensitive: bool = False,
        regex: bool = False,
    ) -> list[dict[str, Any]]:
        """Search for questions containing the query text (unified search method)."""
        return self._question_manager.search_questions(query, match_all, fields, case_sensitive, regex)

    def get_questions_by_author(self, author: str) -> list[dict[str, Any]]:
        """Get questions created by a specific author."""
        return self._question_manager.get_questions_by_author(author)

    def get_questions_with_rubric(self) -> list[dict[str, Any]]:
        """Get questions that have question-specific rubrics."""
        return self._question_manager.get_questions_with_rubric()

    def count_by_field(self, field_path: str, questions: list[dict[str, Any]] | None = None) -> dict[Any, int]:
        """Count questions grouped by a field value using dot notation."""
        return self._question_manager.count_by_field(field_path, questions)

    # ── Scenario management ─────────────────────────────────────────────

    @property
    def is_scenario_benchmark(self) -> bool:
        """True if this benchmark contains scenarios instead of standalone questions."""
        return len(self._scenarios) > 0

    @property
    def scenario_count(self) -> int:
        """Return the number of scenarios in the benchmark."""
        return len(self._scenarios)

    def add_scenario(self, scenario: "ScenarioDefinition | Any") -> None:
        """Add a scenario to the benchmark.

        Accepts either a ScenarioDefinition (frozen) or a Scenario builder
        (which will be validated and frozen automatically).

        Args:
            scenario: A ScenarioDefinition or a Scenario builder instance.

        Raises:
            ValueError: If standalone questions already exist (homogeneous enforcement),
                or if a scenario with the same name already exists.
        """
        if self._base._questions_cache:
            raise ValueError(
                "Cannot add scenarios to a benchmark that already contains standalone questions. "
                "Scenarios and standalone questions cannot coexist in the same benchmark."
            )

        # Accept Scenario builder: call validate() to get a ScenarioDefinition
        if not isinstance(scenario, ScenarioDefinition):
            scenario = scenario.validate()

        if scenario.name in self._scenarios:
            raise ValueError(f"Scenario '{scenario.name}' already exists")

        self._scenarios[scenario.name] = scenario

        # Write to checkpoint (checkpoint is source of truth)
        from ..scenario.checkpoint import scenario_to_schema_org
        from ..schemas.checkpoint import SchemaOrgPropertyValue

        schema_org = scenario_to_schema_org(scenario)
        if self._base._checkpoint.hasPart is None:
            self._base._checkpoint.hasPart = []
        self._base._checkpoint.hasPart.append(schema_org)

        # Set benchmark_type flag (once)
        props = self._base._checkpoint.additionalProperty or []
        if not any(p.name == "benchmark_type" for p in props):
            if self._base._checkpoint.additionalProperty is None:
                self._base._checkpoint.additionalProperty = []
            self._base._checkpoint.additionalProperty.append(
                SchemaOrgPropertyValue(name="benchmark_type", value="scenario")
            )

    def get_scenarios(self) -> list[ScenarioDefinition]:
        """Get all scenario definitions.

        Returns:
            List of ScenarioDefinition instances.
        """
        return list(self._scenarios.values())

    def get_scenario(self, name: str) -> ScenarioDefinition:
        """Get a scenario by name.

        Args:
            name: The scenario name.

        Returns:
            The ScenarioDefinition.

        Raises:
            KeyError: If no scenario with that name exists.
        """
        try:
            return self._scenarios[name]
        except KeyError:
            raise KeyError(f"Scenario '{name}' not found") from None

    def remove_scenario(self, name: str) -> None:
        """Remove a scenario by name.

        Args:
            name: The scenario name.

        Raises:
            KeyError: If no scenario with that name exists.
        """
        try:
            del self._scenarios[name]
        except KeyError:
            raise KeyError(f"Scenario '{name}' not found") from None

        # Remove from checkpoint
        if self._base._checkpoint.hasPart:
            self._base._checkpoint.hasPart = [s for s in self._base._checkpoint.hasPart if s.name != name]
            if not self._base._checkpoint.hasPart:
                self._base._checkpoint.hasPart = None
                # Clear benchmark_type flag when no scenarios remain
                if self._base._checkpoint.additionalProperty:
                    self._base._checkpoint.additionalProperty = [
                        p for p in self._base._checkpoint.additionalProperty if p.name != "benchmark_type"
                    ]

    # ── Template management ──────────────────────────────────────────────

    def add_answer_template(self, question_id: str, template_code: str) -> None:
        """Add or update an answer template for a question."""
        self._template_manager.add_answer_template(question_id, template_code)

    def has_template(self, question_id: str) -> bool:
        """Check if a question has a non-default template."""
        return self._template_manager.has_template(question_id)

    def get_template(self, question_id: str) -> str:
        """Get template code for a question."""
        return self._template_manager.get_template(question_id)

    def update_template(self, question_id: str, template_code: str | type) -> None:
        """Update existing template.

        Args:
            question_id: The question ID
            template_code: Python code defining the Answer class, or a BaseAnswer subclass
        """
        self._template_manager.update_template(question_id, template_code)

    def copy_template(self, from_id: str, to_id: str) -> None:
        """Copy template from one question to another."""
        self._template_manager.copy_template(from_id, to_id)

    def get_finished_templates(self, question_ids: set[str] | None = None) -> list[FinishedTemplate]:
        """Get all finished templates for verification."""
        return self._template_manager.get_finished_templates(question_ids=question_ids)

    def get_missing_templates(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
        """Get questions that don't have non-default templates."""
        return self._template_manager.get_missing_templates(ids_only)

    def apply_global_template(self, template_code: str) -> list[str]:
        """Apply a template to all questions that don't have one."""
        return self._template_manager.apply_global_template(template_code)

    def validate_templates(self) -> tuple[bool, list[dict[str, str]]]:
        """Validate all templates are valid Python code."""
        return self._template_manager.validate_templates()

    # ── Template generation (delegated to benchmark_helpers) ─────────────

    def generate_template_for_question(
        self,
        question_id: str,
        model: str = "claude-haiku-4-5",
        model_provider: str = "anthropic",
        temperature: float = 0,
        interface: str = "langchain",
        force_regenerate: bool = False,
        endpoint_base_url: str | None = None,
        endpoint_api_key: str | None = None,
    ) -> dict[str, Any]:
        """Generate an answer template for a specific question using LLM."""
        return _helpers.generate_template_for_question(
            self,
            question_id,
            model,
            model_provider,
            temperature,
            interface,
            force_regenerate,
            endpoint_base_url,
            endpoint_api_key,
        )

    def generate_templates(
        self,
        question_ids: list[str],
        model: str = "claude-haiku-4-5",
        model_provider: str = "anthropic",
        temperature: float = 0,
        interface: str = "langchain",
        force_regenerate: bool = False,
        progress_callback: "Callable[[_helpers.TemplateProgressEvent], None] | None" = None,
        endpoint_base_url: str | None = None,
        endpoint_api_key: str | None = None,
        max_workers: int | None = None,
        cancel_event: "threading.Event | None" = None,
    ) -> dict[str, dict[str, Any]]:
        """Generate templates for multiple questions using LLM."""
        return _helpers.generate_templates(
            self,
            question_ids,
            model,
            model_provider,
            temperature,
            interface,
            force_regenerate,
            progress_callback,
            endpoint_base_url,
            endpoint_api_key,
            max_workers=max_workers,
            cancel_event=cancel_event,
        )

    def generate_all_templates(
        self,
        model: str = "claude-haiku-4-5",
        model_provider: str = "anthropic",
        temperature: float = 0,
        interface: str = "langchain",
        force_regenerate: bool = False,
        progress_callback: "Callable[[_helpers.TemplateProgressEvent], None] | None" = None,
        only_missing: bool = True,
        endpoint_base_url: str | None = None,
        endpoint_api_key: str | None = None,
        progressive_backup: bool = True,
        backup_path: Path | None = None,
        max_workers: int | None = None,
        cancel_event: "threading.Event | None" = None,
    ) -> dict[str, dict[str, Any]]:
        """Generate templates for all questions in the benchmark using LLM.

        Args:
            model: Model name.
            model_provider: Model provider.
            temperature: Generation temperature.
            interface: Adapter interface.
            force_regenerate: If True, regenerate existing templates.
            progress_callback: Optional progress callback.
            only_missing: If True, only generate for questions without templates.
            endpoint_base_url: Optional custom endpoint URL.
            endpoint_api_key: Optional API key for custom endpoint.
            progressive_backup: If True (default), save generated templates to a
                backup file after each successful generation so interrupted runs
                can be resumed.
            backup_path: Path for the backup file. Defaults to
                ``{benchmark_name}_templates_backup.json`` in the current directory.
            max_workers: Number of parallel workers. None reads from
                KARENINA_ASYNC_MAX_WORKERS env var (default 1). 1 = sequential.
            cancel_event: If set, stops generation after the current task(s) complete.
        """
        return _helpers.generate_all_templates(
            self,
            model,
            model_provider,
            temperature,
            interface,
            force_regenerate,
            progress_callback,
            only_missing,
            endpoint_base_url,
            endpoint_api_key,
            progressive_backup,
            backup_path,
            max_workers=max_workers,
            cancel_event=cancel_event,
        )

    def export_generated_templates(self, file_path: Path) -> None:
        """Export all generated templates to a JSON file."""
        _helpers.export_generated_templates(self, file_path)

    def import_generated_templates(self, file_path: Path, force_overwrite: bool = False) -> dict[str, bool]:
        """Import templates from a JSON file generated by export_generated_templates."""
        return _helpers.import_generated_templates(self, file_path, force_overwrite)

    # ── Rubric management ────────────────────────────────────────────────

    def add_global_rubric_trait(
        self, trait: LLMRubricTrait | RegexRubricTrait | CallableRubricTrait | MetricRubricTrait | AgenticRubricTrait
    ) -> None:
        """Add a global rubric trait to the benchmark."""
        self._rubric_manager.add_global_rubric_trait(trait)

    def add_question_rubric_trait(
        self,
        question_id: str,
        trait: LLMRubricTrait | RegexRubricTrait | CallableRubricTrait | MetricRubricTrait | AgenticRubricTrait,
    ) -> None:
        """Add a question-specific rubric trait."""
        self._rubric_manager.add_question_rubric_trait(question_id, trait)

    def set_global_rubric(self, rubric: Rubric) -> None:
        """Set the complete global rubric (replaces existing)."""
        self._rubric_manager.set_global_rubric(rubric)

    def set_question_rubric(self, question_id: str, rubric: Rubric) -> None:
        """Set the complete question-specific rubric (replaces existing)."""
        self._rubric_manager.set_question_rubric(question_id, rubric)

    def get_global_rubric(self) -> Rubric | None:
        """Get the global rubric from the benchmark."""
        return self._rubric_manager.get_global_rubric()

    def get_question_rubric(self, question_id: str) -> Rubric | None:
        """Get the question-specific rubric for a question.

        Args:
            question_id: The question ID.

        Returns:
            Rubric containing the question-specific traits, or None.
        """
        raw = self._rubric_manager.get_question_rubric(question_id)
        if raw is None:
            return None
        if isinstance(raw, dict):
            return Rubric(
                llm_traits=raw.get("llm_traits", []),
                regex_traits=raw.get("regex_traits", []),
                callable_traits=raw.get("callable_traits", []),
                metric_traits=raw.get("metric_traits", []),
                agentic_traits=raw.get("agentic_traits", []),
            )
        return Rubric.from_traits(raw)

    def clear_global_rubric(self) -> bool:
        """Remove the global rubric."""
        return self._rubric_manager.clear_global_rubric()

    def remove_question_rubric(self, question_id: str) -> bool:
        """Remove question-specific rubric."""
        return self._rubric_manager.remove_question_rubric(question_id)

    def clear_all_rubrics(self) -> int:
        """Remove all rubrics (global and question-specific)."""
        return self._rubric_manager.clear_all_rubrics()

    def validate_rubrics(self) -> tuple[bool, list[dict[str, str]]]:
        """Validate all rubrics are properly configured."""
        return self._rubric_manager.validate_rubrics()

    # ── Dynamic rubric management ──────────────────────────────────────

    def get_global_dynamic_rubric(self) -> DynamicRubric | None:
        """Get the global dynamic rubric from the benchmark."""
        return self._rubric_manager.get_global_dynamic_rubric()

    def set_global_dynamic_rubric(self, dynamic_rubric: DynamicRubric | None) -> None:
        """Set or clear the global dynamic rubric.

        Persists the rubric to the checkpoint so it survives save/load cycles.

        Args:
            dynamic_rubric: The DynamicRubric to set, or None to clear.
        """
        self._base._global_dynamic_rubric = dynamic_rubric
        if dynamic_rubric is not None:
            self._rubric_manager.set_global_dynamic_rubric_in_checkpoint(dynamic_rubric)
        else:
            # Clear from checkpoint: remove dynamic rubric ratings
            if self._base._checkpoint.rating:
                self._base._checkpoint.rating = [
                    r for r in self._base._checkpoint.rating if r.additionalType != "karenina:GlobalDynamicRubricTrait"
                ]

    def get_merged_dynamic_rubric_for_question(self, question_id: str) -> DynamicRubric | None:
        """Get merged dynamic rubric for a question (global + question-specific).

        Args:
            question_id: The question ID.

        Returns:
            Merged DynamicRubric or None if neither global nor question-level exists.
        """
        return self._rubric_manager.get_merged_dynamic_rubric_for_question(question_id)

    # ── Verification ─────────────────────────────────────────────────────

    def run_verification(
        self,
        config: VerificationConfig,
        question_ids: list[str] | None = None,
        run_name: str | None = None,
        async_enabled: bool | None = None,
        progress_callback: Callable[[float, str], None] | None = None,
        sink: Any = None,
    ) -> VerificationResultSet:
        """Run verification on the benchmark using existing execution system.

        For scenario benchmarks, dispatches to ``_run_scenario_verification``
        which iterates over the scenario x model cross-product.
        For standalone question benchmarks, delegates to VerificationManager.

        Args:
            config: Verification configuration.
            question_ids: Optional filter for which questions to run.
            run_name: Optional run label.
            async_enabled: Whether to run in parallel.
            progress_callback: Optional UI progress callback.
            sink: Optional :class:`ResultSink` for progressive save and
                crash recovery. Pass a :class:`ProgressiveFileSink` to get
                ``--resume``-compatible sidecars written incrementally, or a
                :class:`CompositeSink` to combine multiple persistence
                strategies. Scenario mode stores per-turn verification rows
                and skips already-completed scenario/model/parser combos on
                resume.
        """
        # Auto-build replay store from legacy interface="manual" models.
        # The benchmark is only reachable here (not from VerificationConfig
        # validators), so this is the right layer for the translation.
        # The resulting store is strict so existing manual semantics are
        # preserved bit-identically.
        if config.replay_store is None:
            manual_model = next(
                (m for m in config.answering_models if getattr(m, "interface", None) == "manual"),
                None,
            )
            if manual_model is not None and getattr(manual_model, "manual_traces", None) is not None:
                from karenina.replay import ReplayStore

                config = config.model_copy(
                    update={
                        "replay_store": ReplayStore.from_manual_traces(
                            manual_model.manual_traces,
                            benchmark=self,
                            miss_policy="strict",
                        ),
                    }
                )

        if self.is_scenario_benchmark:
            return self._run_scenario_verification(
                config=config,
                run_name=run_name,
                async_enabled=async_enabled,
                progress_callback=progress_callback,
                sink=sink,
            )
        return self._verification_manager.run_verification(
            config,
            question_ids,
            run_name,
            async_enabled,
            progress_callback,
            workspace_root=self._workspace_root,
            sink=sink,
        )

    def resume_verification(
        self,
        state_path: "str | Path",
        *,
        config: VerificationConfig | None = None,
        question_ids: list[str] | None = None,
        run_name: str | None = None,
        async_enabled: bool | None = None,
        progress_callback: Callable[[float, str], None] | None = None,
    ) -> VerificationResultSet:
        """Resume a progressive-save run from its ``.state`` file.

        Reconstructs a :class:`ProgressiveFileSink` from ``state_path``,
        populates ``config.skip_triples`` via the sink, and delegates to
        :meth:`run_verification`. The config stored in the state file is
        used unless ``config`` is provided explicitly (useful for tweaking
        e.g. ``request_timeout`` before the retry pass).

        Args:
            state_path: Path to the ``.state`` sidecar from a prior run.
            config: Optional override config; defaults to the one snapshot
                in the state file.
            question_ids: Optional subset of questions.
            run_name: Optional run label (defaults to the state's label).
            async_enabled: Override for async execution.
            progress_callback: UI progress callback.

        Returns:
            The batch's :class:`VerificationResultSet`. Previously-completed
            results remain in the sink's buffer; the returned set reflects
            whatever the executor produced in this resume pass.
        """
        from karenina.benchmark.verification.sinks import ProgressiveFileSink

        sink = ProgressiveFileSink.load_for_resume(Path(state_path))
        sink.set_global_rubric(self._rubric_manager.get_global_rubric())

        effective_config = config if config is not None else sink.config
        return self.run_verification(
            config=effective_config,
            question_ids=question_ids,
            run_name=run_name,
            async_enabled=async_enabled,
            progress_callback=progress_callback,
            sink=sink,
        )

    def _run_scenario_verification(
        self,
        config: VerificationConfig,
        run_name: str | None = None,
        async_enabled: bool | None = None,
        progress_callback: Callable[[float, str], None] | None = None,
        sink: Any = None,
    ) -> VerificationResultSet:
        """Run verification for scenario benchmarks.

        Delegates to ScenarioExecutor for parallel/sequential dispatch,
        answer caching, and global LLM semaphore management.

        Args:
            config: Verification configuration.
            run_name: Optional run name for tracking.
            async_enabled: If True, run combinations in parallel.
            progress_callback: Optional callback for progress updates.
            sink: Optional :class:`ResultSink` for progressive save / resume.
                Combo-atomic: each scenario combo (scenario_id, ans, parse,
                replicate) is persisted as a single completed task once its
                turn_results are emitted. Interrupted combos re-run from
                turn 1 on resume.

        Returns:
            VerificationResultSet containing all per-turn results.
        """
        from ..benchmark.verification.model_stamping import stamp_pipeline_defaults
        from ..benchmark.verification.scenario_executor import ScenarioCombo, ScenarioExecutor, ScenarioExecutorConfig
        from ..benchmark.verification.utils.task_helpers import stamp_agentic_trait_overrides
        from ..schemas.scenario.types import ModelOverride, ScenarioNode

        global_rubric = self._rubric_manager.get_global_rubric()
        # Stamp pipeline-level retry policy and request timeout onto any
        # agentic rubric trait model_override so that AgenticTraitEvaluator
        # picks up the same defaults the top-level answering and parsing
        # models receive. Returns the original instance unchanged when no
        # stamping is needed (no agentic traits, no overrides, or all
        # override fields already set).
        global_rubric = stamp_agentic_trait_overrides(global_rubric, config)

        def _prepare_model(model: Any) -> Any:
            """Stamp pipeline-level request_timeout and retry_policy when unset.

            Delegates to the shared stamping helper so the scenario path
            matches the QA task-queue expansion exactly (only-when-unset,
            original instance preserved when nothing changes).
            """
            return stamp_pipeline_defaults(
                model,
                request_timeout=config.request_timeout,
                retry_policy=config.retry_policy,
            )

        def _prepare_override(override: ModelOverride | None) -> ModelOverride | None:
            """Stamp pipeline-level timeout and retry policy onto a node override.

            Mirrors :func:`_prepare_model` for ``ModelOverride.answering_model``
            and ``ModelOverride.parsing_model`` so that per-node overrides
            inherit the same defaults the top-level answering and parsing
            models receive. Fields that are already set on the override are
            preserved (matching the ``is None`` guard in ``_prepare_model``).

            Returns the original instance unchanged when no stamping is needed
            so frozen scenario definitions are not rebuilt unnecessarily.
            """
            if override is None:
                return None
            updates: dict[str, Any] = {}
            if override.answering_model is not None:
                stamped_ans = _prepare_model(override.answering_model)
                if stamped_ans is not override.answering_model:
                    updates["answering_model"] = stamped_ans
            if override.parsing_model is not None:
                stamped_parse = _prepare_model(override.parsing_model)
                if stamped_parse is not override.parsing_model:
                    updates["parsing_model"] = stamped_parse
            if not updates:
                return override
            return override.model_copy(update=updates)

        def _prepare_scenario(scenario_def: ScenarioDefinition) -> ScenarioDefinition:
            """Return a scenario definition with stamped per-node overrides.

            Walks each node in ``scenario_def.nodes`` and replaces any
            ``ModelOverride`` whose answering or parsing model is missing the
            pipeline-level ``request_timeout`` or ``retry_policy``. The
            scenario definition is frozen, so a new instance is constructed
            via ``model_copy``. The original definition is returned unchanged
            when no nodes need stamping, avoiding rebuild costs in the common
            case where no per-node overrides are configured.
            """
            new_nodes: dict[str, ScenarioNode] | None = None
            for node_id, node in scenario_def.nodes.items():
                prepared_override = _prepare_override(node.model_override)
                if prepared_override is node.model_override:
                    continue
                if new_nodes is None:
                    new_nodes = dict(scenario_def.nodes)
                new_nodes[node_id] = node.model_copy(update={"model_override": prepared_override})
            if new_nodes is None:
                return scenario_def
            return scenario_def.model_copy(update={"nodes": new_nodes})

        def _replicate_values(count: int) -> list[int | None]:
            """Expand the replicate axis for scenario combos.

            Returns ``[None]`` for the default single-replicate case (preserves
            today's metadata and cache-key shape) and ``1..N`` otherwise.
            Mirrors the QA convention at ``batch_runner.py:117-119``.
            """
            return [None] if count == 1 else list(range(1, count + 1))

        combos: list[ScenarioCombo] = [
            (
                _prepare_scenario(scenario_def),
                _prepare_model(ans_model),
                _prepare_model(parse_model),
                replicate,
            )
            for scenario_def in self._scenarios.values()
            for ans_model in config.answering_models
            for parse_model in config.parsing_models
            for replicate in _replicate_values(config.replicate_count)
        ]

        # Apply task ordering to scenario combos (routes auto/prefix_cache/
        # distribute_answerers/random/generation_order through the shared helper).
        combos = _apply_scenario_ordering(combos, config)

        # Honor config.skip_triples at combo level. For scenarios, slot-0
        # of the triple holds the scenario_id (not question_id), matching
        # metadata.scenario_id on turn_results stored by ProgressiveFileSink.
        # See TaskIdentifier.from_result. This mirrors the QA path where
        # generate_task_queue drops triples listed in skip_triples before
        # execution. Gate strictly on concrete frozenset/set instances so
        # MagicMock configs in legacy tests do not trigger the key-compute
        # path (they cannot be passed to ModelIdentity.from_model_config).
        cfg_skip = getattr(config, "skip_triples", None)
        has_cfg_skip = isinstance(cfg_skip, set | frozenset) and len(cfg_skip) > 0
        need_keys = sink is not None or has_cfg_skip
        skipped_combo_keys: set[tuple[str, str, str, int | None]] = set()
        combo_by_key: dict[tuple[str, str, str, int | None], ScenarioCombo] = {}
        if need_keys:
            from karenina.schemas.verification.model_identity import ModelIdentity
            from karenina.utils.progressive_save import TaskIdentifier

            def _combo_key(combo: ScenarioCombo) -> tuple[str, str, str, int | None]:
                scen, ans, parse, rep = combo
                ans_key = ModelIdentity.from_model_config(ans, role="answering").canonical_key
                parse_key = ModelIdentity.from_model_config(parse, role="parsing").canonical_key
                return (scen.name, ans_key, parse_key, rep)

            combo_by_key = {_combo_key(c): c for c in combos}
            full_manifest: list[str] = [
                TaskIdentifier(
                    question_id=k[0],
                    answering_canonical_key=k[1],
                    parsing_canonical_key=k[2],
                    replicate=k[3],
                ).to_key()
                for k in combo_by_key
            ]

            skip_set: set[tuple[str, str, str, int | None]] = set()
            if sink is not None:
                sink_triples = sink.completed_triples()
                if sink_triples:
                    skip_set |= set(sink_triples)
                    logger.info(
                        "Scenario sink reports %d already-completed combos",
                        len(sink_triples),
                    )
            if has_cfg_skip:
                skip_set |= set(cfg_skip)  # type: ignore[arg-type]  # narrowed by isinstance gate above

            if skip_set:
                skipped_combo_keys = skip_set & set(combo_by_key)
                combos = [c for c in combos if _combo_key(c) not in skip_set]
                logger.info("Skipping %d scenario combos; %d remain", len(skip_set), len(combos))

            if sink is not None:
                sink.on_start(full_manifest, config)

        prior_sink_results: list[VerificationResult] = []
        prior_scenario_results: list[Any] = []
        if sink is not None:
            iterator = getattr(sink, "iter_results", None)
            if iterator is not None:
                prior_sink_results = list(iterator())
                if prior_sink_results and skipped_combo_keys:
                    prior_scenario_results = _reconstruct_scenario_results_from_sink_rows(
                        prior_sink_results,
                        combo_by_key=combo_by_key,
                        skipped_combo_keys=skipped_combo_keys,
                    )

        executor = ScenarioExecutor(
            parallel=bool(async_enabled) and len(combos) > 1,
            config=ScenarioExecutorConfig(
                max_workers=config.async_max_workers,
                max_concurrent_requests=config.max_concurrent_requests,
                enable_cache=True,
                timeout_seconds=config.batch_timeout_seconds,
            ),
        )

        # Adapt the facade callback (float, str) to the executor callback
        # (completed: int, total: int, result_or_none). Also fan out each
        # completed combo's turn_results to the sink so it can persist them
        # incrementally.
        total = len(combos)

        def _adapter(completed: int, _total: int, exec_result: Any) -> None:
            if sink is not None and exec_result is not None:
                for tr in getattr(exec_result, "turn_results", []) or []:
                    try:
                        sink.on_result(tr)
                    except Exception:  # noqa: BLE001
                        logger.warning("Scenario sink on_result raised; continuing", exc_info=True)
            if progress_callback is not None and total > 0:
                pct = completed / total if total > 0 else 1.0
                progress_callback(pct, f"Scenario {completed}/{total}")

        executor_callback = _adapter if (progress_callback is not None or sink is not None) else None

        exec_results, errors = executor.run_batch(
            combos=combos,
            config=config,
            global_rubric=global_rubric,
            run_name=run_name,
            progress_callback=executor_callback,
            workspace_root=self._workspace_root,
        )

        if sink is not None:
            all_complete = not errors and len(exec_results) == len(combos)
            try:
                sink.on_finalize(all_complete=all_complete)
            except Exception:  # noqa: BLE001
                logger.warning("Scenario sink on_finalize raised; continuing", exc_info=True)

        all_turn_results: list[VerificationResult] = list(prior_sink_results)
        for er in exec_results:
            all_turn_results.extend(er.turn_results)

        return VerificationResultSet(
            results=all_turn_results,
            scenario_results=[*prior_scenario_results, *exec_results]
            if (prior_scenario_results or exec_results)
            else None,
            errors=[(d, e) for d, e in errors] if errors else None,
        )

    def extend_template(
        self,
        prior_results: VerificationResultSet,
        config: VerificationConfig,
        *,
        run_name: str | None = None,
        question_ids: list[str] | None = None,
        async_enabled: bool | None = None,
        progress_callback: Callable[[float, str], None] | None = None,
        sink: Any = None,
        store: bool = True,
    ) -> VerificationResultSet:
        """Extend a prior verification run along any combination of three axes.

        Axes, all optional and composable in a single call:

        1. New judges (``config.parsing_models``).
        2. New answerers (``config.answering_models``).
        3. More replicates (higher ``config.replicate_count``).

        Prior ``(question, answerer, replicate)`` answering traces are served
        from a :class:`~karenina.replay.ReplayStore`; new answerers, new
        replicates, or any combination thereof run answering live. Parsing
        always runs live. Triples already present in ``prior_results`` are
        skipped so prior rows pass through verbatim. The merged output matches
        a joint run over the full ``(answerers × judges × replicates)`` matrix.

        Args:
            prior_results: Result set from an earlier ``run_verification``
                call to extend.
            config: Verification configuration describing the **final**
                state (full union, not deltas): every judge in
                ``parsing_models``, every answerer in ``answering_models``,
                and the final ``replicate_count`` (must be ``>=`` observed
                in ``prior_results``). ``config.replay_store`` must be
                ``None``.
            run_name: Optional override for the merged run name. Defaults to
                the run name carried by ``prior_results``.
            question_ids: Optional subset of question IDs to re-judge.
                Defaults to every question present in ``prior_results``.
            async_enabled: Optional async control forwarded to
                ``run_verification``.
            progress_callback: Optional progress callback forwarded to
                ``run_verification``.
            sink: Optional :class:`~karenina.benchmark.verification.sinks.ResultSink`
                forwarded to ``run_verification`` so newly produced rows are
                persisted as they complete. Prior rows pass through the merge
                verbatim and never reach the sink. Pair with
                :class:`~karenina.benchmark.verification.sinks.ProgressiveFileSink`
                to make the extension resumable.
            store: When True (default), also store the merged set into the
                in-memory results manager under ``run_name`` so it is
                available to ``get_verification_results`` and the exporters.

        Returns:
            Merged ``VerificationResultSet`` with the prior rows plus the
            newly-produced rows, all stamped with the effective ``run_name``.
        """
        from .verification.extension import extend_template_run

        merged = extend_template_run(
            self,
            prior_results,
            config,
            run_name=run_name,
            question_ids=question_ids,
            async_enabled=async_enabled,
            progress_callback=progress_callback,
            sink=sink,
        )
        if store:
            results_dict: dict[str, VerificationResult] = {}
            for idx, row in enumerate(merged.results):
                md = row.metadata
                key = f"{md.question_id}_{md.answering_model}_{md.parsing_model}"
                if md.replicate is not None:
                    key += f"_rep{md.replicate}"
                if md.timestamp:
                    key += f"_{md.timestamp}"
                if key in results_dict:
                    key += f"_{idx}"
                results_dict[key] = row
            effective_run_name = merged.results[0].metadata.run_name if merged.results else run_name
            self._results_manager.store_verification_results(results_dict, effective_run_name)
        return merged

    def extend_rubric(
        self,
        prior_results: VerificationResultSet,
        config: VerificationConfig,
        *,
        run_name: str | None = None,
        question_ids: list[str] | None = None,
        async_enabled: bool | None = None,
        progress_callback: Callable[[float, str], None] | None = None,
        sink: Any = None,
        store: bool = True,
    ) -> VerificationResultSet:
        """Attach a new rubric to a prior verification run.

        Enriches every row of ``prior_results`` with rubric scores
        produced against the rubric currently attached to this benchmark
        (global and per-question). Answering is replayed from the prior
        traces; template parsing and verification are skipped. Row count
        is preserved: the merged set has the same shape as
        ``prior_results`` with a populated ``rubric`` sub-object on each
        row.

        Trait scores are unioned with any rubric fields already present
        on prior rows. Same-name trait collisions raise ``ValueError``.

        Args:
            prior_results: Result set from an earlier ``run_verification``
                call.
            config: Verification configuration matching the shape of
                ``prior_results``. ``answering_models``,
                ``parsing_models``, and ``replicate_count`` must equal
                the values observed in ``prior_results``.
                ``replay_store`` must be ``None``;
                ``evaluation_mode`` must be ``"template_only"`` or
                ``"rubric_only"`` (the helper rewrites it internally).
            run_name: Optional override for the merged run name.
            question_ids: Optional subset of question IDs.
            async_enabled: Forwarded to ``run_verification``.
            progress_callback: Forwarded to ``run_verification``.
            sink: Optional :class:`~karenina.benchmark.verification.sinks.ResultSink`
                forwarded to ``run_verification``. The sink receives the
                ``rubric_only`` rows produced by the extension pipeline (one
                per prior triple); it does not see the enriched rows returned
                to the caller. Pair with
                :class:`~karenina.benchmark.verification.sinks.ProgressiveFileSink`
                to make rubric extension resumable.
            store: When True (default), write the merged set into the
                results manager.

        Returns:
            ``VerificationResultSet`` of enriched prior rows.

        Raises:
            ValueError: On input validation failure; see
                :func:`~karenina.benchmark.verification.extension.extend_rubric_run`.
        """
        from .verification.extension import extend_rubric_run

        merged = extend_rubric_run(
            self,
            prior_results,
            config,
            run_name=run_name,
            question_ids=question_ids,
            async_enabled=async_enabled,
            progress_callback=progress_callback,
            sink=sink,
        )
        if store:
            results_dict: dict[str, VerificationResult] = {}
            for idx, row in enumerate(merged.results):
                md = row.metadata
                key = f"{md.question_id}_{md.answering_model}_{md.parsing_model}"
                if md.replicate is not None:
                    key += f"_rep{md.replicate}"
                if md.timestamp:
                    key += f"_{md.timestamp}"
                if key in results_dict:
                    key += f"_{idx}"
                results_dict[key] = row
            effective_run_name = merged.results[0].metadata.run_name if merged.results else run_name
            self._results_manager.store_verification_results(results_dict, effective_run_name)
        return merged

    # ── Results management ───────────────────────────────────────────────

    def store_verification_results(
        self,
        results: VerificationResultSet | dict[str, VerificationResult],
        run_name: str | None = None,
    ) -> None:
        """Store verification results in the benchmark metadata."""
        warnings.warn(
            "store_verification_results is deprecated. Use ResultsStore.add() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        _helpers.store_verification_results(self, results, run_name)

    def get_verification_results(
        self,
        question_ids: list[str] | None = None,
        run_name: str | None = None,
    ) -> dict[str, VerificationResult]:
        """Get verification results for specific questions and/or runs."""
        warnings.warn(
            "get_verification_results is deprecated. Use ResultsStore.get_by_run() or ResultsStore.get_latest() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.get_verification_results(question_ids, run_name)

    def get_verification_history(self, question_id: str | None = None) -> dict[str, dict[str, VerificationResult]]:
        """Get verification history organized by run name."""
        warnings.warn(
            "get_verification_history is deprecated. Use ResultsStore.get_by_question() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.get_verification_history(question_id)

    def clear_verification_results(
        self,
        question_ids: list[str] | None = None,
        run_name: str | None = None,
    ) -> int:
        """Clear verification results."""
        warnings.warn(
            "clear_verification_results is deprecated. Use ResultsStore.clear() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.clear_verification_results(question_ids, run_name)

    def export_verification_results(
        self,
        question_ids: list[str] | None = None,
        run_name: str | None = None,
        format: str = "json",
        global_rubric: "Rubric | None" = None,
    ) -> str:
        """Export verification results in specified format."""
        warnings.warn(
            "export_verification_results is deprecated. Use ResultsStore.export() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.export_verification_results(question_ids, run_name, format, global_rubric)

    def export_verification_results_to_file(
        self,
        file_path: Path,
        question_ids: list[str] | None = None,
        run_name: str | None = None,
        format: str | None = None,
        global_rubric: "Rubric | None" = None,
    ) -> None:
        """Export verification results directly to a file."""
        warnings.warn(
            "export_verification_results_to_file is deprecated. Use ResultsStore.export_to_file() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        self._results_manager.export_results_to_file(file_path, question_ids, run_name, format, global_rubric)

    def load_verification_results_from_file(
        self,
        file_path: Path,
        run_name: str | None = None,
    ) -> dict[str, VerificationResult]:
        """Load verification results from a previously exported file."""
        warnings.warn(
            "load_verification_results_from_file is deprecated. Use ResultsStore.from_file() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.load_results_from_file(file_path, run_name)

    def get_verification_summary(self, run_name: str | None = None) -> dict[str, Any]:
        """Get summary statistics for verification results."""
        warnings.warn(
            "get_verification_summary is deprecated. Use ResultsStore.get_summary() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.get_verification_summary(run_name)

    def get_all_run_names(self) -> list[str]:
        """Get all verification run names."""
        warnings.warn(
            "get_all_run_names is deprecated. Use ResultsStore.get_all_runs() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.get_all_run_names()

    def get_results_statistics_by_run(self) -> dict[str, dict[str, Any]]:
        """Get verification statistics for each run."""
        warnings.warn(
            "get_results_statistics_by_run is deprecated. Use ResultsStore.get_statistics_by_run() instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        return self._results_manager.get_results_statistics_by_run()

    # ── GEPA optimization (delegated to benchmark_helpers) ───────────────

    def optimize(
        self,
        targets: list[str],
        config: VerificationConfig | None = None,
        train_ratio: float = 0.8,
        val_ratio: float = 0.2,
        test_ratio: float | None = None,
        seed: int | None = None,
        reflection_model: str = "openai/gpt-4o",
        max_metric_calls: int = 150,
        objective_config: "ObjectiveConfig | None" = None,
        frontier_type: "FrontierType" = "objective",
        seed_prompts: dict[str, str] | None = None,
        tracker_path: Path | str | None = None,
        export_preset_path: Path | str | None = None,
        progress_callback: Callable[[float, str], None] | None = None,
        verbose: bool = False,
    ) -> "KareninaOutput":
        """
        Optimize text components using GEPA with karenina verification as the metric.

        Requires the 'gepa' optional dependency: pip install karenina[gepa]

        Args:
            targets: List of components to optimize. Valid values:
                     "answering_system_prompt", "parsing_instructions", "mcp_tool_descriptions"
            config: Base VerificationConfig to use. If None, uses default minimal config.
            train_ratio: Fraction of questions for training (default 0.8)
            val_ratio: Fraction of questions for validation (default 0.2)
            test_ratio: Optional fraction for testing. If None, no test set created.
            seed: Random seed for reproducibility
            reflection_model: Model for GEPA's reflection LLM (default: openai/gpt-4o)
            max_metric_calls: Maximum GEPA optimization iterations (default: 150)
            objective_config: Configuration for multi-objective optimization dimensions.
            frontier_type: GEPA Pareto frontier tracking strategy.
            seed_prompts: Optional initial prompts. If None, uses empty strings.
            tracker_path: Optional path to SQLite file for tracking optimization history
            export_preset_path: Optional path to export optimized config as preset
            progress_callback: Optional callback for progress updates (percentage, message)
            verbose: If True, display detailed progress during optimization

        Returns:
            KareninaOutput with optimized prompts and metrics

        Example:
            >>> result = benchmark.optimize(
            ...     targets=["answering_system_prompt"],
            ...     reflection_model="openai/gpt-4o",
            ...     max_metric_calls=100,
            ... )
            >>> print(f"Improvement: {result.improvement:.1%}")
        """
        return _helpers.run_optimize(
            self,
            targets,
            config,
            train_ratio,
            val_ratio,
            test_ratio,
            seed,
            reflection_model,
            max_metric_calls,
            objective_config,
            frontier_type,
            seed_prompts,
            tracker_path,
            export_preset_path,
            progress_callback,
            verbose,
        )

    def optimization_history(
        self,
        tracker_path: Path | str = "~/.karenina/optimization_history.db",
        limit: int = 20,
    ) -> list["OptimizationRun"]:
        """Get optimization history for this benchmark."""
        try:
            from karenina.integrations.gepa import OptimizationTracker
        except ImportError:
            return []

        tracker = OptimizationTracker(tracker_path)
        return tracker.list_runs(benchmark_name=self.name, limit=limit)

    # ── Metadata management ──────────────────────────────────────────────

    def get_custom_property(self, name: str) -> Any:
        """Get a custom property from benchmark metadata."""
        return self._metadata_manager.get_custom_property(name)

    def set_custom_property(self, name: str, value: Any) -> None:
        """Set a custom property in benchmark metadata."""
        self._metadata_manager.set_custom_property(name, value)

    def remove_custom_property(self, name: str) -> bool:
        """Remove a custom property from benchmark metadata."""
        return self._metadata_manager.remove_custom_property(name)

    def get_all_custom_properties(self) -> dict[str, Any]:
        """Get all custom properties as a dictionary."""
        return self._metadata_manager.get_all_custom_properties()

    def set_multiple_custom_properties(self, properties: dict[str, Any]) -> None:
        """Set multiple custom properties at once."""
        self._metadata_manager.set_multiple_custom_properties(properties)

    # ── Export and reporting ──────────────────────────────────────────────

    def to_dict(self) -> dict[str, Any]:
        """Export benchmark as a plain dictionary."""
        return self._export_manager.to_dict()

    def to_markdown(self) -> str:
        """Export benchmark as markdown document."""
        return self._export_manager.to_markdown()

    def to_csv(self) -> str:
        """Export questions as CSV format."""
        return self._export_manager.to_csv()

    def get_summary(self) -> dict[str, Any]:
        """Get comprehensive benchmark statistics."""
        return self._export_manager.get_summary()

    def get_statistics(self) -> dict[str, Any]:
        """Get detailed statistics about the benchmark."""
        return self._export_manager.get_statistics()

    def check_readiness(self) -> dict[str, Any]:
        """Comprehensive readiness check for verification."""
        return self._export_manager.check_readiness()

    def get_health_report(self) -> dict[str, Any]:
        """Get comprehensive health/status report."""
        return self._export_manager.get_health_report()

    def clone(self) -> "Benchmark":
        """Create a deep copy of the benchmark."""
        cloned_base = self._export_manager.clone()
        instance = Benchmark.__new__(Benchmark)
        instance._base = cloned_base
        instance._workspace_root = self._workspace_root
        instance._init_managers()
        return instance

    def validate(self) -> tuple[bool, str]:
        """Validate the benchmark structure and all templates."""
        from .verification.utils.validation import validate_answer_template

        is_valid, error_msg = self._base.validate()
        if not is_valid:
            return False, error_msg

        for q_id, q_data in self._questions_cache.items():
            template_code = q_data.get("answer_template")
            if template_code is not None:
                is_valid, error_msg_or_none, _ = validate_answer_template(template_code)
                error_msg = error_msg_or_none or "Unknown validation error"
                if not is_valid:
                    return False, f"Invalid template for {q_id}: {error_msg}"

        return True, "Benchmark is valid"

    def set_metadata(self, **metadata: Any) -> None:
        """Set benchmark metadata."""
        self._base.set_metadata(**metadata)

    # ── Base class property delegation ───────────────────────────────────

    @property
    def _checkpoint(self) -> Any:
        """Get the raw JSON-LD checkpoint data (for backward compatibility)."""
        return self._base._checkpoint

    @property
    def _questions_cache(self) -> dict[str, Any]:
        """Get the questions cache (for backward compatibility)."""
        return self._base._questions_cache

    @property
    def _question_registry(self) -> dict[str, Any]:
        """Get the question registry (for backward compatibility)."""
        return self._base._question_registry

    def _get_item_id(self, item: Any) -> str:
        """Get the ID for a DataFeedItem (for backward compatibility)."""
        return self._base._get_item_id(item)

    def _rebuild_cache(self) -> None:
        """Rebuild the internal questions cache (for backward compatibility)."""
        return self._base._rebuild_cache()

    def _get_merged_rubric_for_question(self, question_id: str) -> Rubric | None:
        """Get merged rubric for a question (for backward compatibility)."""
        return self._rubric_manager.get_merged_rubric_for_question(question_id)

    @property
    def jsonld_data(self) -> Any:
        """Get the raw JSON-LD benchmark data."""
        return self._base.jsonld_data

    @property
    def name(self) -> str:
        """Get the benchmark name."""
        return self._base.name

    @name.setter
    def name(self, value: str) -> None:
        self._base.name = value

    @property
    def description(self) -> str:
        """Get the benchmark description."""
        return self._base.description

    @description.setter
    def description(self, value: str) -> None:
        self._base.description = value

    @property
    def version(self) -> str:
        """Get the benchmark version."""
        return self._base.version

    @version.setter
    def version(self, value: str) -> None:
        self._base.version = value

    @property
    def creator(self) -> str:
        """Get the benchmark creator."""
        return self._base.creator

    @creator.setter
    def creator(self, value: str) -> None:
        self._base.creator = value

    @property
    def id(self) -> str | None:
        """Get the benchmark ID."""
        return self._base.id

    @id.setter
    def id(self, value: str | None) -> None:
        self._base.id = value

    @property
    def created_at(self) -> str:
        """Get the creation timestamp."""
        return self._base.created_at

    @created_at.setter
    def created_at(self, value: str) -> None:
        self._base.created_at = value

    @property
    def modified_at(self) -> str:
        """Get the last modification timestamp."""
        return self._base.modified_at

    @modified_at.setter
    def modified_at(self, value: str) -> None:
        self._base.modified_at = value

    @property
    def question_count(self) -> int:
        """Get the total number of questions."""
        return self._base.question_count

    @property
    def finished_count(self) -> int:
        """Get the number of finished questions."""
        return self._base.finished_count

    @property
    def is_empty(self) -> bool:
        """Check if the benchmark has no questions and no scenarios."""
        return len(self._base._questions_cache) == 0 and len(self._scenarios) == 0

    @property
    def is_complete(self) -> bool:
        """Check if all questions have templates and are finished."""
        return self._base.is_complete

    def get_progress(self) -> float:
        """Get completion progress as percentage (0-100)."""
        return self._base.get_progress()

    # ── Magic methods ────────────────────────────────────────────────────

    def __repr__(self) -> str:
        """Developer-friendly representation with detailed statistics."""
        return _helpers.build_repr(self)

    def __str__(self) -> str:
        """String representation (same as repr for developer-friendly output)."""
        return self.__repr__()

    def __len__(self) -> int:
        """Return the number of questions or scenarios in the benchmark."""
        if self._scenarios:
            return len(self._scenarios)
        return len(self._base)

    def __iter__(self) -> Iterator[dict[str, Any]]:
        """Iterate over questions in the benchmark."""
        return iter(self._question_manager)

    def __contains__(self, question_id: str) -> bool:
        """Check if a question ID exists in the benchmark."""
        return question_id in self._base

    def __getitem__(self, key: str | int | slice) -> "SchemaOrgQuestion | list[SchemaOrgQuestion]":
        """Get question(s) as SchemaOrgQuestion object(s) using bracket notation."""
        from ..schemas.entities.question import QuestionRegistryEntry

        if isinstance(key, str):
            question_data = self._base[key]
            finished = self._base._question_registry.get(key, QuestionRegistryEntry()).finished
            return _helpers.convert_to_schema_org_question(question_data, finished=finished)
        elif isinstance(key, int):
            question_ids = self.get_question_ids()
            original_key = key
            if key < 0:
                key += len(question_ids)
            if not 0 <= key < len(question_ids):
                raise IndexError(f"Question index {original_key} out of range (0-{len(question_ids) - 1})")
            question_id = question_ids[key]
            question_data = self._base[question_id]
            finished = self._base._question_registry.get(question_id, QuestionRegistryEntry()).finished
            return _helpers.convert_to_schema_org_question(question_data, finished=finished)
        elif isinstance(key, slice):
            question_ids = self.get_question_ids()
            selected_ids = question_ids[key]
            return [
                _helpers.convert_to_schema_org_question(
                    self._base[qid],
                    finished=self._base._question_registry.get(qid, QuestionRegistryEntry()).finished,
                )
                for qid in selected_ids
            ]
        else:
            raise TypeError(f"Invalid key type {type(key)}. Expected str, int, or slice.")

    def _convert_to_schema_org_question(self, question_data: dict[str, Any]) -> "SchemaOrgQuestion":
        """Convert internal question dictionary to SchemaOrgQuestion object."""
        from ..schemas.entities.question import QuestionRegistryEntry

        q_id = question_data.get("id", "")
        finished = self._base._question_registry.get(q_id, QuestionRegistryEntry()).finished
        return _helpers.convert_to_schema_org_question(question_data, finished=finished)

    def __eq__(self, other: object) -> bool:
        """Compare two benchmarks for equality."""
        if not isinstance(other, Benchmark):
            return NotImplemented
        return self._base == other._base

Attributes

created_at property writable

created_at: str

Get the creation timestamp.

creator property writable

creator: str

Get the benchmark creator.

description property writable

description: str

Get the benchmark description.

finished_count property

finished_count: int

Get the number of finished questions.

id property writable

id: str | None

Get the benchmark ID.

is_complete property

is_complete: bool

Check if all questions have templates and are finished.

is_empty property

is_empty: bool

Check if the benchmark has no questions and no scenarios.

is_scenario_benchmark property

is_scenario_benchmark: bool

True if this benchmark contains scenarios instead of standalone questions.

jsonld_data property

jsonld_data: Any

Get the raw JSON-LD benchmark data.

modified_at property writable

modified_at: str

Get the last modification timestamp.

name property writable

name: str

Get the benchmark name.

question_count property

question_count: int

Get the total number of questions.

scenario_count property

scenario_count: int

Return the number of scenarios in the benchmark.

version property writable

version: str

Get the benchmark version.

workspace_root property

workspace_root: Path | None

Root directory for task workspaces (not persisted in checkpoint).

Functions

__init__

__init__(
    name: str,
    description: str = "",
    version: str = "0.1.0",
    creator: str = "Karenina Benchmarking System",
    workspace_root: Path | None = None,
)

Parameters:

Name	Type	Description	Default
name	str	Name of the benchmark	required
description	str	Description of the benchmark	''
version	str	Version of the benchmark content	'0.1.0'
creator	str	Creator name or organization	'Karenina Benchmarking System'
workspace_root	Path | None	Root directory containing task workspaces. Question workspace paths are resolved relative to this root. Not persisted in the checkpoint (it is a local filesystem path).	None

Source code in src/karenina/benchmark/benchmark.py

def __init__(
    self,
    name: str,
    description: str = "",
    version: str = "0.1.0",
    creator: str = "Karenina Benchmarking System",
    workspace_root: Path | None = None,
):
    """
    Initialize a new benchmark.

    Args:
        name: Name of the benchmark
        description: Description of the benchmark
        version: Version of the benchmark content
        creator: Creator name or organization
        workspace_root: Root directory containing task workspaces.
            Question workspace paths are resolved relative to this root.
            Not persisted in the checkpoint (it is a local filesystem path).
    """
    self._base = BenchmarkBase(name, description, version, creator)
    self._workspace_root = workspace_root
    self._scenarios: dict[str, ScenarioDefinition] = {}
    self._metadata_manager = MetadataManager(self._base)
    self._question_manager = QuestionManager(self._base)
    self._rubric_manager = RubricManager(self._base)
    self._template_manager = TemplateManager(self._base)
    self._results_manager = ResultsManager(self._base)
    self._verification_manager = VerificationManager(self._base, self._rubric_manager)
    self._export_manager = ExportManager(self._base, self._template_manager, self._rubric_manager)

add_answer_template

add_answer_template(
    question_id: str, template_code: str
) -> None

Add or update an answer template for a question.

Source code in src/karenina/benchmark/benchmark.py

def add_answer_template(self, question_id: str, template_code: str) -> None:
    """Add or update an answer template for a question."""
    self._template_manager.add_answer_template(question_id, template_code)

add_global_rubric_trait

add_global_rubric_trait(
    trait: LLMRubricTrait
    | RegexRubricTrait
    | CallableRubricTrait
    | MetricRubricTrait
    | AgenticRubricTrait,
) -> None

Add a global rubric trait to the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def add_global_rubric_trait(
    self, trait: LLMRubricTrait | RegexRubricTrait | CallableRubricTrait | MetricRubricTrait | AgenticRubricTrait
) -> None:
    """Add a global rubric trait to the benchmark."""
    self._rubric_manager.add_global_rubric_trait(trait)

add_question

add_question(
    question: Union[str, dict[str, Any], Question],
    raw_answer: str | None = None,
    answer_template: str | type | None = None,
    question_id: str | None = None,
    finished: bool | object = _NOT_PROVIDED,
    author: dict[str, Any] | None = None,
    sources: list[dict[str, Any]] | None = None,
    custom_metadata: dict[str, Any] | None = None,
    few_shot_examples: list[dict[str, str]] | None = None,
    answer_notes: str | None = None,
) -> str

Add a question to the benchmark.

Accepts a question string, a Question object, or a dict with keys question and raw_answer (plus any optional kwargs).

Raises:

Type	Description
ValueError	If scenarios already exist (homogeneous enforcement).

Source code in src/karenina/benchmark/benchmark.py

def add_question(
    self,
    question: Union[str, dict[str, Any], "Question"],
    raw_answer: str | None = None,
    answer_template: str | type | None = None,
    question_id: str | None = None,
    finished: bool | object = _NOT_PROVIDED,
    author: dict[str, Any] | None = None,
    sources: list[dict[str, Any]] | None = None,
    custom_metadata: dict[str, Any] | None = None,
    few_shot_examples: list[dict[str, str]] | None = None,
    answer_notes: str | None = None,
) -> str:
    """Add a question to the benchmark.

    Accepts a question string, a Question object, or a dict with keys
    ``question`` and ``raw_answer`` (plus any optional kwargs).

    Raises:
        ValueError: If scenarios already exist (homogeneous enforcement).
    """
    if self._scenarios:
        raise ValueError(
            "Cannot add standalone questions to a scenario benchmark. "
            "Scenarios and standalone questions cannot coexist in the same benchmark."
        )
    return self._question_manager.add_question(
        question,
        raw_answer,
        answer_template,
        question_id,
        finished,
        author,
        sources,
        custom_metadata,
        few_shot_examples,
        answer_notes=answer_notes,
    )

add_question_from_object

add_question_from_object(
    question_obj: Question, **metadata: Any
) -> str

Add a question to the benchmark from a Question object.

Source code in src/karenina/benchmark/benchmark.py

def add_question_from_object(self, question_obj: "Question", **metadata: Any) -> str:
    """Add a question to the benchmark from a Question object."""
    return self._question_manager.add_question_from_object(question_obj, **metadata)

add_question_rubric_trait

add_question_rubric_trait(
    question_id: str,
    trait: LLMRubricTrait
    | RegexRubricTrait
    | CallableRubricTrait
    | MetricRubricTrait
    | AgenticRubricTrait,
) -> None

Add a question-specific rubric trait.

Source code in src/karenina/benchmark/benchmark.py

def add_question_rubric_trait(
    self,
    question_id: str,
    trait: LLMRubricTrait | RegexRubricTrait | CallableRubricTrait | MetricRubricTrait | AgenticRubricTrait,
) -> None:
    """Add a question-specific rubric trait."""
    self._rubric_manager.add_question_rubric_trait(question_id, trait)

add_questions

add_questions(
    questions_data: Sequence[dict[str, Any] | Question],
    finished: bool | object = _NOT_PROVIDED,
) -> list[str]

Add multiple questions at once.

Accepts a list of dicts, Question objects, or a mix of both.

Parameters:

Name	Type	Description	Default
questions_data	Sequence[dict[str, Any] | Question]	List of dicts or Question objects.	required
finished	bool | object	Default finished state for items that do not define one. Pass False when importing items for template drafting and curator review. Omission preserves the existing default.	_NOT_PROVIDED

Returns:

Type	Description
list[str]	List of question IDs that were created.

Raises:

Type	Description
ValueError	If scenarios already exist (homogeneous enforcement).

Source code in src/karenina/benchmark/benchmark.py

def add_questions(
    self,
    questions_data: "Sequence[dict[str, Any] | Question]",
    finished: bool | object = _NOT_PROVIDED,
) -> list[str]:
    """Add multiple questions at once.

    Accepts a list of dicts, Question objects, or a mix of both.

    Args:
        questions_data: List of dicts or Question objects.
        finished: Default finished state for items that do not define one.
            Pass False when importing items for template drafting and
            curator review. Omission preserves the existing default.

    Returns:
        List of question IDs that were created.

    Raises:
        ValueError: If scenarios already exist (homogeneous enforcement).
    """
    if self._scenarios:
        raise ValueError(
            "Cannot add standalone questions to a scenario benchmark. "
            "Scenarios and standalone questions cannot coexist in the same benchmark."
        )
    return self._question_manager.add_questions(questions_data, finished=finished)

add_questions_batch

add_questions_batch(
    questions_data: Sequence[dict[str, Any] | Question],
    finished: bool | object = _NOT_PROVIDED,
) -> list[str]

Add multiple questions at once with an optional finished state.

Source code in src/karenina/benchmark/benchmark.py

def add_questions_batch(
    self,
    questions_data: "Sequence[dict[str, Any] | Question]",
    finished: bool | object = _NOT_PROVIDED,
) -> list[str]:
    """Add multiple questions at once with an optional finished state."""
    return self._question_manager.add_questions_batch(questions_data, finished=finished)

add_scenario

add_scenario(scenario: ScenarioDefinition | Any) -> None

Add a scenario to the benchmark.

Accepts either a ScenarioDefinition (frozen) or a Scenario builder (which will be validated and frozen automatically).

Parameters:

Name	Type	Description	Default
scenario	ScenarioDefinition | Any	A ScenarioDefinition or a Scenario builder instance.	required

Raises:

Type	Description
ValueError	If standalone questions already exist (homogeneous enforcement), or if a scenario with the same name already exists.

Source code in src/karenina/benchmark/benchmark.py

def add_scenario(self, scenario: "ScenarioDefinition | Any") -> None:
    """Add a scenario to the benchmark.

    Accepts either a ScenarioDefinition (frozen) or a Scenario builder
    (which will be validated and frozen automatically).

    Args:
        scenario: A ScenarioDefinition or a Scenario builder instance.

    Raises:
        ValueError: If standalone questions already exist (homogeneous enforcement),
            or if a scenario with the same name already exists.
    """
    if self._base._questions_cache:
        raise ValueError(
            "Cannot add scenarios to a benchmark that already contains standalone questions. "
            "Scenarios and standalone questions cannot coexist in the same benchmark."
        )

    # Accept Scenario builder: call validate() to get a ScenarioDefinition
    if not isinstance(scenario, ScenarioDefinition):
        scenario = scenario.validate()

    if scenario.name in self._scenarios:
        raise ValueError(f"Scenario '{scenario.name}' already exists")

    self._scenarios[scenario.name] = scenario

    # Write to checkpoint (checkpoint is source of truth)
    from ..scenario.checkpoint import scenario_to_schema_org
    from ..schemas.checkpoint import SchemaOrgPropertyValue

    schema_org = scenario_to_schema_org(scenario)
    if self._base._checkpoint.hasPart is None:
        self._base._checkpoint.hasPart = []
    self._base._checkpoint.hasPart.append(schema_org)

    # Set benchmark_type flag (once)
    props = self._base._checkpoint.additionalProperty or []
    if not any(p.name == "benchmark_type" for p in props):
        if self._base._checkpoint.additionalProperty is None:
            self._base._checkpoint.additionalProperty = []
        self._base._checkpoint.additionalProperty.append(
            SchemaOrgPropertyValue(name="benchmark_type", value="scenario")
        )

apply_global_template

apply_global_template(template_code: str) -> list[str]

Apply a template to all questions that don't have one.

Source code in src/karenina/benchmark/benchmark.py

def apply_global_template(self, template_code: str) -> list[str]:
    """Apply a template to all questions that don't have one."""
    return self._template_manager.apply_global_template(template_code)

check_readiness

check_readiness() -> dict[str, Any]

Comprehensive readiness check for verification.

Source code in src/karenina/benchmark/benchmark.py

def check_readiness(self) -> dict[str, Any]:
    """Comprehensive readiness check for verification."""
    return self._export_manager.check_readiness()

clear_all_rubrics

clear_all_rubrics() -> int

Remove all rubrics (global and question-specific).

Source code in src/karenina/benchmark/benchmark.py

def clear_all_rubrics(self) -> int:
    """Remove all rubrics (global and question-specific)."""
    return self._rubric_manager.clear_all_rubrics()

clear_global_rubric

clear_global_rubric() -> bool

Remove the global rubric.

Source code in src/karenina/benchmark/benchmark.py

def clear_global_rubric(self) -> bool:
    """Remove the global rubric."""
    return self._rubric_manager.clear_global_rubric()

clear_questions

clear_questions() -> int

Remove all questions from the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def clear_questions(self) -> int:
    """Remove all questions from the benchmark."""
    return self._question_manager.clear_questions()

clear_verification_results

clear_verification_results(
    question_ids: list[str] | None = None,
    run_name: str | None = None,
) -> int

Clear verification results.

Source code in src/karenina/benchmark/benchmark.py

def clear_verification_results(
    self,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
) -> int:
    """Clear verification results."""
    warnings.warn(
        "clear_verification_results is deprecated. Use ResultsStore.clear() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.clear_verification_results(question_ids, run_name)

clone

clone() -> Benchmark

Create a deep copy of the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def clone(self) -> "Benchmark":
    """Create a deep copy of the benchmark."""
    cloned_base = self._export_manager.clone()
    instance = Benchmark.__new__(Benchmark)
    instance._base = cloned_base
    instance._workspace_root = self._workspace_root
    instance._init_managers()
    return instance

copy_template

copy_template(from_id: str, to_id: str) -> None

Copy template from one question to another.

Source code in src/karenina/benchmark/benchmark.py

def copy_template(self, from_id: str, to_id: str) -> None:
    """Copy template from one question to another."""
    self._template_manager.copy_template(from_id, to_id)

count_by_field

count_by_field(
    field_path: str,
    questions: list[dict[str, Any]] | None = None,
) -> dict[Any, int]

Count questions grouped by a field value using dot notation.

Source code in src/karenina/benchmark/benchmark.py

def count_by_field(self, field_path: str, questions: list[dict[str, Any]] | None = None) -> dict[Any, int]:
    """Count questions grouped by a field value using dot notation."""
    return self._question_manager.count_by_field(field_path, questions)

create classmethod

create(
    name: str,
    description: str = "",
    version: str = "0.1.0",
    creator: str = "Karenina Benchmarking System",
    workspace_root: Path | None = None,
) -> Benchmark

Create a new benchmark (alias for constructor).

Source code in src/karenina/benchmark/benchmark.py

@classmethod
def create(
    cls,
    name: str,
    description: str = "",
    version: str = "0.1.0",
    creator: str = "Karenina Benchmarking System",
    workspace_root: Path | None = None,
) -> "Benchmark":
    """Create a new benchmark (alias for constructor)."""
    return cls(name, description, version, creator, workspace_root=workspace_root)

export_generated_templates

export_generated_templates(file_path: Path) -> None

Export all generated templates to a JSON file.

Source code in src/karenina/benchmark/benchmark.py

def export_generated_templates(self, file_path: Path) -> None:
    """Export all generated templates to a JSON file."""
    _helpers.export_generated_templates(self, file_path)

export_verification_results

export_verification_results(
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    format: str = "json",
    global_rubric: Rubric | None = None,
) -> str

Export verification results in specified format.

Source code in src/karenina/benchmark/benchmark.py

def export_verification_results(
    self,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    format: str = "json",
    global_rubric: "Rubric | None" = None,
) -> str:
    """Export verification results in specified format."""
    warnings.warn(
        "export_verification_results is deprecated. Use ResultsStore.export() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.export_verification_results(question_ids, run_name, format, global_rubric)

export_verification_results_to_file

export_verification_results_to_file(
    file_path: Path,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    format: str | None = None,
    global_rubric: Rubric | None = None,
) -> None

Export verification results directly to a file.

Source code in src/karenina/benchmark/benchmark.py

def export_verification_results_to_file(
    self,
    file_path: Path,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    format: str | None = None,
    global_rubric: "Rubric | None" = None,
) -> None:
    """Export verification results directly to a file."""
    warnings.warn(
        "export_verification_results_to_file is deprecated. Use ResultsStore.export_to_file() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    self._results_manager.export_results_to_file(file_path, question_ids, run_name, format, global_rubric)

extend_rubric

extend_rubric(
    prior_results: VerificationResultSet,
    config: VerificationConfig,
    *,
    run_name: str | None = None,
    question_ids: list[str] | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None]
    | None = None,
    sink: Any = None,
    store: bool = True,
) -> VerificationResultSet

Attach a new rubric to a prior verification run.

Enriches every row of prior_results with rubric scores produced against the rubric currently attached to this benchmark (global and per-question). Answering is replayed from the prior traces; template parsing and verification are skipped. Row count is preserved: the merged set has the same shape as prior_results with a populated rubric sub-object on each row.

Trait scores are unioned with any rubric fields already present on prior rows. Same-name trait collisions raise ValueError.

Parameters:

Name	Type	Description	Default
prior_results	VerificationResultSet	Result set from an earlier run_verification call.	required
config	VerificationConfig	Verification configuration matching the shape of prior_results. answering_models, parsing_models, and replicate_count must equal the values observed in prior_results. replay_store must be None; evaluation_mode must be "template_only" or "rubric_only" (the helper rewrites it internally).	required
run_name	str | None	Optional override for the merged run name.	None
question_ids	list[str] | None	Optional subset of question IDs.	None
async_enabled	bool | None	Forwarded to run_verification.	None
progress_callback	Callable[[float, str], None] | None	Forwarded to run_verification.	None
sink	Any	Optional :class:~karenina.benchmark.verification.sinks.ResultSink forwarded to run_verification. The sink receives the rubric_only rows produced by the extension pipeline (one per prior triple); it does not see the enriched rows returned to the caller. Pair with :class:~karenina.benchmark.verification.sinks.ProgressiveFileSink to make rubric extension resumable.	None
store	bool	When True (default), write the merged set into the results manager.	True

Returns:

Type	Description
VerificationResultSet	VerificationResultSet of enriched prior rows.

Raises:

Type	Description
ValueError	On input validation failure; see :func:~karenina.benchmark.verification.extension.extend_rubric_run.

Source code in src/karenina/benchmark/benchmark.py

def extend_rubric(
    self,
    prior_results: VerificationResultSet,
    config: VerificationConfig,
    *,
    run_name: str | None = None,
    question_ids: list[str] | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None] | None = None,
    sink: Any = None,
    store: bool = True,
) -> VerificationResultSet:
    """Attach a new rubric to a prior verification run.

    Enriches every row of ``prior_results`` with rubric scores
    produced against the rubric currently attached to this benchmark
    (global and per-question). Answering is replayed from the prior
    traces; template parsing and verification are skipped. Row count
    is preserved: the merged set has the same shape as
    ``prior_results`` with a populated ``rubric`` sub-object on each
    row.

    Trait scores are unioned with any rubric fields already present
    on prior rows. Same-name trait collisions raise ``ValueError``.

    Args:
        prior_results: Result set from an earlier ``run_verification``
            call.
        config: Verification configuration matching the shape of
            ``prior_results``. ``answering_models``,
            ``parsing_models``, and ``replicate_count`` must equal
            the values observed in ``prior_results``.
            ``replay_store`` must be ``None``;
            ``evaluation_mode`` must be ``"template_only"`` or
            ``"rubric_only"`` (the helper rewrites it internally).
        run_name: Optional override for the merged run name.
        question_ids: Optional subset of question IDs.
        async_enabled: Forwarded to ``run_verification``.
        progress_callback: Forwarded to ``run_verification``.
        sink: Optional :class:`~karenina.benchmark.verification.sinks.ResultSink`
            forwarded to ``run_verification``. The sink receives the
            ``rubric_only`` rows produced by the extension pipeline (one
            per prior triple); it does not see the enriched rows returned
            to the caller. Pair with
            :class:`~karenina.benchmark.verification.sinks.ProgressiveFileSink`
            to make rubric extension resumable.
        store: When True (default), write the merged set into the
            results manager.

    Returns:
        ``VerificationResultSet`` of enriched prior rows.

    Raises:
        ValueError: On input validation failure; see
            :func:`~karenina.benchmark.verification.extension.extend_rubric_run`.
    """
    from .verification.extension import extend_rubric_run

    merged = extend_rubric_run(
        self,
        prior_results,
        config,
        run_name=run_name,
        question_ids=question_ids,
        async_enabled=async_enabled,
        progress_callback=progress_callback,
        sink=sink,
    )
    if store:
        results_dict: dict[str, VerificationResult] = {}
        for idx, row in enumerate(merged.results):
            md = row.metadata
            key = f"{md.question_id}_{md.answering_model}_{md.parsing_model}"
            if md.replicate is not None:
                key += f"_rep{md.replicate}"
            if md.timestamp:
                key += f"_{md.timestamp}"
            if key in results_dict:
                key += f"_{idx}"
            results_dict[key] = row
        effective_run_name = merged.results[0].metadata.run_name if merged.results else run_name
        self._results_manager.store_verification_results(results_dict, effective_run_name)
    return merged

extend_template

extend_template(
    prior_results: VerificationResultSet,
    config: VerificationConfig,
    *,
    run_name: str | None = None,
    question_ids: list[str] | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None]
    | None = None,
    sink: Any = None,
    store: bool = True,
) -> VerificationResultSet

Extend a prior verification run along any combination of three axes.

Axes, all optional and composable in a single call:

1. New judges (config.parsing_models).
2. New answerers (config.answering_models).
3. More replicates (higher config.replicate_count).

Prior (question, answerer, replicate) answering traces are served from a :class:~karenina.replay.ReplayStore; new answerers, new replicates, or any combination thereof run answering live. Parsing always runs live. Triples already present in prior_results are skipped so prior rows pass through verbatim. The merged output matches a joint run over the full (answerers × judges × replicates) matrix.

Parameters:

Name	Type	Description	Default
prior_results	VerificationResultSet	Result set from an earlier run_verification call to extend.	required
config	VerificationConfig	Verification configuration describing the final state (full union, not deltas): every judge in parsing_models, every answerer in answering_models, and the final replicate_count (must be >= observed in prior_results). config.replay_store must be None.	required
run_name	str | None	Optional override for the merged run name. Defaults to the run name carried by prior_results.	None
question_ids	list[str] | None	Optional subset of question IDs to re-judge. Defaults to every question present in prior_results.	None
async_enabled	bool | None	Optional async control forwarded to run_verification.	None
progress_callback	Callable[[float, str], None] | None	Optional progress callback forwarded to run_verification.	None
sink	Any	Optional :class:~karenina.benchmark.verification.sinks.ResultSink forwarded to run_verification so newly produced rows are persisted as they complete. Prior rows pass through the merge verbatim and never reach the sink. Pair with :class:~karenina.benchmark.verification.sinks.ProgressiveFileSink to make the extension resumable.	None
store	bool	When True (default), also store the merged set into the in-memory results manager under run_name so it is available to get_verification_results and the exporters.	True

Returns:

Type	Description
VerificationResultSet	Merged VerificationResultSet with the prior rows plus the
VerificationResultSet	newly-produced rows, all stamped with the effective run_name.

Source code in src/karenina/benchmark/benchmark.py

def extend_template(
    self,
    prior_results: VerificationResultSet,
    config: VerificationConfig,
    *,
    run_name: str | None = None,
    question_ids: list[str] | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None] | None = None,
    sink: Any = None,
    store: bool = True,
) -> VerificationResultSet:
    """Extend a prior verification run along any combination of three axes.

    Axes, all optional and composable in a single call:

    1. New judges (``config.parsing_models``).
    2. New answerers (``config.answering_models``).
    3. More replicates (higher ``config.replicate_count``).

    Prior ``(question, answerer, replicate)`` answering traces are served
    from a :class:`~karenina.replay.ReplayStore`; new answerers, new
    replicates, or any combination thereof run answering live. Parsing
    always runs live. Triples already present in ``prior_results`` are
    skipped so prior rows pass through verbatim. The merged output matches
    a joint run over the full ``(answerers × judges × replicates)`` matrix.

    Args:
        prior_results: Result set from an earlier ``run_verification``
            call to extend.
        config: Verification configuration describing the **final**
            state (full union, not deltas): every judge in
            ``parsing_models``, every answerer in ``answering_models``,
            and the final ``replicate_count`` (must be ``>=`` observed
            in ``prior_results``). ``config.replay_store`` must be
            ``None``.
        run_name: Optional override for the merged run name. Defaults to
            the run name carried by ``prior_results``.
        question_ids: Optional subset of question IDs to re-judge.
            Defaults to every question present in ``prior_results``.
        async_enabled: Optional async control forwarded to
            ``run_verification``.
        progress_callback: Optional progress callback forwarded to
            ``run_verification``.
        sink: Optional :class:`~karenina.benchmark.verification.sinks.ResultSink`
            forwarded to ``run_verification`` so newly produced rows are
            persisted as they complete. Prior rows pass through the merge
            verbatim and never reach the sink. Pair with
            :class:`~karenina.benchmark.verification.sinks.ProgressiveFileSink`
            to make the extension resumable.
        store: When True (default), also store the merged set into the
            in-memory results manager under ``run_name`` so it is
            available to ``get_verification_results`` and the exporters.

    Returns:
        Merged ``VerificationResultSet`` with the prior rows plus the
        newly-produced rows, all stamped with the effective ``run_name``.
    """
    from .verification.extension import extend_template_run

    merged = extend_template_run(
        self,
        prior_results,
        config,
        run_name=run_name,
        question_ids=question_ids,
        async_enabled=async_enabled,
        progress_callback=progress_callback,
        sink=sink,
    )
    if store:
        results_dict: dict[str, VerificationResult] = {}
        for idx, row in enumerate(merged.results):
            md = row.metadata
            key = f"{md.question_id}_{md.answering_model}_{md.parsing_model}"
            if md.replicate is not None:
                key += f"_rep{md.replicate}"
            if md.timestamp:
                key += f"_{md.timestamp}"
            if key in results_dict:
                key += f"_{idx}"
            results_dict[key] = row
        effective_run_name = merged.results[0].metadata.run_name if merged.results else run_name
        self._results_manager.store_verification_results(results_dict, effective_run_name)
    return merged

filter_by_custom_metadata

filter_by_custom_metadata(
    match_all: bool = True, **criteria: Any
) -> list[dict[str, Any]]

Filter questions by custom metadata fields with AND/OR logic.

Source code in src/karenina/benchmark/benchmark.py

def filter_by_custom_metadata(self, match_all: bool = True, **criteria: Any) -> list[dict[str, Any]]:
    """Filter questions by custom metadata fields with AND/OR logic."""
    return self._question_manager.filter_by_custom_metadata(match_all, **criteria)

filter_by_metadata

filter_by_metadata(
    field_path: str, value: Any, match_mode: str = "exact"
) -> list[dict[str, Any]]

Filter questions by a metadata field using dot notation.

Source code in src/karenina/benchmark/benchmark.py

def filter_by_metadata(self, field_path: str, value: Any, match_mode: str = "exact") -> list[dict[str, Any]]:
    """Filter questions by a metadata field using dot notation."""
    return self._question_manager.filter_by_metadata(field_path, value, match_mode)

filter_questions

filter_questions(
    finished: bool | None = None,
    has_template: bool | None = None,
    has_rubric: bool | None = None,
    author: str | None = None,
    custom_filter: Any = None,
) -> list[dict[str, Any]]

Filter questions based on criteria.

Source code in src/karenina/benchmark/benchmark.py

def filter_questions(
    self,
    finished: bool | None = None,
    has_template: bool | None = None,
    has_rubric: bool | None = None,
    author: str | None = None,
    custom_filter: Any = None,
) -> list[dict[str, Any]]:
    """Filter questions based on criteria."""
    return self._question_manager.filter_questions(finished, has_template, has_rubric, author, custom_filter)

generate_all_templates

generate_all_templates(
    model: str = "claude-haiku-4-5",
    model_provider: str = "anthropic",
    temperature: float = 0,
    interface: str = "langchain",
    force_regenerate: bool = False,
    progress_callback: Callable[
        [TemplateProgressEvent], None
    ]
    | None = None,
    only_missing: bool = True,
    endpoint_base_url: str | None = None,
    endpoint_api_key: str | None = None,
    progressive_backup: bool = True,
    backup_path: Path | None = None,
    max_workers: int | None = None,
    cancel_event: Event | None = None,
) -> dict[str, dict[str, Any]]

Generate templates for all questions in the benchmark using LLM.

Parameters:

Name	Type	Description	Default
model	str	Model name.	'claude-haiku-4-5'
model_provider	str	Model provider.	'anthropic'
temperature	float	Generation temperature.	0
interface	str	Adapter interface.	'langchain'
force_regenerate	bool	If True, regenerate existing templates.	False
progress_callback	Callable[[TemplateProgressEvent], None] | None	Optional progress callback.	None
only_missing	bool	If True, only generate for questions without templates.	True
endpoint_base_url	str | None	Optional custom endpoint URL.	None
endpoint_api_key	str | None	Optional API key for custom endpoint.	None
progressive_backup	bool	If True (default), save generated templates to a backup file after each successful generation so interrupted runs can be resumed.	True
backup_path	Path | None	Path for the backup file. Defaults to {benchmark_name}_templates_backup.json in the current directory.	None
max_workers	int | None	Number of parallel workers. None reads from KARENINA_ASYNC_MAX_WORKERS env var (default 1). 1 = sequential.	None
cancel_event	Event | None	If set, stops generation after the current task(s) complete.	None

Source code in src/karenina/benchmark/benchmark.py

def generate_all_templates(
    self,
    model: str = "claude-haiku-4-5",
    model_provider: str = "anthropic",
    temperature: float = 0,
    interface: str = "langchain",
    force_regenerate: bool = False,
    progress_callback: "Callable[[_helpers.TemplateProgressEvent], None] | None" = None,
    only_missing: bool = True,
    endpoint_base_url: str | None = None,
    endpoint_api_key: str | None = None,
    progressive_backup: bool = True,
    backup_path: Path | None = None,
    max_workers: int | None = None,
    cancel_event: "threading.Event | None" = None,
) -> dict[str, dict[str, Any]]:
    """Generate templates for all questions in the benchmark using LLM.

    Args:
        model: Model name.
        model_provider: Model provider.
        temperature: Generation temperature.
        interface: Adapter interface.
        force_regenerate: If True, regenerate existing templates.
        progress_callback: Optional progress callback.
        only_missing: If True, only generate for questions without templates.
        endpoint_base_url: Optional custom endpoint URL.
        endpoint_api_key: Optional API key for custom endpoint.
        progressive_backup: If True (default), save generated templates to a
            backup file after each successful generation so interrupted runs
            can be resumed.
        backup_path: Path for the backup file. Defaults to
            ``{benchmark_name}_templates_backup.json`` in the current directory.
        max_workers: Number of parallel workers. None reads from
            KARENINA_ASYNC_MAX_WORKERS env var (default 1). 1 = sequential.
        cancel_event: If set, stops generation after the current task(s) complete.
    """
    return _helpers.generate_all_templates(
        self,
        model,
        model_provider,
        temperature,
        interface,
        force_regenerate,
        progress_callback,
        only_missing,
        endpoint_base_url,
        endpoint_api_key,
        progressive_backup,
        backup_path,
        max_workers=max_workers,
        cancel_event=cancel_event,
    )

generate_template_for_question

generate_template_for_question(
    question_id: str,
    model: str = "claude-haiku-4-5",
    model_provider: str = "anthropic",
    temperature: float = 0,
    interface: str = "langchain",
    force_regenerate: bool = False,
    endpoint_base_url: str | None = None,
    endpoint_api_key: str | None = None,
) -> dict[str, Any]

Generate an answer template for a specific question using LLM.

Source code in src/karenina/benchmark/benchmark.py

def generate_template_for_question(
    self,
    question_id: str,
    model: str = "claude-haiku-4-5",
    model_provider: str = "anthropic",
    temperature: float = 0,
    interface: str = "langchain",
    force_regenerate: bool = False,
    endpoint_base_url: str | None = None,
    endpoint_api_key: str | None = None,
) -> dict[str, Any]:
    """Generate an answer template for a specific question using LLM."""
    return _helpers.generate_template_for_question(
        self,
        question_id,
        model,
        model_provider,
        temperature,
        interface,
        force_regenerate,
        endpoint_base_url,
        endpoint_api_key,
    )

generate_templates

generate_templates(
    question_ids: list[str],
    model: str = "claude-haiku-4-5",
    model_provider: str = "anthropic",
    temperature: float = 0,
    interface: str = "langchain",
    force_regenerate: bool = False,
    progress_callback: Callable[
        [TemplateProgressEvent], None
    ]
    | None = None,
    endpoint_base_url: str | None = None,
    endpoint_api_key: str | None = None,
    max_workers: int | None = None,
    cancel_event: Event | None = None,
) -> dict[str, dict[str, Any]]

Generate templates for multiple questions using LLM.

Source code in src/karenina/benchmark/benchmark.py

def generate_templates(
    self,
    question_ids: list[str],
    model: str = "claude-haiku-4-5",
    model_provider: str = "anthropic",
    temperature: float = 0,
    interface: str = "langchain",
    force_regenerate: bool = False,
    progress_callback: "Callable[[_helpers.TemplateProgressEvent], None] | None" = None,
    endpoint_base_url: str | None = None,
    endpoint_api_key: str | None = None,
    max_workers: int | None = None,
    cancel_event: "threading.Event | None" = None,
) -> dict[str, dict[str, Any]]:
    """Generate templates for multiple questions using LLM."""
    return _helpers.generate_templates(
        self,
        question_ids,
        model,
        model_provider,
        temperature,
        interface,
        force_regenerate,
        progress_callback,
        endpoint_base_url,
        endpoint_api_key,
        max_workers=max_workers,
        cancel_event=cancel_event,
    )

get_all_custom_properties

get_all_custom_properties() -> dict[str, Any]

Get all custom properties as a dictionary.

Source code in src/karenina/benchmark/benchmark.py

def get_all_custom_properties(self) -> dict[str, Any]:
    """Get all custom properties as a dictionary."""
    return self._metadata_manager.get_all_custom_properties()

get_all_questions

get_all_questions(
    ids_only: bool = False,
) -> list[str] | list[dict[str, Any]]

Get all questions in the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def get_all_questions(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
    """Get all questions in the benchmark."""
    return self._question_manager.get_all_questions(ids_only)

get_all_questions_as_objects

get_all_questions_as_objects() -> list[Question]

Get all questions as Question objects.

Source code in src/karenina/benchmark/benchmark.py

def get_all_questions_as_objects(self) -> list["Question"]:
    """Get all questions as Question objects."""
    return self._question_manager.get_all_questions_as_objects()

get_all_run_names

get_all_run_names() -> list[str]

Get all verification run names.

Source code in src/karenina/benchmark/benchmark.py

def get_all_run_names(self) -> list[str]:
    """Get all verification run names."""
    warnings.warn(
        "get_all_run_names is deprecated. Use ResultsStore.get_all_runs() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.get_all_run_names()

get_custom_property

get_custom_property(name: str) -> Any

Get a custom property from benchmark metadata.

Source code in src/karenina/benchmark/benchmark.py

def get_custom_property(self, name: str) -> Any:
    """Get a custom property from benchmark metadata."""
    return self._metadata_manager.get_custom_property(name)

get_finished_questions

get_finished_questions(
    ids_only: bool = False,
) -> list[str] | list[dict[str, Any]]

Get questions that are marked as finished.

Source code in src/karenina/benchmark/benchmark.py

def get_finished_questions(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
    """Get questions that are marked as finished."""
    return self._question_manager.get_finished_questions(ids_only)

get_finished_templates

get_finished_templates(
    question_ids: set[str] | None = None,
) -> list[FinishedTemplate]

Get all finished templates for verification.

Source code in src/karenina/benchmark/benchmark.py

def get_finished_templates(self, question_ids: set[str] | None = None) -> list[FinishedTemplate]:
    """Get all finished templates for verification."""
    return self._template_manager.get_finished_templates(question_ids=question_ids)

get_global_dynamic_rubric

get_global_dynamic_rubric() -> DynamicRubric | None

Get the global dynamic rubric from the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def get_global_dynamic_rubric(self) -> DynamicRubric | None:
    """Get the global dynamic rubric from the benchmark."""
    return self._rubric_manager.get_global_dynamic_rubric()

get_global_rubric

get_global_rubric() -> Rubric | None

Get the global rubric from the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def get_global_rubric(self) -> Rubric | None:
    """Get the global rubric from the benchmark."""
    return self._rubric_manager.get_global_rubric()

get_health_report

get_health_report() -> dict[str, Any]

Get comprehensive health/status report.

Source code in src/karenina/benchmark/benchmark.py

def get_health_report(self) -> dict[str, Any]:
    """Get comprehensive health/status report."""
    return self._export_manager.get_health_report()

get_merged_dynamic_rubric_for_question

get_merged_dynamic_rubric_for_question(
    question_id: str,
) -> DynamicRubric | None

Get merged dynamic rubric for a question (global + question-specific).

Parameters:

Name	Type	Description	Default
question_id	str	The question ID.	required

Returns:

Type	Description
DynamicRubric | None	Merged DynamicRubric or None if neither global nor question-level exists.

Source code in src/karenina/benchmark/benchmark.py

def get_merged_dynamic_rubric_for_question(self, question_id: str) -> DynamicRubric | None:
    """Get merged dynamic rubric for a question (global + question-specific).

    Args:
        question_id: The question ID.

    Returns:
        Merged DynamicRubric or None if neither global nor question-level exists.
    """
    return self._rubric_manager.get_merged_dynamic_rubric_for_question(question_id)

get_missing_templates

get_missing_templates(
    ids_only: bool = False,
) -> list[str] | list[dict[str, Any]]

Get questions that don't have non-default templates.

Source code in src/karenina/benchmark/benchmark.py

def get_missing_templates(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
    """Get questions that don't have non-default templates."""
    return self._template_manager.get_missing_templates(ids_only)

get_progress

get_progress() -> float

Get completion progress as percentage (0-100).

Source code in src/karenina/benchmark/benchmark.py

def get_progress(self) -> float:
    """Get completion progress as percentage (0-100)."""
    return self._base.get_progress()

get_question

get_question(question_id: str) -> dict[str, Any]

Get a question by ID.

Source code in src/karenina/benchmark/benchmark.py

def get_question(self, question_id: str) -> dict[str, Any]:
    """Get a question by ID."""
    return self._question_manager.get_question(question_id)

get_question_as_object

get_question_as_object(question_id: str) -> Question

Get a question as a Question object.

Source code in src/karenina/benchmark/benchmark.py

def get_question_as_object(self, question_id: str) -> "Question":
    """Get a question as a Question object."""
    return self._question_manager.get_question_as_object(question_id)

get_question_author

get_question_author(
    question_id: str,
) -> dict[str, Any] | None

Get author information for a question.

Source code in src/karenina/benchmark/benchmark.py

def get_question_author(self, question_id: str) -> dict[str, Any] | None:
    """Get author information for a question."""
    return self._question_manager.get_question_author(question_id)

get_question_custom_property

get_question_custom_property(
    question_id: str, name: str
) -> Any

Get a custom property from question metadata.

Source code in src/karenina/benchmark/benchmark.py

def get_question_custom_property(self, question_id: str, name: str) -> Any:
    """Get a custom property from question metadata."""
    return self._question_manager.get_question_custom_property(question_id, name)

get_question_ids

get_question_ids() -> list[str]

Get all question IDs in the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def get_question_ids(self) -> list[str]:
    """Get all question IDs in the benchmark."""
    return self._question_manager.get_question_ids()

get_question_metadata

get_question_metadata(question_id: str) -> dict[str, Any]

Get all metadata for a specific question.

Source code in src/karenina/benchmark/benchmark.py

def get_question_metadata(self, question_id: str) -> dict[str, Any]:
    """Get all metadata for a specific question."""
    return self._question_manager.get_question_metadata(question_id)

get_question_rubric

get_question_rubric(question_id: str) -> Rubric | None

Get the question-specific rubric for a question.

Parameters:

Name	Type	Description	Default
question_id	str	The question ID.	required

Returns:

Type	Description
Rubric | None	Rubric containing the question-specific traits, or None.

Source code in src/karenina/benchmark/benchmark.py

def get_question_rubric(self, question_id: str) -> Rubric | None:
    """Get the question-specific rubric for a question.

    Args:
        question_id: The question ID.

    Returns:
        Rubric containing the question-specific traits, or None.
    """
    raw = self._rubric_manager.get_question_rubric(question_id)
    if raw is None:
        return None
    if isinstance(raw, dict):
        return Rubric(
            llm_traits=raw.get("llm_traits", []),
            regex_traits=raw.get("regex_traits", []),
            callable_traits=raw.get("callable_traits", []),
            metric_traits=raw.get("metric_traits", []),
            agentic_traits=raw.get("agentic_traits", []),
        )
    return Rubric.from_traits(raw)

get_question_sources

get_question_sources(
    question_id: str,
) -> list[dict[str, Any]] | None

Get source documents for a question.

Source code in src/karenina/benchmark/benchmark.py

def get_question_sources(self, question_id: str) -> list[dict[str, Any]] | None:
    """Get source documents for a question."""
    return self._question_manager.get_question_sources(question_id)

get_question_timestamps

get_question_timestamps(question_id: str) -> dict[str, str]

Get creation and modification timestamps for a question.

Source code in src/karenina/benchmark/benchmark.py

def get_question_timestamps(self, question_id: str) -> dict[str, str]:
    """Get creation and modification timestamps for a question."""
    return self._question_manager.get_question_timestamps(question_id)

get_questions_by_author

get_questions_by_author(
    author: str,
) -> list[dict[str, Any]]

Get questions created by a specific author.

Source code in src/karenina/benchmark/benchmark.py

def get_questions_by_author(self, author: str) -> list[dict[str, Any]]:
    """Get questions created by a specific author."""
    return self._question_manager.get_questions_by_author(author)

get_questions_with_rubric

get_questions_with_rubric() -> list[dict[str, Any]]

Get questions that have question-specific rubrics.

Source code in src/karenina/benchmark/benchmark.py

def get_questions_with_rubric(self) -> list[dict[str, Any]]:
    """Get questions that have question-specific rubrics."""
    return self._question_manager.get_questions_with_rubric()

get_results_statistics_by_run

get_results_statistics_by_run() -> dict[
    str, dict[str, Any]
]

Get verification statistics for each run.

Source code in src/karenina/benchmark/benchmark.py

def get_results_statistics_by_run(self) -> dict[str, dict[str, Any]]:
    """Get verification statistics for each run."""
    warnings.warn(
        "get_results_statistics_by_run is deprecated. Use ResultsStore.get_statistics_by_run() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.get_results_statistics_by_run()

get_scenario

get_scenario(name: str) -> ScenarioDefinition

Get a scenario by name.

Parameters:

Name	Type	Description	Default
name	str	The scenario name.	required

Returns:

Type	Description
ScenarioDefinition	The ScenarioDefinition.

Raises:

Type	Description
KeyError	If no scenario with that name exists.

Source code in src/karenina/benchmark/benchmark.py

def get_scenario(self, name: str) -> ScenarioDefinition:
    """Get a scenario by name.

    Args:
        name: The scenario name.

    Returns:
        The ScenarioDefinition.

    Raises:
        KeyError: If no scenario with that name exists.
    """
    try:
        return self._scenarios[name]
    except KeyError:
        raise KeyError(f"Scenario '{name}' not found") from None

get_scenarios

get_scenarios() -> list[ScenarioDefinition]

Get all scenario definitions.

Returns:

Type	Description
list[ScenarioDefinition]	List of ScenarioDefinition instances.

Source code in src/karenina/benchmark/benchmark.py

def get_scenarios(self) -> list[ScenarioDefinition]:
    """Get all scenario definitions.

    Returns:
        List of ScenarioDefinition instances.
    """
    return list(self._scenarios.values())

get_statistics

get_statistics() -> dict[str, Any]

Get detailed statistics about the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def get_statistics(self) -> dict[str, Any]:
    """Get detailed statistics about the benchmark."""
    return self._export_manager.get_statistics()

get_summary

get_summary() -> dict[str, Any]

Get comprehensive benchmark statistics.

Source code in src/karenina/benchmark/benchmark.py

def get_summary(self) -> dict[str, Any]:
    """Get comprehensive benchmark statistics."""
    return self._export_manager.get_summary()

get_template

get_template(question_id: str) -> str

Get template code for a question.

Source code in src/karenina/benchmark/benchmark.py

def get_template(self, question_id: str) -> str:
    """Get template code for a question."""
    return self._template_manager.get_template(question_id)

get_unfinished_questions

get_unfinished_questions(
    ids_only: bool = False,
) -> list[str] | list[dict[str, Any]]

Get questions that are not marked as finished.

Source code in src/karenina/benchmark/benchmark.py

def get_unfinished_questions(self, ids_only: bool = False) -> list[str] | list[dict[str, Any]]:
    """Get questions that are not marked as finished."""
    return self._question_manager.get_unfinished_questions(ids_only)

get_verification_history

get_verification_history(
    question_id: str | None = None,
) -> dict[str, dict[str, VerificationResult]]

Get verification history organized by run name.

Source code in src/karenina/benchmark/benchmark.py

def get_verification_history(self, question_id: str | None = None) -> dict[str, dict[str, VerificationResult]]:
    """Get verification history organized by run name."""
    warnings.warn(
        "get_verification_history is deprecated. Use ResultsStore.get_by_question() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.get_verification_history(question_id)

get_verification_results

get_verification_results(
    question_ids: list[str] | None = None,
    run_name: str | None = None,
) -> dict[str, VerificationResult]

Get verification results for specific questions and/or runs.

Source code in src/karenina/benchmark/benchmark.py

def get_verification_results(
    self,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
) -> dict[str, VerificationResult]:
    """Get verification results for specific questions and/or runs."""
    warnings.warn(
        "get_verification_results is deprecated. Use ResultsStore.get_by_run() or ResultsStore.get_latest() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.get_verification_results(question_ids, run_name)

get_verification_summary

get_verification_summary(
    run_name: str | None = None,
) -> dict[str, Any]

Get summary statistics for verification results.

Source code in src/karenina/benchmark/benchmark.py

def get_verification_summary(self, run_name: str | None = None) -> dict[str, Any]:
    """Get summary statistics for verification results."""
    warnings.warn(
        "get_verification_summary is deprecated. Use ResultsStore.get_summary() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.get_verification_summary(run_name)

has_template

has_template(question_id: str) -> bool

Check if a question has a non-default template.

Source code in src/karenina/benchmark/benchmark.py

def has_template(self, question_id: str) -> bool:
    """Check if a question has a non-default template."""
    return self._template_manager.has_template(question_id)

import_generated_templates

import_generated_templates(
    file_path: Path, force_overwrite: bool = False
) -> dict[str, bool]

Import templates from a JSON file generated by export_generated_templates.

Source code in src/karenina/benchmark/benchmark.py

def import_generated_templates(self, file_path: Path, force_overwrite: bool = False) -> dict[str, bool]:
    """Import templates from a JSON file generated by export_generated_templates."""
    return _helpers.import_generated_templates(self, file_path, force_overwrite)

load classmethod

load(
    path: Path, workspace_root: Path | None = None
) -> Benchmark

Load a benchmark from a JSON-LD file.

Parameters:

Name	Type	Description	Default
path	Path	Path to the JSON-LD benchmark file.	required
workspace_root	Path | None	Optional root directory for task workspaces.	None

Source code in src/karenina/benchmark/benchmark.py

@classmethod
def load(cls, path: Path, workspace_root: Path | None = None) -> "Benchmark":
    """Load a benchmark from a JSON-LD file.

    Args:
        path: Path to the JSON-LD benchmark file.
        workspace_root: Optional root directory for task workspaces.
    """
    base = BenchmarkBase.load(path)
    instance = cls.__new__(cls)
    instance._base = base
    instance._workspace_root = workspace_root
    instance._init_managers()
    return instance

load_from_db classmethod

load_from_db(
    benchmark_name: str, storage: str
) -> Benchmark

Load a benchmark from a database.

Source code in src/karenina/benchmark/benchmark.py

@classmethod
def load_from_db(cls, benchmark_name: str, storage: str) -> "Benchmark":
    """Load a benchmark from a database."""
    from ..storage import load_benchmark

    result = load_benchmark(benchmark_name, storage, load_config=False)
    return result  # type: ignore[return-value]

load_verification_results_from_file

load_verification_results_from_file(
    file_path: Path, run_name: str | None = None
) -> dict[str, VerificationResult]

Load verification results from a previously exported file.

Source code in src/karenina/benchmark/benchmark.py

def load_verification_results_from_file(
    self,
    file_path: Path,
    run_name: str | None = None,
) -> dict[str, VerificationResult]:
    """Load verification results from a previously exported file."""
    warnings.warn(
        "load_verification_results_from_file is deprecated. Use ResultsStore.from_file() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    return self._results_manager.load_results_from_file(file_path, run_name)

mark_finished

mark_finished(question_id: str) -> None

Mark a question as finished.

Source code in src/karenina/benchmark/benchmark.py

def mark_finished(self, question_id: str) -> None:
    """Mark a question as finished."""
    self._question_manager.mark_finished(question_id)

mark_finished_batch

mark_finished_batch(question_ids: list[str]) -> None

Mark multiple questions as finished.

Source code in src/karenina/benchmark/benchmark.py

def mark_finished_batch(self, question_ids: list[str]) -> None:
    """Mark multiple questions as finished."""
    self._question_manager.mark_finished_batch(question_ids)

mark_unfinished

mark_unfinished(question_id: str) -> None

Mark a question as unfinished.

Source code in src/karenina/benchmark/benchmark.py

def mark_unfinished(self, question_id: str) -> None:
    """Mark a question as unfinished."""
    self._question_manager.mark_unfinished(question_id)

mark_unfinished_batch

mark_unfinished_batch(question_ids: list[str]) -> None

Mark multiple questions as unfinished.

Source code in src/karenina/benchmark/benchmark.py

def mark_unfinished_batch(self, question_ids: list[str]) -> None:
    """Mark multiple questions as unfinished."""
    self._question_manager.mark_unfinished_batch(question_ids)

optimization_history

optimization_history(
    tracker_path: Path
    | str = "~/.karenina/optimization_history.db",
    limit: int = 20,
) -> list[OptimizationRun]

Get optimization history for this benchmark.

Source code in src/karenina/benchmark/benchmark.py

def optimization_history(
    self,
    tracker_path: Path | str = "~/.karenina/optimization_history.db",
    limit: int = 20,
) -> list["OptimizationRun"]:
    """Get optimization history for this benchmark."""
    try:
        from karenina.integrations.gepa import OptimizationTracker
    except ImportError:
        return []

    tracker = OptimizationTracker(tracker_path)
    return tracker.list_runs(benchmark_name=self.name, limit=limit)

optimize

optimize(
    targets: list[str],
    config: VerificationConfig | None = None,
    train_ratio: float = 0.8,
    val_ratio: float = 0.2,
    test_ratio: float | None = None,
    seed: int | None = None,
    reflection_model: str = "openai/gpt-4o",
    max_metric_calls: int = 150,
    objective_config: ObjectiveConfig | None = None,
    frontier_type: FrontierType = "objective",
    seed_prompts: dict[str, str] | None = None,
    tracker_path: Path | str | None = None,
    export_preset_path: Path | str | None = None,
    progress_callback: Callable[[float, str], None]
    | None = None,
    verbose: bool = False,
) -> KareninaOutput

Optimize text components using GEPA with karenina verification as the metric.

Requires the 'gepa' optional dependency: pip install karenina[gepa]

Parameters:

Name	Type	Description	Default
targets	list[str]	List of components to optimize. Valid values: "answering_system_prompt", "parsing_instructions", "mcp_tool_descriptions"	required
config	VerificationConfig | None	Base VerificationConfig to use. If None, uses default minimal config.	None
train_ratio	float	Fraction of questions for training (default 0.8)	0.8
val_ratio	float	Fraction of questions for validation (default 0.2)	0.2
test_ratio	float | None	Optional fraction for testing. If None, no test set created.	None
seed	int | None	Random seed for reproducibility	None
reflection_model	str	Model for GEPA's reflection LLM (default: openai/gpt-4o)	'openai/gpt-4o'
max_metric_calls	int	Maximum GEPA optimization iterations (default: 150)	150
objective_config	ObjectiveConfig | None	Configuration for multi-objective optimization dimensions.	None
frontier_type	FrontierType	GEPA Pareto frontier tracking strategy.	'objective'
seed_prompts	dict[str, str] | None	Optional initial prompts. If None, uses empty strings.	None
tracker_path	Path | str | None	Optional path to SQLite file for tracking optimization history	None
export_preset_path	Path | str | None	Optional path to export optimized config as preset	None
progress_callback	Callable[[float, str], None] | None	Optional callback for progress updates (percentage, message)	None
verbose	bool	If True, display detailed progress during optimization	False

Returns:

Type	Description
KareninaOutput	KareninaOutput with optimized prompts and metrics

Example

result = benchmark.optimize( ... targets=["answering_system_prompt"], ... reflection_model="openai/gpt-4o", ... max_metric_calls=100, ... ) print(f"Improvement: {result.improvement:.1%}")

Source code in src/karenina/benchmark/benchmark.py

def optimize(
    self,
    targets: list[str],
    config: VerificationConfig | None = None,
    train_ratio: float = 0.8,
    val_ratio: float = 0.2,
    test_ratio: float | None = None,
    seed: int | None = None,
    reflection_model: str = "openai/gpt-4o",
    max_metric_calls: int = 150,
    objective_config: "ObjectiveConfig | None" = None,
    frontier_type: "FrontierType" = "objective",
    seed_prompts: dict[str, str] | None = None,
    tracker_path: Path | str | None = None,
    export_preset_path: Path | str | None = None,
    progress_callback: Callable[[float, str], None] | None = None,
    verbose: bool = False,
) -> "KareninaOutput":
    """
    Optimize text components using GEPA with karenina verification as the metric.

    Requires the 'gepa' optional dependency: pip install karenina[gepa]

    Args:
        targets: List of components to optimize. Valid values:
                 "answering_system_prompt", "parsing_instructions", "mcp_tool_descriptions"
        config: Base VerificationConfig to use. If None, uses default minimal config.
        train_ratio: Fraction of questions for training (default 0.8)
        val_ratio: Fraction of questions for validation (default 0.2)
        test_ratio: Optional fraction for testing. If None, no test set created.
        seed: Random seed for reproducibility
        reflection_model: Model for GEPA's reflection LLM (default: openai/gpt-4o)
        max_metric_calls: Maximum GEPA optimization iterations (default: 150)
        objective_config: Configuration for multi-objective optimization dimensions.
        frontier_type: GEPA Pareto frontier tracking strategy.
        seed_prompts: Optional initial prompts. If None, uses empty strings.
        tracker_path: Optional path to SQLite file for tracking optimization history
        export_preset_path: Optional path to export optimized config as preset
        progress_callback: Optional callback for progress updates (percentage, message)
        verbose: If True, display detailed progress during optimization

    Returns:
        KareninaOutput with optimized prompts and metrics

    Example:
        >>> result = benchmark.optimize(
        ...     targets=["answering_system_prompt"],
        ...     reflection_model="openai/gpt-4o",
        ...     max_metric_calls=100,
        ... )
        >>> print(f"Improvement: {result.improvement:.1%}")
    """
    return _helpers.run_optimize(
        self,
        targets,
        config,
        train_ratio,
        val_ratio,
        test_ratio,
        seed,
        reflection_model,
        max_metric_calls,
        objective_config,
        frontier_type,
        seed_prompts,
        tracker_path,
        export_preset_path,
        progress_callback,
        verbose,
    )

remove_custom_property

remove_custom_property(name: str) -> bool

Remove a custom property from benchmark metadata.

Source code in src/karenina/benchmark/benchmark.py

def remove_custom_property(self, name: str) -> bool:
    """Remove a custom property from benchmark metadata."""
    return self._metadata_manager.remove_custom_property(name)

remove_question

remove_question(question_id: str) -> bool

Remove a specific question from the benchmark.

Source code in src/karenina/benchmark/benchmark.py

def remove_question(self, question_id: str) -> bool:
    """Remove a specific question from the benchmark."""
    return self._question_manager.remove_question(question_id)

remove_question_custom_property

remove_question_custom_property(
    question_id: str, name: str
) -> bool

Remove a custom property from question metadata.

Source code in src/karenina/benchmark/benchmark.py

def remove_question_custom_property(self, question_id: str, name: str) -> bool:
    """Remove a custom property from question metadata."""
    return self._question_manager.remove_question_custom_property(question_id, name)

remove_question_rubric

remove_question_rubric(question_id: str) -> bool

Remove question-specific rubric.

Source code in src/karenina/benchmark/benchmark.py

def remove_question_rubric(self, question_id: str) -> bool:
    """Remove question-specific rubric."""
    return self._rubric_manager.remove_question_rubric(question_id)

remove_scenario

remove_scenario(name: str) -> None

Remove a scenario by name.

Parameters:

Name	Type	Description	Default
name	str	The scenario name.	required

Raises:

Type	Description
KeyError	If no scenario with that name exists.

Source code in src/karenina/benchmark/benchmark.py

def remove_scenario(self, name: str) -> None:
    """Remove a scenario by name.

    Args:
        name: The scenario name.

    Raises:
        KeyError: If no scenario with that name exists.
    """
    try:
        del self._scenarios[name]
    except KeyError:
        raise KeyError(f"Scenario '{name}' not found") from None

    # Remove from checkpoint
    if self._base._checkpoint.hasPart:
        self._base._checkpoint.hasPart = [s for s in self._base._checkpoint.hasPart if s.name != name]
        if not self._base._checkpoint.hasPart:
            self._base._checkpoint.hasPart = None
            # Clear benchmark_type flag when no scenarios remain
            if self._base._checkpoint.additionalProperty:
                self._base._checkpoint.additionalProperty = [
                    p for p in self._base._checkpoint.additionalProperty if p.name != "benchmark_type"
                ]

resume_verification

resume_verification(
    state_path: str | Path,
    *,
    config: VerificationConfig | None = None,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None]
    | None = None,
) -> VerificationResultSet

Resume a progressive-save run from its .state file.

Reconstructs a :class:ProgressiveFileSink from state_path, populates config.skip_triples via the sink, and delegates to :meth:run_verification. The config stored in the state file is used unless config is provided explicitly (useful for tweaking e.g. request_timeout before the retry pass).

Parameters:

Name	Type	Description	Default
state_path	str | Path	Path to the .state sidecar from a prior run.	required
config	VerificationConfig | None	Optional override config; defaults to the one snapshot in the state file.	None
question_ids	list[str] | None	Optional subset of questions.	None
run_name	str | None	Optional run label (defaults to the state's label).	None
async_enabled	bool | None	Override for async execution.	None
progress_callback	Callable[[float, str], None] | None	UI progress callback.	None

Returns:

Type	Description
VerificationResultSet	The batch's :class:VerificationResultSet. Previously-completed
VerificationResultSet	results remain in the sink's buffer; the returned set reflects
VerificationResultSet	whatever the executor produced in this resume pass.

Source code in src/karenina/benchmark/benchmark.py

def resume_verification(
    self,
    state_path: "str | Path",
    *,
    config: VerificationConfig | None = None,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None] | None = None,
) -> VerificationResultSet:
    """Resume a progressive-save run from its ``.state`` file.

    Reconstructs a :class:`ProgressiveFileSink` from ``state_path``,
    populates ``config.skip_triples`` via the sink, and delegates to
    :meth:`run_verification`. The config stored in the state file is
    used unless ``config`` is provided explicitly (useful for tweaking
    e.g. ``request_timeout`` before the retry pass).

    Args:
        state_path: Path to the ``.state`` sidecar from a prior run.
        config: Optional override config; defaults to the one snapshot
            in the state file.
        question_ids: Optional subset of questions.
        run_name: Optional run label (defaults to the state's label).
        async_enabled: Override for async execution.
        progress_callback: UI progress callback.

    Returns:
        The batch's :class:`VerificationResultSet`. Previously-completed
        results remain in the sink's buffer; the returned set reflects
        whatever the executor produced in this resume pass.
    """
    from karenina.benchmark.verification.sinks import ProgressiveFileSink

    sink = ProgressiveFileSink.load_for_resume(Path(state_path))
    sink.set_global_rubric(self._rubric_manager.get_global_rubric())

    effective_config = config if config is not None else sink.config
    return self.run_verification(
        config=effective_config,
        question_ids=question_ids,
        run_name=run_name,
        async_enabled=async_enabled,
        progress_callback=progress_callback,
        sink=sink,
    )

run_verification

run_verification(
    config: VerificationConfig,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None]
    | None = None,
    sink: Any = None,
) -> VerificationResultSet

Run verification on the benchmark using existing execution system.

For scenario benchmarks, dispatches to _run_scenario_verification which iterates over the scenario x model cross-product. For standalone question benchmarks, delegates to VerificationManager.

Parameters:

Name	Type	Description	Default
config	VerificationConfig	Verification configuration.	required
question_ids	list[str] | None	Optional filter for which questions to run.	None
run_name	str | None	Optional run label.	None
async_enabled	bool | None	Whether to run in parallel.	None
progress_callback	Callable[[float, str], None] | None	Optional UI progress callback.	None
sink	Any	Optional :class:ResultSink for progressive save and crash recovery. Pass a :class:ProgressiveFileSink to get --resume-compatible sidecars written incrementally, or a :class:CompositeSink to combine multiple persistence strategies. Scenario mode stores per-turn verification rows and skips already-completed scenario/model/parser combos on resume.	None

Source code in src/karenina/benchmark/benchmark.py

def run_verification(
    self,
    config: VerificationConfig,
    question_ids: list[str] | None = None,
    run_name: str | None = None,
    async_enabled: bool | None = None,
    progress_callback: Callable[[float, str], None] | None = None,
    sink: Any = None,
) -> VerificationResultSet:
    """Run verification on the benchmark using existing execution system.

    For scenario benchmarks, dispatches to ``_run_scenario_verification``
    which iterates over the scenario x model cross-product.
    For standalone question benchmarks, delegates to VerificationManager.

    Args:
        config: Verification configuration.
        question_ids: Optional filter for which questions to run.
        run_name: Optional run label.
        async_enabled: Whether to run in parallel.
        progress_callback: Optional UI progress callback.
        sink: Optional :class:`ResultSink` for progressive save and
            crash recovery. Pass a :class:`ProgressiveFileSink` to get
            ``--resume``-compatible sidecars written incrementally, or a
            :class:`CompositeSink` to combine multiple persistence
            strategies. Scenario mode stores per-turn verification rows
            and skips already-completed scenario/model/parser combos on
            resume.
    """
    # Auto-build replay store from legacy interface="manual" models.
    # The benchmark is only reachable here (not from VerificationConfig
    # validators), so this is the right layer for the translation.
    # The resulting store is strict so existing manual semantics are
    # preserved bit-identically.
    if config.replay_store is None:
        manual_model = next(
            (m for m in config.answering_models if getattr(m, "interface", None) == "manual"),
            None,
        )
        if manual_model is not None and getattr(manual_model, "manual_traces", None) is not None:
            from karenina.replay import ReplayStore

            config = config.model_copy(
                update={
                    "replay_store": ReplayStore.from_manual_traces(
                        manual_model.manual_traces,
                        benchmark=self,
                        miss_policy="strict",
                    ),
                }
            )

    if self.is_scenario_benchmark:
        return self._run_scenario_verification(
            config=config,
            run_name=run_name,
            async_enabled=async_enabled,
            progress_callback=progress_callback,
            sink=sink,
        )
    return self._verification_manager.run_verification(
        config,
        question_ids,
        run_name,
        async_enabled,
        progress_callback,
        workspace_root=self._workspace_root,
        sink=sink,
    )

save

save(
    path: Path, save_deep_judgment_config: bool = False
) -> None

Save the benchmark to a JSON-LD file.

Parameters:

Name	Type	Description	Default
path	Path	Path where to save the benchmark.	required
save_deep_judgment_config	bool	If True, include deep judgment configuration in LLM rubric traits. If False (default), deep judgment settings are stripped before saving.	False

Source code in src/karenina/benchmark/benchmark.py

def save(self, path: Path, save_deep_judgment_config: bool = False) -> None:
    """Save the benchmark to a JSON-LD file.

    Args:
        path: Path where to save the benchmark.
        save_deep_judgment_config: If True, include deep judgment
            configuration in LLM rubric traits. If False (default),
            deep judgment settings are stripped before saving.
    """
    self._base.save(path, save_deep_judgment_config=save_deep_judgment_config)

save_to_db

save_to_db(
    storage: str, checkpoint_path: Path | None = None
) -> Benchmark

Save this benchmark to a database.

Source code in src/karenina/benchmark/benchmark.py

def save_to_db(self, storage: str, checkpoint_path: Path | None = None) -> "Benchmark":
    """Save this benchmark to a database."""
    from typing import cast

    from ..storage import save_benchmark

    result = save_benchmark(self, storage, checkpoint_path)
    return cast("Benchmark", result)

search_questions

search_questions(
    query: str | list[str],
    match_all: bool = True,
    fields: list[str] | None = None,
    case_sensitive: bool = False,
    regex: bool = False,
) -> list[dict[str, Any]]

Search for questions containing the query text (unified search method).

Source code in src/karenina/benchmark/benchmark.py

def search_questions(
    self,
    query: str | list[str],
    match_all: bool = True,
    fields: list[str] | None = None,
    case_sensitive: bool = False,
    regex: bool = False,
) -> list[dict[str, Any]]:
    """Search for questions containing the query text (unified search method)."""
    return self._question_manager.search_questions(query, match_all, fields, case_sensitive, regex)

set_custom_property

set_custom_property(name: str, value: Any) -> None

Set a custom property in benchmark metadata.

Source code in src/karenina/benchmark/benchmark.py

def set_custom_property(self, name: str, value: Any) -> None:
    """Set a custom property in benchmark metadata."""
    self._metadata_manager.set_custom_property(name, value)

set_global_dynamic_rubric

set_global_dynamic_rubric(
    dynamic_rubric: DynamicRubric | None,
) -> None

Set or clear the global dynamic rubric.

Persists the rubric to the checkpoint so it survives save/load cycles.

Parameters:

Name	Type	Description	Default
dynamic_rubric	DynamicRubric | None	The DynamicRubric to set, or None to clear.	required

Source code in src/karenina/benchmark/benchmark.py

def set_global_dynamic_rubric(self, dynamic_rubric: DynamicRubric | None) -> None:
    """Set or clear the global dynamic rubric.

    Persists the rubric to the checkpoint so it survives save/load cycles.

    Args:
        dynamic_rubric: The DynamicRubric to set, or None to clear.
    """
    self._base._global_dynamic_rubric = dynamic_rubric
    if dynamic_rubric is not None:
        self._rubric_manager.set_global_dynamic_rubric_in_checkpoint(dynamic_rubric)
    else:
        # Clear from checkpoint: remove dynamic rubric ratings
        if self._base._checkpoint.rating:
            self._base._checkpoint.rating = [
                r for r in self._base._checkpoint.rating if r.additionalType != "karenina:GlobalDynamicRubricTrait"
            ]

set_global_rubric

set_global_rubric(rubric: Rubric) -> None

Set the complete global rubric (replaces existing).

Source code in src/karenina/benchmark/benchmark.py

def set_global_rubric(self, rubric: Rubric) -> None:
    """Set the complete global rubric (replaces existing)."""
    self._rubric_manager.set_global_rubric(rubric)

set_metadata

set_metadata(**metadata: Any) -> None

Set benchmark metadata.

Source code in src/karenina/benchmark/benchmark.py

def set_metadata(self, **metadata: Any) -> None:
    """Set benchmark metadata."""
    self._base.set_metadata(**metadata)

set_multiple_custom_properties

set_multiple_custom_properties(
    properties: dict[str, Any],
) -> None

Set multiple custom properties at once.

Source code in src/karenina/benchmark/benchmark.py

def set_multiple_custom_properties(self, properties: dict[str, Any]) -> None:
    """Set multiple custom properties at once."""
    self._metadata_manager.set_multiple_custom_properties(properties)

set_question_author

set_question_author(
    question_id: str, author: dict[str, Any] | None
) -> None

Set author information for a question.

Source code in src/karenina/benchmark/benchmark.py

def set_question_author(self, question_id: str, author: dict[str, Any] | None) -> None:
    """Set author information for a question."""
    self._question_manager.set_question_author(question_id, author)

set_question_custom_property

set_question_custom_property(
    question_id: str, name: str, value: Any
) -> None

Set a custom property on question metadata.

Source code in src/karenina/benchmark/benchmark.py

def set_question_custom_property(self, question_id: str, name: str, value: Any) -> None:
    """Set a custom property on question metadata."""
    self._question_manager.set_question_custom_property(question_id, name, value)

set_question_rubric

set_question_rubric(
    question_id: str, rubric: Rubric
) -> None

Set the complete question-specific rubric (replaces existing).

Source code in src/karenina/benchmark/benchmark.py

def set_question_rubric(self, question_id: str, rubric: Rubric) -> None:
    """Set the complete question-specific rubric (replaces existing)."""
    self._rubric_manager.set_question_rubric(question_id, rubric)

set_question_sources

set_question_sources(
    question_id: str, sources: list[dict[str, Any]] | None
) -> None

Set source documents for a question.

Source code in src/karenina/benchmark/benchmark.py

def set_question_sources(self, question_id: str, sources: list[dict[str, Any]] | None) -> None:
    """Set source documents for a question."""
    self._question_manager.set_question_sources(question_id, sources)

set_workspace_root

set_workspace_root(path: Path) -> None

Set the root directory for task workspaces.

Parameters:

Name	Type	Description	Default
path	Path	Directory containing task workspace subdirectories. Question workspace paths are resolved relative to this root.	required

Source code in src/karenina/benchmark/benchmark.py

def set_workspace_root(self, path: Path) -> None:
    """Set the root directory for task workspaces.

    Args:
        path: Directory containing task workspace subdirectories.
            Question workspace paths are resolved relative to this root.
    """
    self._workspace_root = path

store_verification_results

store_verification_results(
    results: VerificationResultSet
    | dict[str, VerificationResult],
    run_name: str | None = None,
) -> None

Store verification results in the benchmark metadata.

Source code in src/karenina/benchmark/benchmark.py

def store_verification_results(
    self,
    results: VerificationResultSet | dict[str, VerificationResult],
    run_name: str | None = None,
) -> None:
    """Store verification results in the benchmark metadata."""
    warnings.warn(
        "store_verification_results is deprecated. Use ResultsStore.add() instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    _helpers.store_verification_results(self, results, run_name)

to_csv

to_csv() -> str

Export questions as CSV format.

Source code in src/karenina/benchmark/benchmark.py

def to_csv(self) -> str:
    """Export questions as CSV format."""
    return self._export_manager.to_csv()

to_dict

to_dict() -> dict[str, Any]

Export benchmark as a plain dictionary.

Source code in src/karenina/benchmark/benchmark.py

def to_dict(self) -> dict[str, Any]:
    """Export benchmark as a plain dictionary."""
    return self._export_manager.to_dict()

to_markdown

to_markdown() -> str

Export benchmark as markdown document.

Source code in src/karenina/benchmark/benchmark.py

def to_markdown(self) -> str:
    """Export benchmark as markdown document."""
    return self._export_manager.to_markdown()

toggle_finished

toggle_finished(question_id: str) -> bool

Toggle finished status of a question.

Source code in src/karenina/benchmark/benchmark.py

def toggle_finished(self, question_id: str) -> bool:
    """Toggle finished status of a question."""
    return self._question_manager.toggle_finished(question_id)

update_question_metadata

update_question_metadata(
    question_id: str, **metadata: Any
) -> None

Update question metadata fields.

Source code in src/karenina/benchmark/benchmark.py

def update_question_metadata(self, question_id: str, **metadata: Any) -> None:
    """Update question metadata fields."""
    self._question_manager.update_question_metadata(question_id, **metadata)

update_template

update_template(
    question_id: str, template_code: str | type
) -> None

Update existing template.

Parameters:

Name	Type	Description	Default
question_id	str	The question ID	required
template_code	str | type	Python code defining the Answer class, or a BaseAnswer subclass	required

Source code in src/karenina/benchmark/benchmark.py

def update_template(self, question_id: str, template_code: str | type) -> None:
    """Update existing template.

    Args:
        question_id: The question ID
        template_code: Python code defining the Answer class, or a BaseAnswer subclass
    """
    self._template_manager.update_template(question_id, template_code)

validate

validate() -> tuple[bool, str]

Validate the benchmark structure and all templates.

Source code in src/karenina/benchmark/benchmark.py

def validate(self) -> tuple[bool, str]:
    """Validate the benchmark structure and all templates."""
    from .verification.utils.validation import validate_answer_template

    is_valid, error_msg = self._base.validate()
    if not is_valid:
        return False, error_msg

    for q_id, q_data in self._questions_cache.items():
        template_code = q_data.get("answer_template")
        if template_code is not None:
            is_valid, error_msg_or_none, _ = validate_answer_template(template_code)
            error_msg = error_msg_or_none or "Unknown validation error"
            if not is_valid:
                return False, f"Invalid template for {q_id}: {error_msg}"

    return True, "Benchmark is valid"

validate_rubrics

validate_rubrics() -> tuple[bool, list[dict[str, str]]]

Validate all rubrics are properly configured.

Source code in src/karenina/benchmark/benchmark.py

def validate_rubrics(self) -> tuple[bool, list[dict[str, str]]]:
    """Validate all rubrics are properly configured."""
    return self._rubric_manager.validate_rubrics()

validate_templates

validate_templates() -> tuple[bool, list[dict[str, str]]]

Validate all templates are valid Python code.

Source code in src/karenina/benchmark/benchmark.py

def validate_templates(self) -> tuple[bool, list[dict[str, str]]]:
    """Validate all templates are valid Python code."""
    return self._template_manager.validate_templates()

CurationWorkbookSummary dataclass

Summary returned after writing a benchmark curation workbook.

Source code in src/karenina/benchmark/curation_workbook.py

@dataclass(frozen=True)
class CurationWorkbookSummary:
    """Summary returned after writing a benchmark curation workbook."""

    output_path: Path
    question_count: int
    field_count: int
    draft_count: int
    finished_count: int
    template_parse_error_count: int

FinishedTemplate

Bases: BaseModel

Metadata for a finished answer template.

Source code in src/karenina/schemas/verification/api_models.py

class FinishedTemplate(BaseModel):
    """Metadata for a finished answer template."""

    model_config = ConfigDict(extra="forbid")

    question_id: str
    question_text: str
    question_preview: str  # Truncated version for UI
    raw_answer: str | None = None  # Ground truth answer from checkpoint
    template_code: str
    last_modified: str
    finished: bool = True
    question_rubric: dict[str, Any] | None = None  # Question-specific rubric as dict
    question_dynamic_rubric: dict[str, Any] | None = None  # Question-specific dynamic rubric as dict
    keywords: list[str] | None = None  # Keywords associated with the question
    few_shot_examples: list[dict[str, str]] | None = None  # Few-shot examples for this question
    workspace_path: str | None = None  # Relative workspace path from Question

MaskStats

Bases: NamedTuple

Masked trace plus counts describing what changed.

Source code in src/karenina/benchmark/verification/utils/trace_masking.py

class MaskStats(NamedTuple):
    """Masked trace plus counts describing what changed."""

    text: str
    messages_masked: int
    chars_removed: int

ModelConfig

Bases: BaseModel

Configuration for a single model.

Source code in src/karenina/schemas/config/models.py

class ModelConfig(BaseModel):
    """Configuration for a single model."""

    model_config = ConfigDict(extra="forbid")

    id: str | None = None  # Optional - defaults to "manual" for manual interface
    model_provider: str | None = None  # Optional - only required for langchain interface
    model_name: str | None = None  # Optional - defaults to "manual" for manual interface
    temperature: float = 0.1
    max_tokens: int = DEFAULT_MAX_TOKENS  # Maximum tokens for model response
    interface: str = "langchain"
    system_prompt: str | None = None  # Optional - defaults applied based on context (answering/parsing)
    max_retries: int = 2  # Optional max retries for template generation
    mcp_urls_dict: dict[str, str] | None = None  # Optional MCP server URLs
    mcp_tool_filter: list[str] | None = None  # Optional list of MCP tools to include
    mcp_tool_description_overrides: dict[str, str] | None = (
        None  # Optional tool description overrides for GEPA optimization
    )
    # HTTP timeout in seconds for MCP streamable HTTP connections.
    # None uses the MCP SDK default (30 seconds).
    mcp_http_timeout: float | None = None
    # SSE read timeout in seconds for MCP streamable HTTP connections.
    # Governs in-session tool-call reads. None uses the MCP SDK default (300 seconds).
    mcp_sse_read_timeout: float | None = None
    # OpenAI Endpoint configuration (for openai_endpoint interface)
    endpoint_base_url: str | None = None  # Custom endpoint base URL
    endpoint_api_key: SecretStr | None = None  # User-provided API key
    # Anthropic-specific configuration (for Anthropic models on supported interfaces)
    anthropic_base_url: str | None = None  # Custom Anthropic API endpoint (for proxies, self-hosted)
    anthropic_api_key: SecretStr | None = None  # Override ANTHROPIC_API_KEY env var
    # Typed filesystem, sandbox, or container settings for agent adapters.
    agent_runtime: AgentRuntimeConfig | None = None
    # Extra keyword arguments to pass to the underlying model interface.
    # Legacy exports may retain runtime settings under extra_kwargs["agent_runtime"].
    # Useful for passing vendor-specific API keys, custom parameters, etc.
    extra_kwargs: dict[str, Any] | None = None
    # Manual interface configuration
    manual_traces: Any = Field(default=None, exclude=True)  # Excluded from serialization; type: ManualTraces | None
    # Agent middleware configuration (only used when mcp_urls_dict is provided)
    # Controls retry behavior, execution limits, and summarization for MCP-enabled agents
    agent_middleware: AgentMiddlewareConfig | None = None
    # Token threshold for triggering summarization middleware.
    # When specified, summarization triggers at exactly this token count.
    # For langchain interface without this value, fraction-based triggering is used (auto-detected from model).
    # For openai_endpoint interface without this value, auto-detected from /v1/models API if available.
    # For openrouter interface without this value, defaults to 100000 * trigger_fraction.
    max_context_tokens: int | None = None
    # Timeout in seconds for agent execution. Overrides the default timeout (180s)
    # used in answer generation. Set higher for complex questions with many tool calls.
    agent_timeout: int | None = None
    # HTTP request timeout in seconds for individual LLM API calls.
    # Typically set by the pipeline from VerificationConfig.request_timeout.
    # None means use the provider SDK default (no timeout).
    request_timeout: float | None = None
    # Per-category retry policy for transient LLM errors.
    # None means use the pipeline-level RetryPolicy from VerificationConfig.
    # Typically stamped by the pipeline from VerificationConfig.retry_policy.
    retry_policy: RetryPolicy | None = None

    @model_validator(mode="after")
    def validate_manual_interface(self) -> "ModelConfig":
        """Validate manual interface configuration and set defaults."""
        if self.interface == INTERFACE_MANUAL:
            # Reject bool values: True/False are not ManualTraces instances
            if isinstance(self.manual_traces, bool):
                raise ValueError(
                    "manual_traces must be a ManualTraces instance, not a bool. "
                    "Create a ManualTraces instance and pass it to ModelConfig."
                )
            # Manual interface requires manual_traces
            if self.manual_traces is None:
                raise ValueError(
                    "manual_traces is required when interface='manual'. "
                    "Create a ManualTraces instance and pass it to ModelConfig."
                )

            # Set defaults for manual interface
            if self.id is None:
                self.id = "manual"
            if self.model_name is None:
                self.model_name = "manual"

            # MCP not supported with manual interface
            if self.mcp_urls_dict is not None:
                raise ValueError(
                    "MCP tools are not supported with manual interface. "
                    "Manual traces are precomputed and cannot use dynamic tools."
                )
        else:
            # Non-manual interfaces require id and model_name
            if self.id is None:
                raise ValueError("id is required for non-manual interfaces")
            if self.model_name is None:
                raise ValueError("model_name is required for non-manual interfaces")

        return self

    @model_validator(mode="after")
    def validate_anthropic_connection_fields(self) -> "ModelConfig":
        """Prevent Anthropic credentials from being silently ignored.

        The dedicated fields are consumed by the Claude adapters and by
        Anthropic-backed LangChain adapters. Other interfaces must reject them
        so a configured credential can never fall back to the environment at a
        different host.
        """
        if self.anthropic_base_url is None and self.anthropic_api_key is None:
            return self

        supported_interfaces = {
            "claude_agent_sdk",
            "claude_tool",
            "langchain",
            "langchain_deep_agents",
        }
        if self.interface not in supported_interfaces:
            raise ValueError(
                "anthropic_base_url and anthropic_api_key are only supported for "
                f"interfaces {sorted(supported_interfaces)}, got {self.interface!r}"
            )

        if self.interface in {"langchain", "langchain_deep_agents"} and self.model_provider != "anthropic":
            raise ValueError(
                "anthropic_base_url and anthropic_api_key require model_provider='anthropic' "
                f"for interface={self.interface!r}"
            )

        extra_kwargs = self.extra_kwargs or {}
        conflicting_fields = [
            field
            for field, value in (("base_url", self.anthropic_base_url), ("api_key", self.anthropic_api_key))
            if value is not None and extra_kwargs.get(field) is not None
        ]
        if conflicting_fields:
            raise ValueError(
                "anthropic_base_url/anthropic_api_key conflict with extra_kwargs "
                f"values for {conflicting_fields}. Configure each connection setting only once."
            )

        return self

    @model_validator(mode="after")
    def validate_interface_registered(self) -> "ModelConfig":
        """Validate that the interface is registered in AdapterRegistry.

        Skips validation while the registry is initializing to avoid
        re-entrant initialization when registration modules create ModelConfig
        instances during _load_builtins().
        """
        from karenina.adapters.registry import AdapterRegistry

        # During initialization, registration modules may create ModelConfig
        # instances (e.g., in tests or default configs). Skip validation to
        # avoid re-entrant calls into _ensure_initialized() via the RLock.
        if AdapterRegistry._initializing:
            return self

        if AdapterRegistry.get_spec(self.interface) is None:
            registered = AdapterRegistry.get_interfaces()
            raise ValueError(f"Unknown interface '{self.interface}'. Registered interfaces: {sorted(registered)}")
        return self

Functions

validate_anthropic_connection_fields

validate_anthropic_connection_fields() -> ModelConfig

Prevent Anthropic credentials from being silently ignored.

The dedicated fields are consumed by the Claude adapters and by Anthropic-backed LangChain adapters. Other interfaces must reject them so a configured credential can never fall back to the environment at a different host.

Source code in src/karenina/schemas/config/models.py

@model_validator(mode="after")
def validate_anthropic_connection_fields(self) -> "ModelConfig":
    """Prevent Anthropic credentials from being silently ignored.

    The dedicated fields are consumed by the Claude adapters and by
    Anthropic-backed LangChain adapters. Other interfaces must reject them
    so a configured credential can never fall back to the environment at a
    different host.
    """
    if self.anthropic_base_url is None and self.anthropic_api_key is None:
        return self

    supported_interfaces = {
        "claude_agent_sdk",
        "claude_tool",
        "langchain",
        "langchain_deep_agents",
    }
    if self.interface not in supported_interfaces:
        raise ValueError(
            "anthropic_base_url and anthropic_api_key are only supported for "
            f"interfaces {sorted(supported_interfaces)}, got {self.interface!r}"
        )

    if self.interface in {"langchain", "langchain_deep_agents"} and self.model_provider != "anthropic":
        raise ValueError(
            "anthropic_base_url and anthropic_api_key require model_provider='anthropic' "
            f"for interface={self.interface!r}"
        )

    extra_kwargs = self.extra_kwargs or {}
    conflicting_fields = [
        field
        for field, value in (("base_url", self.anthropic_base_url), ("api_key", self.anthropic_api_key))
        if value is not None and extra_kwargs.get(field) is not None
    ]
    if conflicting_fields:
        raise ValueError(
            "anthropic_base_url/anthropic_api_key conflict with extra_kwargs "
            f"values for {conflicting_fields}. Configure each connection setting only once."
        )

    return self

validate_interface_registered

validate_interface_registered() -> ModelConfig

Validate that the interface is registered in AdapterRegistry.

Skips validation while the registry is initializing to avoid re-entrant initialization when registration modules create ModelConfig instances during _load_builtins().

Source code in src/karenina/schemas/config/models.py

@model_validator(mode="after")
def validate_interface_registered(self) -> "ModelConfig":
    """Validate that the interface is registered in AdapterRegistry.

    Skips validation while the registry is initializing to avoid
    re-entrant initialization when registration modules create ModelConfig
    instances during _load_builtins().
    """
    from karenina.adapters.registry import AdapterRegistry

    # During initialization, registration modules may create ModelConfig
    # instances (e.g., in tests or default configs). Skip validation to
    # avoid re-entrant calls into _ensure_initialized() via the RLock.
    if AdapterRegistry._initializing:
        return self

    if AdapterRegistry.get_spec(self.interface) is None:
        registered = AdapterRegistry.get_interfaces()
        raise ValueError(f"Unknown interface '{self.interface}'. Registered interfaces: {sorted(registered)}")
    return self

validate_manual_interface

validate_manual_interface() -> ModelConfig

Validate manual interface configuration and set defaults.

Source code in src/karenina/schemas/config/models.py

@model_validator(mode="after")
def validate_manual_interface(self) -> "ModelConfig":
    """Validate manual interface configuration and set defaults."""
    if self.interface == INTERFACE_MANUAL:
        # Reject bool values: True/False are not ManualTraces instances
        if isinstance(self.manual_traces, bool):
            raise ValueError(
                "manual_traces must be a ManualTraces instance, not a bool. "
                "Create a ManualTraces instance and pass it to ModelConfig."
            )
        # Manual interface requires manual_traces
        if self.manual_traces is None:
            raise ValueError(
                "manual_traces is required when interface='manual'. "
                "Create a ManualTraces instance and pass it to ModelConfig."
            )

        # Set defaults for manual interface
        if self.id is None:
            self.id = "manual"
        if self.model_name is None:
            self.model_name = "manual"

        # MCP not supported with manual interface
        if self.mcp_urls_dict is not None:
            raise ValueError(
                "MCP tools are not supported with manual interface. "
                "Manual traces are precomputed and cannot use dynamic tools."
            )
    else:
        # Non-manual interfaces require id and model_name
        if self.id is None:
            raise ValueError("id is required for non-manual interfaces")
        if self.model_name is None:
            raise ValueError("model_name is required for non-manual interfaces")

    return self

PostHocJudgment dataclass

Rubric scores for one stored row or parser-collapsed trace.

Source code in src/karenina/benchmark/verification/post_hoc.py

@dataclass(frozen=True, slots=True)
class PostHocJudgment:
    """Rubric scores for one stored row or parser-collapsed trace."""

    key: ResultRowKey
    sibling_keys: tuple[ResultRowKey, ...] = ()
    scores: dict[str, Any] = field(default_factory=dict)
    labels: dict[str, str] = field(default_factory=dict)
    error: str | None = None
    representative_result_id: str | None = None
    sibling_result_ids: tuple[str, ...] = ()

ProgressiveFileSink

Append-only JSONL sidecar + .state manifest for crash recovery.

On disk during a run (paths relative to output_path):

• <output>.state: JSON manifest. Contains the task manifest, the completed / failed triple sets, a config snapshot, and a config hash. Rewritten atomically on every completion.
• <output>.results.jsonl: one JSON-serialized :class:VerificationResult per line, appended as tasks complete.

On successful completion (:meth:on_finalize with all_complete=True):

• The JSONL sidecar is reassembled into the final frontend-export JSON at output_path via :func:export_verification_results_json_stream.
• The .state and .results.jsonl files are deleted.

On partial completion (:meth:on_finalize with all_complete=False):

• Both sidecars are kept so --resume can continue.

Resume path: :meth:load_for_resume reads the state and JSONL and returns a sink populated with the already-completed triples.

Source code in src/karenina/benchmark/verification/sinks.py

class ProgressiveFileSink:
    """Append-only JSONL sidecar + ``.state`` manifest for crash recovery.

    On disk during a run (paths relative to ``output_path``):

    - ``<output>.state``: JSON manifest. Contains the task manifest, the
      completed / failed triple sets, a config snapshot, and a config hash.
      Rewritten atomically on every completion.
    - ``<output>.results.jsonl``: one JSON-serialized :class:`VerificationResult`
      per line, appended as tasks complete.

    On successful completion (:meth:`on_finalize` with ``all_complete=True``):

    - The JSONL sidecar is reassembled into the final frontend-export JSON
      at ``output_path`` via :func:`export_verification_results_json_stream`.
    - The ``.state`` and ``.results.jsonl`` files are deleted.

    On partial completion (:meth:`on_finalize` with ``all_complete=False``):

    - Both sidecars are kept so ``--resume`` can continue.

    Resume path: :meth:`load_for_resume` reads the state and JSONL and
    returns a sink populated with the already-completed triples.
    """

    def __init__(
        self,
        output_path: Path,
        config: VerificationConfig,
        benchmark_path: str,
        global_rubric: Rubric | None = None,
    ) -> None:
        """Initialize a fresh sink targeting ``output_path``.

        Args:
            output_path: Final export target (e.g. ``results.json``). The
                sidecars live next to it at ``results.json.state`` and
                ``results.json.results.jsonl``.
            config: Verification configuration to snapshot into the state.
            benchmark_path: Path of the benchmark file, stored in the state
                for reference and diagnostics.
            global_rubric: Optional rubric folded into the final export.
        """
        self.output_path = output_path
        self.state_path = output_path.with_suffix(output_path.suffix + ".state")
        self.jsonl_path = output_path.with_suffix(output_path.suffix + ".results.jsonl")
        self.config = config
        self.benchmark_path = benchmark_path
        self.global_rubric = global_rubric

        self._manifest: list[str] = []
        self._completed: set[str] = set()
        self._failed: set[str] = set()
        self._start_time: float | None = None
        self._config_hash: str = ""
        # In-memory mirror of JSONL content, built from appends and from
        # load_for_resume so finalize() can reassemble without re-reading.
        self._results: list[VerificationResult] = []
        # Internal sink lock guarding the JSONL append, the state rewrite,
        # and the completed/failed bookkeeping. Reentrant so subclass
        # overrides (AgenticProgressiveFileSink) can hold it around a
        # super() call. Innermost lock: never call executor code while held.
        self._lock = threading.RLock()

    # -- Resume ------------------------------------------------------------

    @classmethod
    def load_for_resume(
        cls,
        state_path: Path,
        *,
        global_rubric: Rubric | None = None,
        config_updater: Callable[[VerificationConfig], VerificationConfig] | None = None,
        manual_traces: Any | None = None,
    ) -> ProgressiveFileSink:
        """Reconstruct a sink from an existing ``.state`` + JSONL pair.

        Args:
            state_path: Path to the ``.state`` file on disk.
            global_rubric: Rubric to attach to the sink for final export.
            config_updater: Optional transform applied to the rehydrated
                stored config, for example to re-inject secrets that
                pydantic masked when serializing the state file.
            manual_traces: Optional :class:`ManualTraces` re-attached to
                stored answering models with ``interface='manual'`` before
                config revalidation. The state file excludes
                ``manual_traces`` (not serializable), so manual-interface
                runs must re-supply them here; without them the config
                still fails with the ModelConfig validation error.

        Returns:
            A ``ProgressiveFileSink`` whose :meth:`completed_triples` returns
            the set of triples already persisted, and whose internal
            ``_results`` buffer is pre-populated.

        Raises:
            FileNotFoundError: If ``state_path`` does not exist.
            ValueError: If the state format version is incompatible.
        """
        if not state_path.exists():
            raise FileNotFoundError(f"State file not found: {state_path}")

        with open(state_path) as f:
            state_data = json.load(f)

        fmt = state_data.get("format_version")
        if fmt != STATE_FORMAT_VERSION:
            raise ValueError(f"Incompatible state format version: {fmt} (expected {STATE_FORMAT_VERSION})")

        output_path = Path(state_data["output_path"])
        config_data = state_data["config"]
        if manual_traces is not None:
            for model in config_data.get("answering_models", []):
                if model.get("interface") == "manual":
                    model["manual_traces"] = manual_traces
        config = VerificationConfig(**config_data)
        if config_updater is not None:
            config = config_updater(config)
        if _has_masked_secrets(config):
            logger.warning(
                "Resumed sink config contains masked secrets from the state file. "
                "Pass a fresh config to run_verification or supply config_updater "
                "to re-inject credentials."
            )
        benchmark_path = state_data["benchmark_path"]

        sink = cls(output_path, config, benchmark_path, global_rubric=global_rubric)
        sink._manifest = list(state_data.get("task_manifest", []))
        sink._completed = set(state_data.get("completed_task_ids", []))
        sink._failed = set(state_data.get("failed_task_ids", []))
        sink._start_time = state_data.get("start_time")
        sink._config_hash = state_data.get("config_hash", "")

        if sink.jsonl_path.exists():
            sink._results = list(_read_jsonl_results(sink.jsonl_path))
            # Cross-check: _completed should match the JSONL contents. If the
            # JSONL has more entries than the state tracked (e.g. a crash
            # right after append, before state rewrite), trust the JSONL.
            for result in sink._results:
                sink._completed.add(_scenario_manifest_key(result, sink._manifest))

        logger.info(
            "Loaded progressive sink: %d/%d tasks completed (%d failed)",
            len(sink._completed),
            len(sink._manifest),
            len(sink._failed),
        )
        return sink

    @classmethod
    def open_or_resume(
        cls,
        output_path: Path,
        config: VerificationConfig,
        benchmark_path: str,
        *,
        global_rubric: Rubric | None = None,
        config_updater: Callable[[VerificationConfig], VerificationConfig] | None = None,
    ) -> ProgressiveFileSink:
        """Resume from an existing state sidecar or create a fresh sink.

        Resumes when the ``.state`` sidecar for ``output_path`` exists,
        otherwise creates parent directories and a fresh sink. On resume,
        ``config`` is not used for the sink itself (the stored config is
        rehydrated), but callers that rebuild their config each run should
        keep passing the fresh config to ``run_verification``, which
        overwrites the sink's copy on start.

        Args:
            output_path: Destination results file.
            config: Config for a fresh run.
            benchmark_path: Benchmark checkpoint path recorded in state.
            global_rubric: Optional global rubric to attach.
            config_updater: Optional transform applied to the rehydrated
                stored config on resume, for example to re-inject secrets.

        Returns:
            A fresh or resumed sink.
        """
        state_path = output_path.with_suffix(output_path.suffix + ".state")
        if state_path.exists():
            logger.info("Resuming progressive save from %s", state_path)
            return cls.load_for_resume(state_path, global_rubric=global_rubric, config_updater=config_updater)
        output_path.parent.mkdir(parents=True, exist_ok=True)
        return cls(output_path, config, benchmark_path, global_rubric=global_rubric)

    # -- Seeding ----------------------------------------------------------

    def seed_prior_results(self, prior_results: VerificationResultSet) -> None:
        """Pre-populate JSONL, ``_results``, ``_completed``, ``_manifest``.

        Called by the ``extend_*`` facades before ``run_verification`` so
        the sink's final export covers prior rows in addition to the new
        rows produced by the pipeline. Idempotent: rows whose triple is
        already tracked are skipped. Does not require :meth:`on_start` to
        have run first; the JSONL file is created on demand.
        """
        with self._lock:
            added = 0
            manifest_set = set(self._manifest)
            for row in prior_results.results:
                if not is_completed_result(row):
                    continue
                task_id = _scenario_manifest_key(row, self._manifest)
                if task_id in self._completed:
                    continue
                _append_jsonl_line(self.jsonl_path, row.model_dump_json())
                self._results.append(row)
                self._completed.add(task_id)
                self._failed.discard(task_id)
                if task_id not in manifest_set:
                    self._manifest.append(task_id)
                    manifest_set.add(task_id)
                added += 1
            if added:
                self._save_state()
                logger.info("ProgressiveFileSink seeded with %d prior rows", added)

    # -- ResultSink protocol ----------------------------------------------

    def completed_triples(self) -> set[TripleKey]:
        """Return already-persisted triples for ``config.skip_triples``."""
        with self._lock:
            completed_snapshot = set(self._completed)
        out: set[TripleKey] = set()
        for key in completed_snapshot:
            try:
                ident = TaskIdentifier.from_key(key)
            except ValueError:
                logger.warning("Skipping malformed task key in sink: %r", key)
                continue
            out.add(
                (
                    ident.question_id,
                    ident.answering_canonical_key,
                    ident.parsing_canonical_key,
                    ident.replicate,
                )
            )
        return out

    def on_start(self, manifest: list[str], config: VerificationConfig) -> None:
        """Record the manifest and write the initial state file.

        Merge semantics: the stored manifest is unioned with the provided
        one, preserving insertion order. This handles both the resume path
        (provided manifest is a subset of the stored full manifest) and
        the seeded-extend path (stored holds prior keys from
        :meth:`seed_prior_results`, provided holds the new-task keys
        from :func:`generate_task_queue`). Repeated keys are dropped.
        """
        with self._lock:
            if not self._manifest:
                self._manifest = list(manifest)
            else:
                existing = set(self._manifest)
                for key in manifest:
                    if key not in existing:
                        self._manifest.append(key)
                        existing.add(key)

            # Always refresh config + hash: a resume may pass a config that
            # was rehydrated from the state, but we keep whatever the caller
            # handed us so later callers get the current object identity.
            self.config = config
            if not self._config_hash:
                self._config_hash = _compute_config_hash(config)
            if self._start_time is None:
                self._start_time = time.time()

            self._save_state()
            # Touch the JSONL so readers can rely on its existence.
            if not self.jsonl_path.exists():
                self.jsonl_path.touch()

    def on_result(self, result: VerificationResult) -> None:
        """Append a completed result to the JSONL and refresh the state.

        The JSONL line is always written. The ``_completed`` set is
        idempotent: scenario runs emit one ``on_result`` per turn, all
        sharing the same combo-level task key (scenario_id-keyed), and the
        combo is only marked complete once even though N turn lines are
        appended. QA runs emit exactly one result per task so the guard
        is a no-op.
        """
        if not is_completed_result(result):
            return

        with self._lock:
            task_id = _scenario_manifest_key(result, self._manifest)
            _append_jsonl_line(self.jsonl_path, result.model_dump_json())
            self._results.append(result)
            if task_id not in self._completed:
                self._completed.add(task_id)
                self._failed.discard(task_id)
            self._save_state()

    def mark_failed(self, task_id: str) -> None:
        """Record a task that was attempted but did not produce a result.

        Optional helper for the batch runner so partial-failure state can
        be preserved across resumes. Not part of the :class:`ResultSink`
        protocol.
        """
        with self._lock:
            if task_id in self._completed:
                return
            self._failed.add(task_id)
            self._save_state()

    def on_finalize(self, *, all_complete: bool) -> None:
        """Handle terminal state.

        On ``all_complete=True``, write the final frontend-export JSON at
        :attr:`output_path` and remove the sidecars. On partial completion,
        keep the sidecars intact so ``resume`` can continue.
        """
        with self._lock:
            if all_complete:
                self.write_final_export()
                self._delete_sidecars()
                logger.info("ProgressiveFileSink finalized: full export at %s", self.output_path)
            else:
                # Preserve sidecars so a later `--resume` can continue.
                # Refresh the state one last time so the last batch's
                # activity lands.
                self._save_state()
                logger.info(
                    "ProgressiveFileSink left state in place for resume: %d/%d complete",
                    len(self._completed),
                    len(self._manifest),
                )

    # -- Accessors used by CLI / tests ------------------------------------

    @property
    def completed_count(self) -> int:
        """Number of completed tasks."""
        with self._lock:
            return len(self._completed)

    @property
    def total_tasks(self) -> int:
        """Total tasks in the manifest."""
        with self._lock:
            return len(self._manifest)

    @property
    def task_manifest(self) -> list[str]:
        """Task manifest keys recorded for this run."""
        with self._lock:
            return list(self._manifest)

    def get_all_results(self) -> list[VerificationResult]:
        """Return a shallow copy of buffered completed results."""
        with self._lock:
            return list(self._results)

    def iter_results(self) -> Iterable[VerificationResult]:
        """Iterate over a snapshot of buffered completed results.

        Used by the batch runner to hydrate the workspace
        :class:`AnswerTraceCache` on resume so new parser variants for
        already-completed answerer triples reuse the prior trace instead
        of regenerating it. The snapshot is taken under the sink lock so
        the iterator stays valid even if results land concurrently.
        """
        with self._lock:
            return iter(list(self._results))

    def get_result_set(self) -> VerificationResultSet:
        """Return buffered results as a :class:`VerificationResultSet`."""
        with self._lock:
            return VerificationResultSet(results=list(self._results))

    def set_global_rubric(self, rubric: Rubric | None) -> None:
        """Attach a global rubric used by the final export."""
        self.global_rubric = rubric

    # -- Internals --------------------------------------------------------

    def _save_state(self) -> None:
        """Atomically rewrite the ``.state`` file."""
        state = {
            "format_version": STATE_FORMAT_VERSION,
            "created_at": _fmt_utc(self._start_time or time.time()),
            "last_updated_at": _fmt_utc(time.time()),
            "benchmark_path": self.benchmark_path,
            "output_path": str(self.output_path),
            "config_hash": self._config_hash,
            "config": self.config.model_dump(mode="json", exclude={"manual_traces": True}),
            "task_manifest": self._manifest,
            "completed_task_ids": sorted(self._completed),
            "failed_task_ids": sorted(self._failed),
            "total_tasks": len(self._manifest),
            "completed_count": len(self._completed),
            "failed_count": len(self._failed),
            "start_time": self._start_time,
        }
        atomic_write(self.state_path, json.dumps(state, indent=2))

    def write_final_export(self, output_path: Path | None = None) -> Path:
        """Assemble and atomically write the frontend-export JSON.

        Args:
            output_path: Destination. Defaults to :attr:`output_path`.

        Returns:
            The path the export was written to.
        """
        with self._lock:
            target = output_path or self.output_path
            now = time.time()
            results = list(_read_jsonl_results(self.jsonl_path)) if self.jsonl_path.exists() else []
            latest_rows = _dedupe_latest_by_manifest_key(results, manifest=self._manifest)
            successful_count, failed_count = _count_success_failure(latest_rows)
            job = VerificationJob(
                job_id=f"progressive-{int(self._start_time or now)}",
                run_name="progressive",
                status="completed",
                config=self.config,
                total_questions=self.total_tasks,
                successful_count=successful_count,
                failed_count=failed_count,
                start_time=self._start_time,
                end_time=now,
            )
            # Stream from the JSONL sidecar if it exists. write_final_export
            # may be called before any on_result (e.g. the empty-run case),
            # in which case jsonl_path was never created; guard is mandatory
            # because _read_jsonl_results opens the path unconditionally.
            scenario_results = _reconstruct_scenario_results_for_export(results, manifest=self._manifest)
            export_verification_results_json_stream(
                job,
                iter(results),
                self.global_rubric,
                is_complete=True,
                scenario_results=scenario_results or None,
                out_path=target,
            )
            return target

    def _delete_sidecars(self) -> None:
        """Remove the ``.state`` and ``.results.jsonl`` files."""
        for path in (self.state_path, self.jsonl_path):
            try:
                if path.exists():
                    path.unlink()
            except OSError as e:
                logger.warning("Failed to remove %s: %s", path, e)

Attributes

completed_count property

completed_count: int

Number of completed tasks.

task_manifest property

task_manifest: list[str]

Task manifest keys recorded for this run.

total_tasks property

total_tasks: int

Total tasks in the manifest.

Functions

__init__

__init__(
    output_path: Path,
    config: VerificationConfig,
    benchmark_path: str,
    global_rubric: Rubric | None = None,
) -> None

Parameters:

Name	Type	Description	Default
output_path	Path	Final export target (e.g. results.json). The sidecars live next to it at results.json.state and results.json.results.jsonl.	required
config	VerificationConfig	Verification configuration to snapshot into the state.	required
benchmark_path	str	Path of the benchmark file, stored in the state for reference and diagnostics.	required
global_rubric	Rubric | None	Optional rubric folded into the final export.	None

Source code in src/karenina/benchmark/verification/sinks.py

def __init__(
    self,
    output_path: Path,
    config: VerificationConfig,
    benchmark_path: str,
    global_rubric: Rubric | None = None,
) -> None:
    """Initialize a fresh sink targeting ``output_path``.

    Args:
        output_path: Final export target (e.g. ``results.json``). The
            sidecars live next to it at ``results.json.state`` and
            ``results.json.results.jsonl``.
        config: Verification configuration to snapshot into the state.
        benchmark_path: Path of the benchmark file, stored in the state
            for reference and diagnostics.
        global_rubric: Optional rubric folded into the final export.
    """
    self.output_path = output_path
    self.state_path = output_path.with_suffix(output_path.suffix + ".state")
    self.jsonl_path = output_path.with_suffix(output_path.suffix + ".results.jsonl")
    self.config = config
    self.benchmark_path = benchmark_path
    self.global_rubric = global_rubric

    self._manifest: list[str] = []
    self._completed: set[str] = set()
    self._failed: set[str] = set()
    self._start_time: float | None = None
    self._config_hash: str = ""
    # In-memory mirror of JSONL content, built from appends and from
    # load_for_resume so finalize() can reassemble without re-reading.
    self._results: list[VerificationResult] = []
    # Internal sink lock guarding the JSONL append, the state rewrite,
    # and the completed/failed bookkeeping. Reentrant so subclass
    # overrides (AgenticProgressiveFileSink) can hold it around a
    # super() call. Innermost lock: never call executor code while held.
    self._lock = threading.RLock()

completed_triples

completed_triples() -> set[TripleKey]

Return already-persisted triples for config.skip_triples.

Source code in src/karenina/benchmark/verification/sinks.py

def completed_triples(self) -> set[TripleKey]:
    """Return already-persisted triples for ``config.skip_triples``."""
    with self._lock:
        completed_snapshot = set(self._completed)
    out: set[TripleKey] = set()
    for key in completed_snapshot:
        try:
            ident = TaskIdentifier.from_key(key)
        except ValueError:
            logger.warning("Skipping malformed task key in sink: %r", key)
            continue
        out.add(
            (
                ident.question_id,
                ident.answering_canonical_key,
                ident.parsing_canonical_key,
                ident.replicate,
            )
        )
    return out

get_all_results

get_all_results() -> list[VerificationResult]

Return a shallow copy of buffered completed results.

Source code in src/karenina/benchmark/verification/sinks.py

def get_all_results(self) -> list[VerificationResult]:
    """Return a shallow copy of buffered completed results."""
    with self._lock:
        return list(self._results)

get_result_set

get_result_set() -> VerificationResultSet

Return buffered results as a :class:VerificationResultSet.

Source code in src/karenina/benchmark/verification/sinks.py

def get_result_set(self) -> VerificationResultSet:
    """Return buffered results as a :class:`VerificationResultSet`."""
    with self._lock:
        return VerificationResultSet(results=list(self._results))

iter_results

iter_results() -> Iterable[VerificationResult]

Iterate over a snapshot of buffered completed results.

Used by the batch runner to hydrate the workspace :class:AnswerTraceCache on resume so new parser variants for already-completed answerer triples reuse the prior trace instead of regenerating it. The snapshot is taken under the sink lock so the iterator stays valid even if results land concurrently.

Source code in src/karenina/benchmark/verification/sinks.py

def iter_results(self) -> Iterable[VerificationResult]:
    """Iterate over a snapshot of buffered completed results.

    Used by the batch runner to hydrate the workspace
    :class:`AnswerTraceCache` on resume so new parser variants for
    already-completed answerer triples reuse the prior trace instead
    of regenerating it. The snapshot is taken under the sink lock so
    the iterator stays valid even if results land concurrently.
    """
    with self._lock:
        return iter(list(self._results))

load_for_resume classmethod

load_for_resume(
    state_path: Path,
    *,
    global_rubric: Rubric | None = None,
    config_updater: Callable[
        [VerificationConfig], VerificationConfig
    ]
    | None = None,
    manual_traces: Any | None = None,
) -> ProgressiveFileSink

Reconstruct a sink from an existing .state + JSONL pair.

Parameters:

Name	Type	Description	Default
state_path	Path	Path to the .state file on disk.	required
global_rubric	Rubric | None	Rubric to attach to the sink for final export.	None
config_updater	Callable[[VerificationConfig], VerificationConfig] | None	Optional transform applied to the rehydrated stored config, for example to re-inject secrets that pydantic masked when serializing the state file.	None
manual_traces	Any | None	Optional :class:ManualTraces re-attached to stored answering models with interface='manual' before config revalidation. The state file excludes manual_traces (not serializable), so manual-interface runs must re-supply them here; without them the config still fails with the ModelConfig validation error.	None

Returns:

Type	Description
ProgressiveFileSink	A ProgressiveFileSink whose :meth:completed_triples returns
ProgressiveFileSink	the set of triples already persisted, and whose internal
ProgressiveFileSink	_results buffer is pre-populated.

Raises:

Type	Description
FileNotFoundError	If state_path does not exist.
ValueError	If the state format version is incompatible.

Source code in src/karenina/benchmark/verification/sinks.py

@classmethod
def load_for_resume(
    cls,
    state_path: Path,
    *,
    global_rubric: Rubric | None = None,
    config_updater: Callable[[VerificationConfig], VerificationConfig] | None = None,
    manual_traces: Any | None = None,
) -> ProgressiveFileSink:
    """Reconstruct a sink from an existing ``.state`` + JSONL pair.

    Args:
        state_path: Path to the ``.state`` file on disk.
        global_rubric: Rubric to attach to the sink for final export.
        config_updater: Optional transform applied to the rehydrated
            stored config, for example to re-inject secrets that
            pydantic masked when serializing the state file.
        manual_traces: Optional :class:`ManualTraces` re-attached to
            stored answering models with ``interface='manual'`` before
            config revalidation. The state file excludes
            ``manual_traces`` (not serializable), so manual-interface
            runs must re-supply them here; without them the config
            still fails with the ModelConfig validation error.

    Returns:
        A ``ProgressiveFileSink`` whose :meth:`completed_triples` returns
        the set of triples already persisted, and whose internal
        ``_results`` buffer is pre-populated.

    Raises:
        FileNotFoundError: If ``state_path`` does not exist.
        ValueError: If the state format version is incompatible.
    """
    if not state_path.exists():
        raise FileNotFoundError(f"State file not found: {state_path}")

    with open(state_path) as f:
        state_data = json.load(f)

    fmt = state_data.get("format_version")
    if fmt != STATE_FORMAT_VERSION:
        raise ValueError(f"Incompatible state format version: {fmt} (expected {STATE_FORMAT_VERSION})")

    output_path = Path(state_data["output_path"])
    config_data = state_data["config"]
    if manual_traces is not None:
        for model in config_data.get("answering_models", []):
            if model.get("interface") == "manual":
                model["manual_traces"] = manual_traces
    config = VerificationConfig(**config_data)
    if config_updater is not None:
        config = config_updater(config)
    if _has_masked_secrets(config):
        logger.warning(
            "Resumed sink config contains masked secrets from the state file. "
            "Pass a fresh config to run_verification or supply config_updater "
            "to re-inject credentials."
        )
    benchmark_path = state_data["benchmark_path"]

    sink = cls(output_path, config, benchmark_path, global_rubric=global_rubric)
    sink._manifest = list(state_data.get("task_manifest", []))
    sink._completed = set(state_data.get("completed_task_ids", []))
    sink._failed = set(state_data.get("failed_task_ids", []))
    sink._start_time = state_data.get("start_time")
    sink._config_hash = state_data.get("config_hash", "")

    if sink.jsonl_path.exists():
        sink._results = list(_read_jsonl_results(sink.jsonl_path))
        # Cross-check: _completed should match the JSONL contents. If the
        # JSONL has more entries than the state tracked (e.g. a crash
        # right after append, before state rewrite), trust the JSONL.
        for result in sink._results:
            sink._completed.add(_scenario_manifest_key(result, sink._manifest))

    logger.info(
        "Loaded progressive sink: %d/%d tasks completed (%d failed)",
        len(sink._completed),
        len(sink._manifest),
        len(sink._failed),
    )
    return sink

mark_failed

mark_failed(task_id: str) -> None

Record a task that was attempted but did not produce a result.

Optional helper for the batch runner so partial-failure state can be preserved across resumes. Not part of the :class:ResultSink protocol.

Source code in src/karenina/benchmark/verification/sinks.py

def mark_failed(self, task_id: str) -> None:
    """Record a task that was attempted but did not produce a result.

    Optional helper for the batch runner so partial-failure state can
    be preserved across resumes. Not part of the :class:`ResultSink`
    protocol.
    """
    with self._lock:
        if task_id in self._completed:
            return
        self._failed.add(task_id)
        self._save_state()

on_finalize

on_finalize(*, all_complete: bool) -> None

Handle terminal state.

On all_complete=True, write the final frontend-export JSON at :attr:output_path and remove the sidecars. On partial completion, keep the sidecars intact so resume can continue.

Source code in src/karenina/benchmark/verification/sinks.py

def on_finalize(self, *, all_complete: bool) -> None:
    """Handle terminal state.

    On ``all_complete=True``, write the final frontend-export JSON at
    :attr:`output_path` and remove the sidecars. On partial completion,
    keep the sidecars intact so ``resume`` can continue.
    """
    with self._lock:
        if all_complete:
            self.write_final_export()
            self._delete_sidecars()
            logger.info("ProgressiveFileSink finalized: full export at %s", self.output_path)
        else:
            # Preserve sidecars so a later `--resume` can continue.
            # Refresh the state one last time so the last batch's
            # activity lands.
            self._save_state()
            logger.info(
                "ProgressiveFileSink left state in place for resume: %d/%d complete",
                len(self._completed),
                len(self._manifest),
            )

on_result

on_result(result: VerificationResult) -> None

Append a completed result to the JSONL and refresh the state.

The JSONL line is always written. The _completed set is idempotent: scenario runs emit one on_result per turn, all sharing the same combo-level task key (scenario_id-keyed), and the combo is only marked complete once even though N turn lines are appended. QA runs emit exactly one result per task so the guard is a no-op.

Source code in src/karenina/benchmark/verification/sinks.py

def on_result(self, result: VerificationResult) -> None:
    """Append a completed result to the JSONL and refresh the state.

    The JSONL line is always written. The ``_completed`` set is
    idempotent: scenario runs emit one ``on_result`` per turn, all
    sharing the same combo-level task key (scenario_id-keyed), and the
    combo is only marked complete once even though N turn lines are
    appended. QA runs emit exactly one result per task so the guard
    is a no-op.
    """
    if not is_completed_result(result):
        return

    with self._lock:
        task_id = _scenario_manifest_key(result, self._manifest)
        _append_jsonl_line(self.jsonl_path, result.model_dump_json())
        self._results.append(result)
        if task_id not in self._completed:
            self._completed.add(task_id)
            self._failed.discard(task_id)
        self._save_state()

on_start

on_start(
    manifest: list[str], config: VerificationConfig
) -> None

Record the manifest and write the initial state file.

Merge semantics: the stored manifest is unioned with the provided one, preserving insertion order. This handles both the resume path (provided manifest is a subset of the stored full manifest) and the seeded-extend path (stored holds prior keys from :meth:seed_prior_results, provided holds the new-task keys from :func:generate_task_queue). Repeated keys are dropped.

Source code in src/karenina/benchmark/verification/sinks.py

def on_start(self, manifest: list[str], config: VerificationConfig) -> None:
    """Record the manifest and write the initial state file.

    Merge semantics: the stored manifest is unioned with the provided
    one, preserving insertion order. This handles both the resume path
    (provided manifest is a subset of the stored full manifest) and
    the seeded-extend path (stored holds prior keys from
    :meth:`seed_prior_results`, provided holds the new-task keys
    from :func:`generate_task_queue`). Repeated keys are dropped.
    """
    with self._lock:
        if not self._manifest:
            self._manifest = list(manifest)
        else:
            existing = set(self._manifest)
            for key in manifest:
                if key not in existing:
                    self._manifest.append(key)
                    existing.add(key)

        # Always refresh config + hash: a resume may pass a config that
        # was rehydrated from the state, but we keep whatever the caller
        # handed us so later callers get the current object identity.
        self.config = config
        if not self._config_hash:
            self._config_hash = _compute_config_hash(config)
        if self._start_time is None:
            self._start_time = time.time()

        self._save_state()
        # Touch the JSONL so readers can rely on its existence.
        if not self.jsonl_path.exists():
            self.jsonl_path.touch()

open_or_resume classmethod

open_or_resume(
    output_path: Path,
    config: VerificationConfig,
    benchmark_path: str,
    *,
    global_rubric: Rubric | None = None,
    config_updater: Callable[
        [VerificationConfig], VerificationConfig
    ]
    | None = None,
) -> ProgressiveFileSink

Resume from an existing state sidecar or create a fresh sink.

Resumes when the .state sidecar for output_path exists, otherwise creates parent directories and a fresh sink. On resume, config is not used for the sink itself (the stored config is rehydrated), but callers that rebuild their config each run should keep passing the fresh config to run_verification, which overwrites the sink's copy on start.

Parameters:

Name	Type	Description	Default
output_path	Path	Destination results file.	required
config	VerificationConfig	Config for a fresh run.	required
benchmark_path	str	Benchmark checkpoint path recorded in state.	required
global_rubric	Rubric | None	Optional global rubric to attach.	None
config_updater	Callable[[VerificationConfig], VerificationConfig] | None	Optional transform applied to the rehydrated stored config on resume, for example to re-inject secrets.	None

Returns:

Type	Description
ProgressiveFileSink	A fresh or resumed sink.

Source code in src/karenina/benchmark/verification/sinks.py

@classmethod
def open_or_resume(
    cls,
    output_path: Path,
    config: VerificationConfig,
    benchmark_path: str,
    *,
    global_rubric: Rubric | None = None,
    config_updater: Callable[[VerificationConfig], VerificationConfig] | None = None,
) -> ProgressiveFileSink:
    """Resume from an existing state sidecar or create a fresh sink.

    Resumes when the ``.state`` sidecar for ``output_path`` exists,
    otherwise creates parent directories and a fresh sink. On resume,
    ``config`` is not used for the sink itself (the stored config is
    rehydrated), but callers that rebuild their config each run should
    keep passing the fresh config to ``run_verification``, which
    overwrites the sink's copy on start.

    Args:
        output_path: Destination results file.
        config: Config for a fresh run.
        benchmark_path: Benchmark checkpoint path recorded in state.
        global_rubric: Optional global rubric to attach.
        config_updater: Optional transform applied to the rehydrated
            stored config on resume, for example to re-inject secrets.

    Returns:
        A fresh or resumed sink.
    """
    state_path = output_path.with_suffix(output_path.suffix + ".state")
    if state_path.exists():
        logger.info("Resuming progressive save from %s", state_path)
        return cls.load_for_resume(state_path, global_rubric=global_rubric, config_updater=config_updater)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    return cls(output_path, config, benchmark_path, global_rubric=global_rubric)

seed_prior_results

seed_prior_results(
    prior_results: VerificationResultSet,
) -> None

Pre-populate JSONL, _results, _completed, _manifest.

Called by the extend_* facades before run_verification so the sink's final export covers prior rows in addition to the new rows produced by the pipeline. Idempotent: rows whose triple is already tracked are skipped. Does not require :meth:on_start to have run first; the JSONL file is created on demand.

Source code in src/karenina/benchmark/verification/sinks.py

def seed_prior_results(self, prior_results: VerificationResultSet) -> None:
    """Pre-populate JSONL, ``_results``, ``_completed``, ``_manifest``.

    Called by the ``extend_*`` facades before ``run_verification`` so
    the sink's final export covers prior rows in addition to the new
    rows produced by the pipeline. Idempotent: rows whose triple is
    already tracked are skipped. Does not require :meth:`on_start` to
    have run first; the JSONL file is created on demand.
    """
    with self._lock:
        added = 0
        manifest_set = set(self._manifest)
        for row in prior_results.results:
            if not is_completed_result(row):
                continue
            task_id = _scenario_manifest_key(row, self._manifest)
            if task_id in self._completed:
                continue
            _append_jsonl_line(self.jsonl_path, row.model_dump_json())
            self._results.append(row)
            self._completed.add(task_id)
            self._failed.discard(task_id)
            if task_id not in manifest_set:
                self._manifest.append(task_id)
                manifest_set.add(task_id)
            added += 1
        if added:
            self._save_state()
            logger.info("ProgressiveFileSink seeded with %d prior rows", added)

set_global_rubric

set_global_rubric(rubric: Rubric | None) -> None

Attach a global rubric used by the final export.

Source code in src/karenina/benchmark/verification/sinks.py

def set_global_rubric(self, rubric: Rubric | None) -> None:
    """Attach a global rubric used by the final export."""
    self.global_rubric = rubric

write_final_export

write_final_export(output_path: Path | None = None) -> Path

Assemble and atomically write the frontend-export JSON.

Parameters:

Name	Type	Description	Default
output_path	Path | None	Destination. Defaults to :attr:output_path.	None

Returns:

Type	Description
Path	The path the export was written to.

Source code in src/karenina/benchmark/verification/sinks.py

def write_final_export(self, output_path: Path | None = None) -> Path:
    """Assemble and atomically write the frontend-export JSON.

    Args:
        output_path: Destination. Defaults to :attr:`output_path`.

    Returns:
        The path the export was written to.
    """
    with self._lock:
        target = output_path or self.output_path
        now = time.time()
        results = list(_read_jsonl_results(self.jsonl_path)) if self.jsonl_path.exists() else []
        latest_rows = _dedupe_latest_by_manifest_key(results, manifest=self._manifest)
        successful_count, failed_count = _count_success_failure(latest_rows)
        job = VerificationJob(
            job_id=f"progressive-{int(self._start_time or now)}",
            run_name="progressive",
            status="completed",
            config=self.config,
            total_questions=self.total_tasks,
            successful_count=successful_count,
            failed_count=failed_count,
            start_time=self._start_time,
            end_time=now,
        )
        # Stream from the JSONL sidecar if it exists. write_final_export
        # may be called before any on_result (e.g. the empty-run case),
        # in which case jsonl_path was never created; guard is mandatory
        # because _read_jsonl_results opens the path unconditionally.
        scenario_results = _reconstruct_scenario_results_for_export(results, manifest=self._manifest)
        export_verification_results_json_stream(
            job,
            iter(results),
            self.global_rubric,
            is_complete=True,
            scenario_results=scenario_results or None,
            out_path=target,
        )
        return target

RepairOutcome

Bases: BaseModel

Paths and counts produced by a completed repair.

Source code in src/karenina/benchmark/verification/repair.py

class RepairOutcome(BaseModel):
    """Paths and counts produced by a completed repair."""

    model_config = ConfigDict(extra="forbid")

    source_path: Path
    output_path: Path
    backup_path: Path | None
    provenance_path: Path | None
    selected_count: int
    replaced_count: int
    mode: RepairMode
    dry_run: bool
    selected_keys: list[str]

RepairSelection

Bases: BaseModel

Declarative row filters for completed-run repair.

Source code in src/karenina/benchmark/verification/repair.py

class RepairSelection(BaseModel):
    """Declarative row filters for completed-run repair."""

    model_config = ConfigDict(extra="forbid")

    question_ids: set[str] = Field(default_factory=set)
    answerer_keys: set[str] = Field(default_factory=set)
    parser_keys: set[str] = Field(default_factory=set)
    replicates: set[int | None] = Field(default_factory=set)
    failure_groups: set[str] = Field(default_factory=set)
    failure_categories: set[str] = Field(default_factory=set)
    failure_stages: set[str] = Field(default_factory=set)
    select_all: bool = False

    def has_filters(self) -> bool:
        """Return whether at least one explicit selection constraint exists."""
        return bool(
            self.select_all
            or self.question_ids
            or self.answerer_keys
            or self.parser_keys
            or self.replicates
            or self.failure_groups
            or self.failure_categories
            or self.failure_stages
        )

    def matches(self, result: VerificationResult) -> bool:
        """Return whether a result satisfies every configured filter."""
        if not self.has_filters():
            raise ValueError("Repair selection requires at least one filter or select_all=True")
        key = ResultRowKey.from_result(result)
        failure = result.metadata.failure
        failure_group = failure.group.value if failure is not None else None
        failure_category = failure.category.value if failure is not None else None
        failure_stage = failure.stage if failure is not None else None
        return bool(
            (not self.question_ids or key.question_id in self.question_ids)
            and (not self.answerer_keys or key.answering_key in self.answerer_keys)
            and (not self.parser_keys or key.parsing_key in self.parser_keys)
            and (not self.replicates or key.replicate in self.replicates)
            and (not self.failure_groups or failure_group in self.failure_groups)
            and (not self.failure_categories or failure_category in self.failure_categories)
            and (not self.failure_stages or failure_stage in self.failure_stages)
        )

Functions

has_filters

has_filters() -> bool

Return whether at least one explicit selection constraint exists.

Source code in src/karenina/benchmark/verification/repair.py

def has_filters(self) -> bool:
    """Return whether at least one explicit selection constraint exists."""
    return bool(
        self.select_all
        or self.question_ids
        or self.answerer_keys
        or self.parser_keys
        or self.replicates
        or self.failure_groups
        or self.failure_categories
        or self.failure_stages
    )

matches

matches(result: VerificationResult) -> bool

Return whether a result satisfies every configured filter.

Source code in src/karenina/benchmark/verification/repair.py

def matches(self, result: VerificationResult) -> bool:
    """Return whether a result satisfies every configured filter."""
    if not self.has_filters():
        raise ValueError("Repair selection requires at least one filter or select_all=True")
    key = ResultRowKey.from_result(result)
    failure = result.metadata.failure
    failure_group = failure.group.value if failure is not None else None
    failure_category = failure.category.value if failure is not None else None
    failure_stage = failure.stage if failure is not None else None
    return bool(
        (not self.question_ids or key.question_id in self.question_ids)
        and (not self.answerer_keys or key.answering_key in self.answerer_keys)
        and (not self.parser_keys or key.parsing_key in self.parser_keys)
        and (not self.replicates or key.replicate in self.replicates)
        and (not self.failure_groups or failure_group in self.failure_groups)
        and (not self.failure_categories or failure_category in self.failure_categories)
        and (not self.failure_stages or failure_stage in self.failure_stages)
    )

ResultsIOManager

Manager for verification result export and import operations.

Handles CSV and JSON serialization/deserialization logic for ResultsManager.

Source code in src/karenina/benchmark/core/results_io.py

class ResultsIOManager:
    """Manager for verification result export and import operations.

    Handles CSV and JSON serialization/deserialization logic for ResultsManager.
    """

    @staticmethod
    def export_to_json(results: dict[str, VerificationResult]) -> str:
        """
        Export results to JSON format matching frontend format.

        Args:
            results: Dictionary of verification results

        Returns:
            JSON string with results array including row_index
        """
        results_array = []
        for index, (_, result) in enumerate(results.items(), 1):
            result_dict = result.model_dump()
            result_dict["row_index"] = index
            # Replace success with "abstained" when abstention is detected
            if result.template and result.template.abstention_detected and result.template.abstention_override_applied:
                result_dict["success"] = "abstained"
            results_array.append(result_dict)
        return json.dumps(results_array, indent=2, ensure_ascii=False)

    @staticmethod
    def export_to_csv(results: dict[str, VerificationResult], global_rubric: "Rubric | None" = None) -> str:
        """
        Export results to CSV format matching frontend format exactly.

        Args:
            results: Dictionary of verification results
            global_rubric: Optional global rubric for trait separation

        Returns:
            CSV string in frontend format
        """
        if not results:
            return (
                "row_index,question_id,question_text,success,"
                "failure_category,failure_group,failure_stage,failure_reason,failure_details_json,"
                "caveats,execution_time,timestamp,"
                "embedding_check_performed,embedding_similarity_score,embedding_override_applied,"
                "embedding_model_used\n"
            )

        # Extract all unique rubric trait names from results
        all_rubric_trait_names: set[str] = set()
        for result in results.values():
            if result.rubric:
                scores = result.rubric.get_all_trait_scores()
                if scores:
                    all_rubric_trait_names.update(scores.keys())

        # Determine global vs question-specific rubrics
        global_trait_names = set()
        if global_rubric and hasattr(global_rubric, "traits"):
            for trait in global_rubric.llm_traits:
                global_trait_names.add(trait.name)

        # Separate traits into global and question-specific
        global_traits: list[str] = sorted([trait for trait in all_rubric_trait_names if trait in global_trait_names])
        question_specific_traits: list[str] = sorted(
            [trait for trait in all_rubric_trait_names if trait not in global_trait_names]
        )

        # Create headers for global rubrics only
        global_rubric_headers = [f"rubric_{trait}" for trait in global_traits]

        headers = [
            "row_index",
            "question_id",
            "question_text",
            "raw_llm_response",
            "parsed_gt_response",
            "parsed_llm_response",
            "verify_result",
            "verify_granular_result",
            *global_rubric_headers,
            *(["question_specific_rubrics"] if question_specific_traits else []),
            "rubric_summary",
            "answering_model",
            "parsing_model",
            "replicate",
            "answering_system_prompt",
            "parsing_system_prompt",
            "success",
            "failure_category",
            "failure_group",
            "failure_stage",
            "failure_reason",
            "failure_details_json",
            "caveats",
            "execution_time",
            "timestamp",
            "run_name",
            "job_id",
            # Embedding check fields
            "embedding_check_performed",
            "embedding_similarity_score",
            "embedding_override_applied",
            "embedding_model_used",
        ]

        output = StringIO()
        writer = csv.writer(output)
        writer.writerow(headers)

        for index, result in enumerate(results.values(), start=1):
            row = ResultsIOManager._create_csv_row(index, result, global_traits, question_specific_traits)
            writer.writerow(row)

        return output.getvalue()

    @staticmethod
    def _create_csv_row(
        index: int,
        result: VerificationResult,
        global_traits: list[str],
        question_specific_traits: list[str],
    ) -> list[Any]:
        """Create a CSV row for a single verification result."""
        # Get all trait scores from rubric sub-object
        all_trait_scores = result.rubric.get_all_trait_scores() if result.rubric else None

        # Extract global rubric trait values
        global_rubric_values: list[str] = []
        for trait_name in global_traits:
            if all_trait_scores and trait_name in all_trait_scores:
                value = all_trait_scores[trait_name]
                global_rubric_values.append(str(value) if value is not None else "")
            else:
                global_rubric_values.append("")

        # Create question-specific rubrics JSON
        question_specific_rubrics: dict[str, int | bool | float | str | list[Any] | dict[str, float] | None] = {}
        if all_trait_scores:
            for trait_name in question_specific_traits:
                if trait_name in all_trait_scores:
                    question_specific_rubrics[trait_name] = all_trait_scores[trait_name]

        question_specific_rubrics_value = json.dumps(question_specific_rubrics) if question_specific_traits else ""

        # Create rubric summary
        rubric_summary = ""
        if all_trait_scores:
            traits = list(all_trait_scores.items())
            passed_traits = sum(
                1
                for name, value in traits
                if (isinstance(value, bool) and value) or (isinstance(value, int | float) and value >= 3)
            )
            rubric_summary = f"{passed_traits}/{len(traits)}"

        failure = result.metadata.failure
        success_value: str | bool
        if result.template and result.template.abstention_detected and result.template.abstention_override_applied:
            success_value = "abstained"
        else:
            success_value = failure is None

        return [
            index,  # row_index
            result.metadata.question_id,
            result.metadata.question_text,
            result.template.raw_llm_response if result.template else "",
            (
                json.dumps(result.template.parsed_gt_response)
                if result.template and result.template.parsed_gt_response
                else ""
            ),
            (
                json.dumps(result.template.parsed_llm_response)
                if result.template and result.template.parsed_llm_response
                else ""
            ),
            (
                json.dumps(result.template.verify_result)
                if result.template and result.template.verify_result is not None
                else "N/A"
            ),
            (
                json.dumps(result.template.verify_granular_result)
                if result.template and result.template.verify_granular_result is not None
                else "N/A"
            ),
            *global_rubric_values,
            *([question_specific_rubrics_value] if question_specific_traits else []),
            rubric_summary,
            result.metadata.answering_model,
            result.metadata.parsing_model,
            result.metadata.replicate or "",
            result.metadata.answering_system_prompt or "",
            result.metadata.parsing_system_prompt or "",
            success_value,
            failure.category.value if failure else "",
            failure.group.value if failure else "",
            failure.stage if failure else "",
            failure.reason if failure else "",
            json.dumps(failure.details) if (failure and failure.details) else "",
            ",".join(c.value for c in result.metadata.caveats),
            result.metadata.execution_time,
            result.metadata.timestamp,
            result.metadata.run_name or "",
            "",  # job_id: not stored in VerificationResult, populated by server layer
            # Embedding check fields
            result.template.embedding_check_performed if result.template else False,
            result.template.embedding_similarity_score if result.template else "",
            result.template.embedding_override_applied if result.template else False,
            result.template.embedding_model_used if result.template else "",
        ]

    @staticmethod
    def _escape_csv_field(field: Any) -> str:
        """
        Escape CSV field content matching frontend logic.

        Args:
            field: Field value to escape

        Returns:
            Escaped string value
        """
        if field is None:
            return ""
        str_field = str(field)
        if "," in str_field or '"' in str_field or "\n" in str_field:
            return '"' + str_field.replace('"', '""') + '"'
        return str_field

    @staticmethod
    def load_from_json(file_path: Path) -> dict[str, VerificationResult]:
        """
        Load results from JSON file supporting multiple formats.

        Args:
            file_path: Path to the JSON file

        Returns:
            Dictionary of loaded verification results
        """
        with open(file_path, encoding="utf-8") as f:
            data = json.load(f)

        results = {}

        # Handle different JSON formats
        if isinstance(data, list):
            # Frontend format: array with row_index
            for item in data:
                if isinstance(item, dict):
                    # Remove row_index if present and create key
                    item_copy = dict(item)
                    row_index = item_copy.pop("row_index", None)
                    # Check top-level first, then nested metadata for question_id
                    question_id = item_copy.get("question_id")
                    if question_id is None:
                        metadata = item_copy.get("metadata")
                        if isinstance(metadata, dict):
                            question_id = metadata.get("question_id", "unknown")
                        else:
                            question_id = "unknown"
                    result_key = f"{question_id}_{row_index}" if row_index else question_id
                    try:
                        results[result_key] = VerificationResult(**item_copy)
                    except Exception:
                        logger.warning("Skipping malformed JSON item at key %s", result_key, exc_info=True)
                        continue
        elif isinstance(data, dict):
            # Handle server format with metadata wrapper
            results_data = data["results"] if "results" in data and "metadata" in data else data

            # Reconstruct VerificationResult objects
            for result_key, result_data in results_data.items():
                if isinstance(result_data, dict):
                    try:
                        results[result_key] = VerificationResult(**result_data)
                    except Exception:
                        logger.warning("Skipping malformed result at key %s", result_key, exc_info=True)
                        continue

        return results

    @staticmethod
    def load_result_set_from_json(file_path: Path) -> VerificationResultSet:
        """Load and validate a complete verification result set from JSON.

        Unlike the legacy dictionary loader, this method preserves run metadata
        and compact scenario execution records when the export contains them.
        Malformed rows raise instead of being skipped, making this the preferred
        analysis path for saved experiment outputs.

        Args:
            file_path: Path to a standard JSON verification export.

        Returns:
            A validated result set with optional scenario results and metadata.

        Raises:
            ValueError: If the JSON shape or any result row is invalid.
        """
        with open(file_path, encoding="utf-8") as handle:
            data = json.load(handle)

        metadata: dict[str, Any] = {}
        scenario_data: list[object] = []
        result_data: object

        if isinstance(data, list):
            result_data = data
        elif isinstance(data, dict):
            loaded_metadata = data.get("metadata")
            if loaded_metadata is not None:
                if not isinstance(loaded_metadata, dict):
                    raise ValueError("Invalid results export: metadata must be an object")
                metadata = dict(loaded_metadata)

            if "results" in data:
                result_data = data["results"]
                loaded_scenarios = data.get("scenario_results", [])
                if not isinstance(loaded_scenarios, list):
                    raise ValueError("Invalid results export: scenario_results must be an array")
                scenario_data = loaded_scenarios
            elif "runs" in data:
                runs = data["runs"]
                if not isinstance(runs, dict):
                    raise ValueError("Invalid results export: runs must be an object")
                result_data = [row for rows in runs.values() for row in rows]
                metadata = {**metadata, "run_names": list(runs)}
            else:
                result_data = data
        else:
            raise ValueError("Invalid results export: top-level JSON must be an array or object")

        if isinstance(result_data, dict):
            result_items = list(result_data.values())
        elif isinstance(result_data, list):
            result_items = result_data
        else:
            raise ValueError("Invalid results export: results must be an array or object")

        results: list[VerificationResult] = []
        for index, item in enumerate(result_items):
            if not isinstance(item, dict):
                raise ValueError(f"Invalid results export: result {index} must be an object")
            item_copy = dict(item)
            item_copy.pop("row_index", None)
            try:
                results.append(VerificationResult.model_validate(item_copy))
            except Exception as e:
                raise ValueError(f"Invalid verification result at index {index}") from e

        scenario_results: list[ScenarioResultRecord] = []
        for index, item in enumerate(scenario_data):
            try:
                scenario_results.append(ScenarioResultRecord.model_validate(item))
            except Exception as e:
                raise ValueError(f"Invalid scenario result at index {index}") from e

        return VerificationResultSet(
            results=results,
            scenario_results=scenario_results or None,
            metadata=metadata,
        )

    @staticmethod
    def load_legacy_result_set(file_path: Path) -> VerificationResultSet:
        """Load a legacy (v2.0/v2.1) verification export, migrating rows to the current schema.

        Legacy exports predate nested ModelIdentity metadata and deterministic
        result IDs, so ``load_result_set_from_json`` rejects their rows
        (forbidden flat keys, missing ``answering``/``parsing``/``result_id``).
        This loader migrates each row instead: the forbidden flat keys are
        dropped, ``answering_model``/``parsing_model`` strings
        (``"provider/model"``) become ModelIdentity objects,
        ``answering_replicate``/``parsing_replicate`` fold into ``replicate``,
        and a deterministic ``result_id`` is synthesized from question,
        models, timestamp, and replicate when absent.

        Non-legacy exports are delegated to ``load_result_set_from_json``
        unchanged.

        Args:
            file_path: Path to a legacy (v2.0/v2.1) or current JSON export.

        Returns:
            A validated result set built from the migrated rows.

        Raises:
            ValueError: If any row fails migration; the message names every
                failing row index with its reason.
        """
        with open(file_path, encoding="utf-8") as handle:
            data = json.load(handle)

        if not ResultsIOManager._is_legacy_export(data):
            return ResultsIOManager.load_result_set_from_json(file_path)

        rows = data["results"]
        metadata = data.get("metadata", {})
        if not isinstance(metadata, dict):
            raise ValueError("Invalid legacy export: metadata must be an object")

        results: list[VerificationResult] = []
        errors: list[str] = []
        seen_result_ids: set[str] = set()
        for index, row in enumerate(rows):
            try:
                results.append(ResultsIOManager._migrate_legacy_row(row, index, seen_result_ids))
            except Exception as e:
                errors.append(f"index {index}: {e}")

        if errors:
            raise ValueError(
                f"Legacy export migration failed for {len(errors)} of {len(rows)} rows: " + "; ".join(errors)
            )

        return VerificationResultSet(results=results, metadata=dict(metadata))

    @staticmethod
    def _is_legacy_export(data: Any) -> bool:
        """Detect whether a parsed export uses the legacy (v2.0/v2.1) row shape."""
        if not isinstance(data, dict):
            return False
        version = data.get("format_version")
        if isinstance(version, str):
            return version in LEGACY_FORMAT_VERSIONS
        rows = data.get("results")
        if not isinstance(rows, list) or not rows:
            return False
        first_row = rows[0]
        if not isinstance(first_row, dict):
            return False
        row_metadata = first_row.get("metadata")
        return isinstance(row_metadata, dict) and bool(LEGACY_METADATA_KEYS & row_metadata.keys())

    @staticmethod
    def _migrate_legacy_row(row: Any, index: int, seen_result_ids: set[str]) -> VerificationResult:
        """Migrate one legacy row into a VerificationResult, or raise with a reason."""
        if not isinstance(row, dict):
            raise ValueError("row must be an object")
        legacy_metadata = row.get("metadata")
        if not isinstance(legacy_metadata, dict):
            raise ValueError("row metadata must be an object")

        metadata = dict(legacy_metadata)
        answering = ResultsIOManager._legacy_model_identity(metadata.pop("answering_model", None))
        parsing = ResultsIOManager._legacy_model_identity(metadata.pop("parsing_model", None))
        metadata["answering"] = answering
        metadata["parsing"] = parsing

        answering_replicate = metadata.pop("answering_replicate", None)
        parsing_replicate = metadata.pop("parsing_replicate", None)
        replicate = answering_replicate if answering_replicate is not None else parsing_replicate
        if replicate is not None:
            metadata["replicate"] = replicate

        # Dropped legacy fields with no current-schema equivalent.
        metadata.pop("completed_without_errors", None)
        metadata.pop("error", None)

        question_id = metadata.get("question_id")
        timestamp = metadata.get("timestamp")
        if not isinstance(question_id, str) or not isinstance(timestamp, str):
            raise ValueError("metadata.question_id and metadata.timestamp must be strings")
        if not metadata.get("result_id"):
            result_id = VerificationResultMetadata.compute_result_id(
                question_id, answering, parsing, timestamp, replicate
            )
            if result_id in seen_result_ids:
                # Identical legacy rows (same question, models, timestamp, and
                # replicate) hash identically; disambiguate with the row index.
                result_id = hashlib.sha256(f"{result_id}:{index}".encode()).hexdigest()[:16]
            metadata["result_id"] = result_id
        seen_result_ids.add(metadata["result_id"])

        migrated = dict(row)
        migrated["metadata"] = metadata
        try:
            return VerificationResult.model_validate(migrated)
        except Exception as e:
            raise ValueError(str(e)) from e

    @staticmethod
    def _legacy_model_identity(model_string: Any) -> ModelIdentity:
        """Build a ModelIdentity from a legacy flat ``"provider/model"`` string."""
        if not isinstance(model_string, str) or not model_string:
            return ModelIdentity(interface="unknown", model_name="unknown")
        provider, _, model_name = model_string.partition("/")
        if not model_name:
            return ModelIdentity(interface="unknown", model_name=model_string)
        return ModelIdentity(interface=provider, model_name=model_name)

    @staticmethod
    def iter_from_json(
        file_path: Path,
        *,
        raw: bool = False,
    ) -> Iterator[VerificationResult] | Iterator[dict[str, Any]]:
        """Stream rows from a results JSON export.

        Args:
            file_path: Path to a v2.2 export or legacy array export.
            raw: Yield dictionaries instead of validating current result models.

        Returns:
            An iterator over results or raw row dictionaries.
        """
        from karenina.benchmark.core.results_stream import iter_results_from_json

        if raw:
            return iter_results_from_json(file_path, raw=True)
        return iter_results_from_json(file_path)

    @staticmethod
    def load_from_csv(file_path: Path) -> dict[str, VerificationResult]:
        """Load results from CSV file, reconstructing nested VerificationResult objects.

        Parses the flat CSV columns back into the nested sub-objects
        (metadata, template, rubric) that VerificationResult expects.

        Args:
            file_path: Path to the CSV file.

        Returns:
            Dictionary of loaded verification results keyed by "{question_id}_{row_index}".
        """
        results: dict[str, VerificationResult] = {}

        with open(file_path, encoding="utf-8") as f:
            reader = csv.DictReader(f)
            for row_num, row in enumerate(reader, start=1):
                try:
                    result_key, verification_result = ResultsIOManager._process_csv_row(row)
                    if verification_result is not None:
                        results[result_key] = verification_result
                    else:
                        logger.warning("Skipping invalid CSV row %d: processing returned None", row_num)
                except Exception:
                    logger.warning("Skipping malformed CSV row %d", row_num, exc_info=True)
                    continue

        logger.info("Loaded %d results from CSV file %s", len(results), file_path)
        return results

    @staticmethod
    def _parse_model_identity(display_string: str | None) -> ModelIdentity:
        """Parse a ModelIdentity display string back into a ModelIdentity object.

        Handles these formats:
        - "interface:model_name"
        - "interface:model_name (config_id)"
        - "interface:model_name +[tool1, tool2]"
        - "interface:model_name (config_id) +[tool1, tool2]"

        Args:
            display_string: The display string from CSV export, or None/empty.

        Returns:
            Reconstructed ModelIdentity.
        """
        if not display_string:
            return ModelIdentity(interface="unknown", model_name="unknown")

        tools: list[str] = []
        base = display_string

        # Split on " +[" to extract tools
        if " +[" in display_string:
            base, tools_part = display_string.split(" +[", 1)
            tools = [t.strip() for t in tools_part.rstrip("]").split(",") if t.strip()]

        # Extract config_id from " (config_id)" suffix
        config_id: str | None = None
        if " (" in base and base.endswith(")"):
            candidate_base, config_part = base.rsplit(" (", 1)
            if ":" in candidate_base:
                config_id = config_part.rstrip(")")
                base = candidate_base

        # Split base on ":" to get interface and model_name
        parts = base.split(":", 1)
        interface = parts[0] if parts else "unknown"
        model_name = parts[1] if len(parts) > 1 else "unknown"

        return ModelIdentity(
            interface=interface,
            model_name=model_name,
            tools=tools,
            config_id=config_id,
        )

    @staticmethod
    def _parse_rubric_traits(row: dict[str, str]) -> dict[str, int | bool]:
        """Extract rubric trait scores from CSV row columns.

        Collects traits from "rubric_*" columns and the "question_specific_rubrics"
        JSON column, converting values to int or bool.

        Args:
            row: Dictionary from csv.DictReader.

        Returns:
            Dict of trait name to score (int or bool). Empty dict if no traits found.
        """
        trait_scores: dict[str, int | bool] = {}

        # Extract global rubric traits (columns starting with "rubric_")
        for key, value in row.items():
            if key == "rubric_summary":
                continue
            if key.startswith("rubric_") and value:
                trait_name = key[len("rubric_") :]
                try:
                    if value.isdigit():
                        trait_scores[trait_name] = int(value)
                    elif value.lower() in ("true", "false"):
                        trait_scores[trait_name] = value.lower() == "true"
                    else:
                        # Try float then int conversion for numeric strings like "4.0"
                        trait_scores[trait_name] = int(float(value))
                except (ValueError, AttributeError):
                    logger.debug("Could not convert rubric trait value for %s, skipping", trait_name)

        # Extract question-specific rubrics from JSON column
        qs_raw = row.get("question_specific_rubrics", "")
        if qs_raw:
            try:
                question_specific = json.loads(qs_raw)
                if isinstance(question_specific, dict):
                    for name, value in question_specific.items():
                        if isinstance(value, bool | int):
                            trait_scores[name] = value
                        elif isinstance(value, float) or isinstance(value, str) and value.isdigit():
                            trait_scores[name] = int(value)
            except json.JSONDecodeError:
                logger.debug("Could not parse question_specific_rubrics JSON")

        return trait_scores

    @staticmethod
    def _parse_json_field(value: str | None) -> Any:
        """Parse a JSON string field, returning None for empty or N/A values.

        Args:
            value: Raw string from CSV cell.

        Returns:
            Parsed JSON value, or None if the value is empty/N/A/unparseable.
        """
        if not value or value == "N/A":
            return None
        try:
            return json.loads(value)
        except (json.JSONDecodeError, TypeError):
            return None

    @staticmethod
    def _process_csv_row(row: dict[str, str]) -> tuple[str, VerificationResult | None]:
        """Process a single CSV row into a VerificationResult with nested sub-objects.

        Reconstructs the metadata, template, and rubric sub-objects from the
        flat CSV columns that export_to_csv produces.

        Args:
            row: Dictionary from csv.DictReader.

        Returns:
            Tuple of (result_key, VerificationResult) or (result_key, None) if
            the row cannot be parsed.
        """
        # Generate result key
        row_index = row.get("row_index", "")
        question_id = row.get("question_id", "unknown")
        result_key = f"{question_id}_{row_index}" if row_index else question_id

        # Parse model identities from display strings
        answering = ResultsIOManager._parse_model_identity(row.get("answering_model"))
        parsing = ResultsIOManager._parse_model_identity(row.get("parsing_model"))

        # Parse execution_time
        execution_time = 0.0
        if row.get("execution_time"):
            try:
                execution_time = float(row["execution_time"])
            except ValueError:
                logger.debug("Could not parse execution_time '%s', defaulting to 0.0", row.get("execution_time"))

        # Parse replicate
        replicate: int | None = None
        if row.get("replicate"):
            try:
                replicate = int(row["replicate"])
            except ValueError:
                replicate = None

        # Build timestamp (required field; fall back to current time if missing)
        timestamp = row.get("timestamp") or ""

        # Compute result_id from available data
        result_id = VerificationResultMetadata.compute_result_id(
            question_id=question_id,
            answering=answering,
            parsing=parsing,
            timestamp=timestamp,
            replicate=replicate,
        )

        # Reconstruct failure + caveats from dedicated CSV columns.
        failure: Failure | None = None
        failure_category_raw = row.get("failure_category") or ""
        if failure_category_raw:
            details_raw = row.get("failure_details_json") or ""
            details = json.loads(details_raw) if details_raw else None
            failure = Failure(
                category=FailureCategory(failure_category_raw),
                stage=row.get("failure_stage") or "",
                reason=row.get("failure_reason") or "",
                details=details,
            )

        caveats_raw = row.get("caveats") or ""
        caveats: list[Caveat] = [Caveat(c) for c in caveats_raw.split(",") if c]

        # Build metadata sub-object
        metadata = VerificationResultMetadata(
            question_id=question_id,
            template_id="no_template",
            failure=failure,
            caveats=caveats,
            question_text=row.get("question_text", ""),
            answering=answering,
            parsing=parsing,
            answering_system_prompt=row.get("answering_system_prompt") or None,
            parsing_system_prompt=row.get("parsing_system_prompt") or None,
            execution_time=execution_time,
            timestamp=timestamp,
            result_id=result_id,
            run_name=row.get("run_name") or None,
            replicate=replicate,
        )

        # Build template sub-object (if any template-related data exists)
        template: VerificationResultTemplate | None = None
        raw_llm_response = row.get("raw_llm_response", "")
        verify_result_raw = row.get("verify_result")
        parsed_gt = ResultsIOManager._parse_json_field(row.get("parsed_gt_response"))
        parsed_llm = ResultsIOManager._parse_json_field(row.get("parsed_llm_response"))
        verify_result = ResultsIOManager._parse_json_field(verify_result_raw)
        verify_granular = ResultsIOManager._parse_json_field(row.get("verify_granular_result"))

        # Parse embedding fields
        embedding_check = row.get("embedding_check_performed", "").lower() in ("true", "1")
        embedding_score: float | None = None
        if row.get("embedding_similarity_score"):
            try:
                embedding_score = float(row["embedding_similarity_score"])
            except ValueError:
                embedding_score = None
        embedding_override = row.get("embedding_override_applied", "").lower() in ("true", "1")
        embedding_model = row.get("embedding_model_used") or None

        has_template_data = bool(
            raw_llm_response or parsed_gt or parsed_llm or verify_result is not None or verify_granular is not None
        )
        if has_template_data:
            template = VerificationResultTemplate(
                raw_llm_response=raw_llm_response or "",
                parsed_gt_response=parsed_gt,
                parsed_llm_response=parsed_llm,
                verify_result=verify_result,
                verify_granular_result=verify_granular,
                embedding_check_performed=embedding_check,
                embedding_similarity_score=embedding_score,
                embedding_override_applied=embedding_override,
                embedding_model_used=embedding_model,
            )

        # Build rubric sub-object (if any rubric traits exist)
        rubric: VerificationResultRubric | None = None
        trait_scores = ResultsIOManager._parse_rubric_traits(row)
        if trait_scores:
            rubric = VerificationResultRubric(
                rubric_evaluation_performed=True,
                llm_trait_scores=trait_scores,
            )

        verification_result = VerificationResult(
            metadata=metadata,
            template=template,
            rubric=rubric,
        )

        return result_key, verification_result

Functions

export_to_csv staticmethod

export_to_csv(
    results: dict[str, VerificationResult],
    global_rubric: Rubric | None = None,
) -> str

Export results to CSV format matching frontend format exactly.

Parameters:

Name	Type	Description	Default
results	dict[str, VerificationResult]	Dictionary of verification results	required
global_rubric	Rubric | None	Optional global rubric for trait separation	None

Returns:

Type	Description
str	CSV string in frontend format

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def export_to_csv(results: dict[str, VerificationResult], global_rubric: "Rubric | None" = None) -> str:
    """
    Export results to CSV format matching frontend format exactly.

    Args:
        results: Dictionary of verification results
        global_rubric: Optional global rubric for trait separation

    Returns:
        CSV string in frontend format
    """
    if not results:
        return (
            "row_index,question_id,question_text,success,"
            "failure_category,failure_group,failure_stage,failure_reason,failure_details_json,"
            "caveats,execution_time,timestamp,"
            "embedding_check_performed,embedding_similarity_score,embedding_override_applied,"
            "embedding_model_used\n"
        )

    # Extract all unique rubric trait names from results
    all_rubric_trait_names: set[str] = set()
    for result in results.values():
        if result.rubric:
            scores = result.rubric.get_all_trait_scores()
            if scores:
                all_rubric_trait_names.update(scores.keys())

    # Determine global vs question-specific rubrics
    global_trait_names = set()
    if global_rubric and hasattr(global_rubric, "traits"):
        for trait in global_rubric.llm_traits:
            global_trait_names.add(trait.name)

    # Separate traits into global and question-specific
    global_traits: list[str] = sorted([trait for trait in all_rubric_trait_names if trait in global_trait_names])
    question_specific_traits: list[str] = sorted(
        [trait for trait in all_rubric_trait_names if trait not in global_trait_names]
    )

    # Create headers for global rubrics only
    global_rubric_headers = [f"rubric_{trait}" for trait in global_traits]

    headers = [
        "row_index",
        "question_id",
        "question_text",
        "raw_llm_response",
        "parsed_gt_response",
        "parsed_llm_response",
        "verify_result",
        "verify_granular_result",
        *global_rubric_headers,
        *(["question_specific_rubrics"] if question_specific_traits else []),
        "rubric_summary",
        "answering_model",
        "parsing_model",
        "replicate",
        "answering_system_prompt",
        "parsing_system_prompt",
        "success",
        "failure_category",
        "failure_group",
        "failure_stage",
        "failure_reason",
        "failure_details_json",
        "caveats",
        "execution_time",
        "timestamp",
        "run_name",
        "job_id",
        # Embedding check fields
        "embedding_check_performed",
        "embedding_similarity_score",
        "embedding_override_applied",
        "embedding_model_used",
    ]

    output = StringIO()
    writer = csv.writer(output)
    writer.writerow(headers)

    for index, result in enumerate(results.values(), start=1):
        row = ResultsIOManager._create_csv_row(index, result, global_traits, question_specific_traits)
        writer.writerow(row)

    return output.getvalue()

export_to_json staticmethod

export_to_json(
    results: dict[str, VerificationResult],
) -> str

Export results to JSON format matching frontend format.

Parameters:

Name	Type	Description	Default
results	dict[str, VerificationResult]	Dictionary of verification results	required

Returns:

Type	Description
str	JSON string with results array including row_index

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def export_to_json(results: dict[str, VerificationResult]) -> str:
    """
    Export results to JSON format matching frontend format.

    Args:
        results: Dictionary of verification results

    Returns:
        JSON string with results array including row_index
    """
    results_array = []
    for index, (_, result) in enumerate(results.items(), 1):
        result_dict = result.model_dump()
        result_dict["row_index"] = index
        # Replace success with "abstained" when abstention is detected
        if result.template and result.template.abstention_detected and result.template.abstention_override_applied:
            result_dict["success"] = "abstained"
        results_array.append(result_dict)
    return json.dumps(results_array, indent=2, ensure_ascii=False)

iter_from_json staticmethod

iter_from_json(
    file_path: Path, *, raw: bool = False
) -> (
    Iterator[VerificationResult] | Iterator[dict[str, Any]]
)

Stream rows from a results JSON export.

Parameters:

Name	Type	Description	Default
file_path	Path	Path to a v2.2 export or legacy array export.	required
raw	bool	Yield dictionaries instead of validating current result models.	False

Returns:

Type	Description
Iterator[VerificationResult] | Iterator[dict[str, Any]]	An iterator over results or raw row dictionaries.

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def iter_from_json(
    file_path: Path,
    *,
    raw: bool = False,
) -> Iterator[VerificationResult] | Iterator[dict[str, Any]]:
    """Stream rows from a results JSON export.

    Args:
        file_path: Path to a v2.2 export or legacy array export.
        raw: Yield dictionaries instead of validating current result models.

    Returns:
        An iterator over results or raw row dictionaries.
    """
    from karenina.benchmark.core.results_stream import iter_results_from_json

    if raw:
        return iter_results_from_json(file_path, raw=True)
    return iter_results_from_json(file_path)

load_from_csv staticmethod

load_from_csv(
    file_path: Path,
) -> dict[str, VerificationResult]

Load results from CSV file, reconstructing nested VerificationResult objects.

Parses the flat CSV columns back into the nested sub-objects (metadata, template, rubric) that VerificationResult expects.

Parameters:

Name	Type	Description	Default
file_path	Path	Path to the CSV file.	required

Returns:

Type	Description
dict[str, VerificationResult]	Dictionary of loaded verification results keyed by "{question_id}_{row_index}".

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def load_from_csv(file_path: Path) -> dict[str, VerificationResult]:
    """Load results from CSV file, reconstructing nested VerificationResult objects.

    Parses the flat CSV columns back into the nested sub-objects
    (metadata, template, rubric) that VerificationResult expects.

    Args:
        file_path: Path to the CSV file.

    Returns:
        Dictionary of loaded verification results keyed by "{question_id}_{row_index}".
    """
    results: dict[str, VerificationResult] = {}

    with open(file_path, encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row_num, row in enumerate(reader, start=1):
            try:
                result_key, verification_result = ResultsIOManager._process_csv_row(row)
                if verification_result is not None:
                    results[result_key] = verification_result
                else:
                    logger.warning("Skipping invalid CSV row %d: processing returned None", row_num)
            except Exception:
                logger.warning("Skipping malformed CSV row %d", row_num, exc_info=True)
                continue

    logger.info("Loaded %d results from CSV file %s", len(results), file_path)
    return results

load_from_json staticmethod

load_from_json(
    file_path: Path,
) -> dict[str, VerificationResult]

Load results from JSON file supporting multiple formats.

Parameters:

Name	Type	Description	Default
file_path	Path	Path to the JSON file	required

Returns:

Type	Description
dict[str, VerificationResult]	Dictionary of loaded verification results

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def load_from_json(file_path: Path) -> dict[str, VerificationResult]:
    """
    Load results from JSON file supporting multiple formats.

    Args:
        file_path: Path to the JSON file

    Returns:
        Dictionary of loaded verification results
    """
    with open(file_path, encoding="utf-8") as f:
        data = json.load(f)

    results = {}

    # Handle different JSON formats
    if isinstance(data, list):
        # Frontend format: array with row_index
        for item in data:
            if isinstance(item, dict):
                # Remove row_index if present and create key
                item_copy = dict(item)
                row_index = item_copy.pop("row_index", None)
                # Check top-level first, then nested metadata for question_id
                question_id = item_copy.get("question_id")
                if question_id is None:
                    metadata = item_copy.get("metadata")
                    if isinstance(metadata, dict):
                        question_id = metadata.get("question_id", "unknown")
                    else:
                        question_id = "unknown"
                result_key = f"{question_id}_{row_index}" if row_index else question_id
                try:
                    results[result_key] = VerificationResult(**item_copy)
                except Exception:
                    logger.warning("Skipping malformed JSON item at key %s", result_key, exc_info=True)
                    continue
    elif isinstance(data, dict):
        # Handle server format with metadata wrapper
        results_data = data["results"] if "results" in data and "metadata" in data else data

        # Reconstruct VerificationResult objects
        for result_key, result_data in results_data.items():
            if isinstance(result_data, dict):
                try:
                    results[result_key] = VerificationResult(**result_data)
                except Exception:
                    logger.warning("Skipping malformed result at key %s", result_key, exc_info=True)
                    continue

    return results

load_legacy_result_set staticmethod

load_legacy_result_set(
    file_path: Path,
) -> VerificationResultSet

Load a legacy (v2.0/v2.1) verification export, migrating rows to the current schema.

Legacy exports predate nested ModelIdentity metadata and deterministic result IDs, so load_result_set_from_json rejects their rows (forbidden flat keys, missing answering/parsing/result_id). This loader migrates each row instead: the forbidden flat keys are dropped, answering_model/parsing_model strings ("provider/model") become ModelIdentity objects, answering_replicate/parsing_replicate fold into replicate, and a deterministic result_id is synthesized from question, models, timestamp, and replicate when absent.

Non-legacy exports are delegated to load_result_set_from_json unchanged.

Parameters:

Name	Type	Description	Default
file_path	Path	Path to a legacy (v2.0/v2.1) or current JSON export.	required

Returns:

Type	Description
VerificationResultSet	A validated result set built from the migrated rows.

Raises:

Type	Description
ValueError	If any row fails migration; the message names every failing row index with its reason.

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def load_legacy_result_set(file_path: Path) -> VerificationResultSet:
    """Load a legacy (v2.0/v2.1) verification export, migrating rows to the current schema.

    Legacy exports predate nested ModelIdentity metadata and deterministic
    result IDs, so ``load_result_set_from_json`` rejects their rows
    (forbidden flat keys, missing ``answering``/``parsing``/``result_id``).
    This loader migrates each row instead: the forbidden flat keys are
    dropped, ``answering_model``/``parsing_model`` strings
    (``"provider/model"``) become ModelIdentity objects,
    ``answering_replicate``/``parsing_replicate`` fold into ``replicate``,
    and a deterministic ``result_id`` is synthesized from question,
    models, timestamp, and replicate when absent.

    Non-legacy exports are delegated to ``load_result_set_from_json``
    unchanged.

    Args:
        file_path: Path to a legacy (v2.0/v2.1) or current JSON export.

    Returns:
        A validated result set built from the migrated rows.

    Raises:
        ValueError: If any row fails migration; the message names every
            failing row index with its reason.
    """
    with open(file_path, encoding="utf-8") as handle:
        data = json.load(handle)

    if not ResultsIOManager._is_legacy_export(data):
        return ResultsIOManager.load_result_set_from_json(file_path)

    rows = data["results"]
    metadata = data.get("metadata", {})
    if not isinstance(metadata, dict):
        raise ValueError("Invalid legacy export: metadata must be an object")

    results: list[VerificationResult] = []
    errors: list[str] = []
    seen_result_ids: set[str] = set()
    for index, row in enumerate(rows):
        try:
            results.append(ResultsIOManager._migrate_legacy_row(row, index, seen_result_ids))
        except Exception as e:
            errors.append(f"index {index}: {e}")

    if errors:
        raise ValueError(
            f"Legacy export migration failed for {len(errors)} of {len(rows)} rows: " + "; ".join(errors)
        )

    return VerificationResultSet(results=results, metadata=dict(metadata))

load_result_set_from_json staticmethod

load_result_set_from_json(
    file_path: Path,
) -> VerificationResultSet

Load and validate a complete verification result set from JSON.

Unlike the legacy dictionary loader, this method preserves run metadata and compact scenario execution records when the export contains them. Malformed rows raise instead of being skipped, making this the preferred analysis path for saved experiment outputs.

Parameters:

Name	Type	Description	Default
file_path	Path	Path to a standard JSON verification export.	required

Returns:

Type	Description
VerificationResultSet	A validated result set with optional scenario results and metadata.

Raises:

Type	Description
ValueError	If the JSON shape or any result row is invalid.

Source code in src/karenina/benchmark/core/results_io.py

@staticmethod
def load_result_set_from_json(file_path: Path) -> VerificationResultSet:
    """Load and validate a complete verification result set from JSON.

    Unlike the legacy dictionary loader, this method preserves run metadata
    and compact scenario execution records when the export contains them.
    Malformed rows raise instead of being skipped, making this the preferred
    analysis path for saved experiment outputs.

    Args:
        file_path: Path to a standard JSON verification export.

    Returns:
        A validated result set with optional scenario results and metadata.

    Raises:
        ValueError: If the JSON shape or any result row is invalid.
    """
    with open(file_path, encoding="utf-8") as handle:
        data = json.load(handle)

    metadata: dict[str, Any] = {}
    scenario_data: list[object] = []
    result_data: object

    if isinstance(data, list):
        result_data = data
    elif isinstance(data, dict):
        loaded_metadata = data.get("metadata")
        if loaded_metadata is not None:
            if not isinstance(loaded_metadata, dict):
                raise ValueError("Invalid results export: metadata must be an object")
            metadata = dict(loaded_metadata)

        if "results" in data:
            result_data = data["results"]
            loaded_scenarios = data.get("scenario_results", [])
            if not isinstance(loaded_scenarios, list):
                raise ValueError("Invalid results export: scenario_results must be an array")
            scenario_data = loaded_scenarios
        elif "runs" in data:
            runs = data["runs"]
            if not isinstance(runs, dict):
                raise ValueError("Invalid results export: runs must be an object")
            result_data = [row for rows in runs.values() for row in rows]
            metadata = {**metadata, "run_names": list(runs)}
        else:
            result_data = data
    else:
        raise ValueError("Invalid results export: top-level JSON must be an array or object")

    if isinstance(result_data, dict):
        result_items = list(result_data.values())
    elif isinstance(result_data, list):
        result_items = result_data
    else:
        raise ValueError("Invalid results export: results must be an array or object")

    results: list[VerificationResult] = []
    for index, item in enumerate(result_items):
        if not isinstance(item, dict):
            raise ValueError(f"Invalid results export: result {index} must be an object")
        item_copy = dict(item)
        item_copy.pop("row_index", None)
        try:
            results.append(VerificationResult.model_validate(item_copy))
        except Exception as e:
            raise ValueError(f"Invalid verification result at index {index}") from e

    scenario_results: list[ScenarioResultRecord] = []
    for index, item in enumerate(scenario_data):
        try:
            scenario_results.append(ScenarioResultRecord.model_validate(item))
        except Exception as e:
            raise ValueError(f"Invalid scenario result at index {index}") from e

    return VerificationResultSet(
        results=results,
        scenario_results=scenario_results or None,
        metadata=metadata,
    )

RowContext dataclass

Optional question and reference answer supplied to a row's judge.

Source code in src/karenina/benchmark/verification/post_hoc.py

@dataclass(frozen=True, slots=True)
class RowContext:
    """Optional question and reference answer supplied to a row's judge."""

    question: str = ""
    ground_truth: str | None = None

RunDirectory

Bases: BaseModel

Paths and mutable manifest lifecycle for one run directory.

Source code in src/karenina/benchmark/run_artifacts.py

class RunDirectory(BaseModel):
    """Paths and mutable manifest lifecycle for one run directory."""

    model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")

    path: Path
    manifest: RunManifest

    @property
    def manifest_path(self) -> Path:
        """Return the manifest file path."""
        return self.path / RUN_MANIFEST_FILENAME

    @property
    def results_path(self) -> Path:
        """Return the standard result export path."""
        return self.path / self.manifest.results_path

    @property
    def traces_path(self) -> Path:
        """Return the standard readable-trace directory."""
        return self.path / self.manifest.traces_path

    @property
    def workspaces_path(self) -> Path:
        """Return the standard workspace-sidecar directory."""
        return self.path / self.manifest.workspaces_path

    def update_status(
        self,
        status: RunStatus,
        *,
        now: datetime | None = None,
        error: BaseException | None = None,
    ) -> None:
        """Update and persist the lifecycle status.

        Args:
            status: New run status.
            now: Timestamp override for deterministic callers and tests.
            error: Failure recorded when status is ``failed``.
        """
        timestamp = _format_time(now or datetime.now(UTC))
        updates: dict[str, JsonValue | None] = {
            "status": status,
            "updated_at": timestamp,
            "completed_at": timestamp if status != "running" else None,
            "failure_type": type(error).__name__ if error is not None else None,
            "failure_message": str(error) if error is not None else None,
        }
        self.manifest = self.manifest.model_copy(update=updates)
        self.write_manifest()

    def write_manifest(self) -> None:
        """Persist the current manifest atomically."""
        atomic_write(
            self.manifest_path,
            json.dumps(self.manifest.model_dump(mode="json"), indent=2, sort_keys=True) + "\n",
        )

Attributes

manifest_path property

manifest_path: Path

Return the manifest file path.

results_path property

results_path: Path

Return the standard result export path.

traces_path property

traces_path: Path

Return the standard readable-trace directory.

workspaces_path property

workspaces_path: Path

Return the standard workspace-sidecar directory.

Functions

update_status

update_status(
    status: RunStatus,
    *,
    now: datetime | None = None,
    error: BaseException | None = None,
) -> None

Update and persist the lifecycle status.

Parameters:

Name	Type	Description	Default
status	RunStatus	New run status.	required
now	datetime | None	Timestamp override for deterministic callers and tests.	None
error	BaseException | None	Failure recorded when status is failed.	None

Source code in src/karenina/benchmark/run_artifacts.py

def update_status(
    self,
    status: RunStatus,
    *,
    now: datetime | None = None,
    error: BaseException | None = None,
) -> None:
    """Update and persist the lifecycle status.

    Args:
        status: New run status.
        now: Timestamp override for deterministic callers and tests.
        error: Failure recorded when status is ``failed``.
    """
    timestamp = _format_time(now or datetime.now(UTC))
    updates: dict[str, JsonValue | None] = {
        "status": status,
        "updated_at": timestamp,
        "completed_at": timestamp if status != "running" else None,
        "failure_type": type(error).__name__ if error is not None else None,
        "failure_message": str(error) if error is not None else None,
    }
    self.manifest = self.manifest.model_copy(update=updates)
    self.write_manifest()

write_manifest

write_manifest() -> None

Persist the current manifest atomically.

Source code in src/karenina/benchmark/run_artifacts.py

def write_manifest(self) -> None:
    """Persist the current manifest atomically."""
    atomic_write(
        self.manifest_path,
        json.dumps(self.manifest.model_dump(mode="json"), indent=2, sort_keys=True) + "\n",
    )

RunManifest

Bases: BaseModel

Machine-readable description of one verification run.

Source code in src/karenina/benchmark/run_artifacts.py

class RunManifest(BaseModel):
    """Machine-readable description of one verification run."""

    model_config = ConfigDict(extra="forbid")

    schema_version: str = RUN_MANIFEST_SCHEMA_VERSION
    status: RunStatus
    label: str
    run_name: str
    created_at: str
    updated_at: str
    completed_at: str | None = None
    benchmark_path: str
    selected_question_count: int | None = Field(default=None, ge=0)
    replicate_count: int = Field(ge=1)
    answering_models: list[ModelIdentity]
    parsing_models: list[ModelIdentity]
    configuration: dict[str, JsonValue]
    results_path: str = RESULTS_FILENAME
    traces_path: str = "traces"
    workspaces_path: str = "workspaces"
    failure_type: str | None = None
    failure_message: str | None = None

TemplateProgressEvent dataclass

Progress event emitted during template generation.

Consumers receive these via the progress_callback parameter. Job-level events (job_started, job_completed, job_cancelled) have question_id=None. Task-level events have the question_id set.

Source code in src/karenina/benchmark/benchmark_helpers.py

@dataclass
class TemplateProgressEvent:
    """Progress event emitted during template generation.

    Consumers receive these via the progress_callback parameter.
    Job-level events (job_started, job_completed, job_cancelled) have
    question_id=None. Task-level events have the question_id set.
    """

    event: Literal[
        "job_started",
        "task_started",
        "task_completed",
        "task_failed",
        "job_completed",
        "job_cancelled",
    ]
    question_id: str | None
    processed_count: int
    total_count: int
    successful_count: int
    failed_count: int
    percentage: float
    error: str | None
    template_code: str | None
    task_duration: float | None
    in_progress_questions: list[str] = field(default_factory=list)

VerificationConfig

Bases: BaseModel

Configuration for verification run with multiple models.

Source code in src/karenina/schemas/verification/config.py

class VerificationConfig(BaseModel):
    """Configuration for verification run with multiple models."""

    model_config = ConfigDict(extra="forbid")

    answering_models: list[ModelConfig] = Field(default_factory=list)
    parsing_models: list[ModelConfig]
    replicate_count: int = Field(default=1, ge=1)  # Number of times to run each test combination

    # Parsing-only mode (for TaskEval and similar use cases)
    parsing_only: bool = False  # When True, only parsing models are required

    # Rubric evaluation settings
    rubric_trait_names: list[str] | None = None  # Optional filter for specific traits
    rubric_evaluation_strategy: Literal["batch", "sequential"] | None = "batch"
    # - "batch": Evaluate all LLM traits in a single call (efficient, requires JSON output)
    # - "sequential": Evaluate traits one-by-one (reliable, more expensive)

    # Evaluation mode: determines which stages run in the verification pipeline
    evaluation_mode: Literal["template_only", "template_and_rubric", "rubric_only"] = "template_only"
    # - "template_only": Run template verification only (default behavior)
    # - "template_and_rubric": Run both template verification AND rubric evaluation
    # - "rubric_only": Skip template verification, only evaluate rubrics on raw LLM response

    @computed_field  # type: ignore[prop-decorator]
    @property
    def rubric_enabled(self) -> bool:
        """Whether rubric evaluation is enabled. Derived from evaluation_mode."""
        return self.evaluation_mode in ("template_and_rubric", "rubric_only")

    # Trace input control: determines what portion of MCP agent trace is passed to evaluation
    use_full_trace_for_template: bool = (
        False  # If True, pass full agent trace to template parsing; if False, extract only final AI message
    )
    use_full_trace_for_rubric: bool = (
        True  # If True, pass full agent trace to rubric evaluation; if False, extract only final AI message
    )
    # Note: The full trace is ALWAYS captured and stored in raw_llm_response regardless of these settings.
    # These flags only control what input is provided to the parsing/evaluation models.
    # If False and the trace doesn't end with an AI message, verification stage will fail with error.
    allow_partial_trace_scoring: bool = Field(
        default=True,
        description=(
            "Permit template parsing and scoring to run on responses that were "
            "truncated by answer-generation timeout. Timeout metadata is still "
            "preserved separately so partial scores do not hide agent failures."
        ),
    )

    # Abstention detection settings
    abstention_enabled: bool = False  # Enable abstention/refusal detection

    # Sufficiency detection settings
    sufficiency_enabled: bool = False  # Enable trace sufficiency detection

    # Extraction hint settings (controls whether hints are appended to the parsing prompt)
    include_extraction_hints: bool = True  # Include extraction hints in the parsing prompt

    # Embedding check settings (semantic similarity fallback)
    embedding_check_enabled: bool = False  # Enable semantic similarity fallback
    embedding_check_model: str = DEFAULT_EMBEDDING_MODEL  # SentenceTransformer model for embeddings
    embedding_check_threshold: float = Field(
        default=DEFAULT_EMBEDDING_THRESHOLD, ge=0.0, le=1.0
    )  # Similarity threshold (0.0-1.0)

    # Async execution settings
    async_enabled: bool = DEFAULT_ASYNC_ENABLED  # Enable parallel execution
    async_max_workers: int = Field(default=DEFAULT_ASYNC_MAX_WORKERS, ge=1)  # Number of parallel workers
    max_concurrent_requests: int | None = Field(
        default=None,
        ge=1,
        description="Global cap on concurrent LLM request setups across all "
        "workers, enforced process-wide by the GlobalLLMLimiter on both the "
        "QA and scenario paths. Gates the adapters' single-turn LLM calls "
        "(ainvoke, and the sync wrappers that dispatch to it), stream "
        "establishment (the cap bounds concurrent request setups, not "
        "concurrent open streams), parser-side direct API calls, and "
        "per-model-call requests inside langchain agent loops (via "
        "build_agent_middleware). The claude_tool, claude_agent_sdk, and "
        "langchain_deep_agents agent loops make their own internal model "
        "calls and are not gated (deep_agents is langchain-based but does "
        "not use build_agent_middleware). "
        "None means no global limit (concurrency bounded by max_workers only). "
        "Set to 16-64 for self-hosted inference servers (vLLM, SGLang).",
    )
    batch_timeout_seconds: float | None = Field(
        default=None,
        gt=0,
        description="Wall-clock ceiling in seconds for an entire parallel "
        "verification batch (QA and scenario paths). When the ceiling is "
        "reached, in-flight work is drained or cancelled and every result "
        "completed so far is kept. The QA path then raises "
        "VerificationBatchError carrying the partial results, and the "
        "scenario path returns the partial (results, errors) tuple with a "
        "timeout entry appended to the failed tasks. None (the default) "
        "disables the batch-level timeout: the run executes until all tasks "
        "finish, exactly as before. Sequential (async_enabled=False) runs "
        "are not bounded by this knob.",
    )

    # Task ordering strategy for queue generation
    task_ordering: Literal[
        "auto",
        "prefix_cache",
        "distribute_answerers",
        "generation_order",
        "random",
    ] = Field(
        default="auto",
        description="Task queue ordering strategy. "
        "'auto' (default) picks 'distribute_answerers' when answerers span "
        "more than one distinct identity, else 'prefix_cache'. "
        "'prefix_cache' groups by answering model for KV cache hits on a "
        "single endpoint. "
        "'distribute_answerers' round-robins across answerer identities to "
        "spread load over multiple inference endpoints. "
        "'generation_order' preserves template-first loop order. "
        "'random' shuffles tasks.",
    )

    # Per-answerer concurrency caps (enforced at task start by the executor)
    answerer_concurrency_limits: int | dict[str, int] | None = Field(
        default=None,
        description="Cap on concurrent tasks per answerer, enforced at task start. "
        "Pass an int to apply the same cap to every entry in answering_models "
        "(keyed by ModelConfig.id). Pass a dict keyed by ModelConfig.id for "
        "per-model caps; answerers not in the dict run without a cap. "
        "None disables caps (default).",
    )

    # Deep-judgment settings (multi-stage parsing with excerpts and reasoning)
    deep_judgment_mode: Literal["disabled", "reasoning_only", "full"] = "disabled"  # Template deep-judgment mode
    deep_judgment_max_excerpts_per_attribute: int = DEFAULT_DEEP_JUDGMENT_MAX_EXCERPTS  # Max excerpts per attribute
    deep_judgment_fuzzy_match_threshold: float = DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD  # Similarity threshold
    deep_judgment_excerpt_retry_attempts: int = DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS  # Retry attempts

    # Search-enhanced deep-judgment settings (validate excerpts against external evidence)
    deep_judgment_search_enabled: bool = False  # Enable search validation for excerpts
    deep_judgment_search_tool: str | Callable[..., Any] = "tavily"  # Search tool name or callable instance
    # Supported built-in tools: "tavily"
    # Can also pass any callable: (str | list[str]) -> (str | list[str])
    # Examples: langchain tools, MCP tools, custom functions

    # Deep-judgment rubric settings (global defaults for per-trait configuration)
    deep_judgment_rubric_max_excerpts_default: int = DEFAULT_RUBRIC_MAX_EXCERPTS  # Max excerpts per trait
    deep_judgment_rubric_fuzzy_match_threshold_default: float = DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD  # Fuzzy match
    deep_judgment_rubric_excerpt_retry_attempts_default: int = DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS  # Retry attempts
    deep_judgment_rubric_search_tool: str | Callable[..., Any] = (
        "tavily"  # Search tool for rubric hallucination detection
    )

    # Deep-judgment rubric configuration modes (NEW - runtime control of deep judgment)
    deep_judgment_rubric_mode: Literal["disabled", "enable_all", "use_checkpoint", "custom"] = "disabled"
    # - "disabled": Deep judgment is OFF (default, explicit)
    # - "enable_all": Apply deep judgment to all LLM traits (respects excerpt toggle)
    # - "use_checkpoint": Use deep judgment settings saved in checkpoint (if available)
    # - "custom": Use per-trait configuration from deep_judgment_rubric_config

    deep_judgment_rubric_global_excerpts: bool = True  # For enable_all mode: enable/disable excerpts globally
    deep_judgment_rubric_config: DeepJudgmentRubricCustomConfig | None = None  # For custom mode: per-trait config

    # Few-shot prompting settings
    few_shot_config: FewShotConfig | None = None  # New flexible configuration

    # Per-task-type prompt instructions (optional user-injected instructions for each pipeline stage)
    prompt_config: PromptConfig | None = None

    # Agentic parsing
    agentic_parsing: bool = Field(
        default=False,
        description=(
            "Enable agentic parsing (Stage 7b). The judge uses tools to "
            "independently verify artifacts before extracting structured data."
        ),
    )
    agentic_parsing_trigger: Literal["always", "dynamic"] = Field(
        default="always",
        description=(
            "When agentic_parsing=True, controls whether Stage 7b always runs "
            "the agentic investigation or first tries a dynamic final-message parse. "
            "'always' preserves existing behavior. 'dynamic' escalates to agentic "
            "investigation only when the final response is insufficient or malformed."
        ),
    )
    agentic_judge_context: Literal["workspace_only", "trace_and_workspace", "trace_only"] = Field(
        default="workspace_only",
        description=(
            "What context the investigation agent receives. "
            "'workspace_only': question + workspace path (maximum independence). "
            "'trace_and_workspace': answering agent trace + workspace path. "
            "'trace_only': equivalent to classical Stage 7a parsing."
        ),
    )
    agentic_parsing_max_turns: int = Field(
        default=15,
        ge=1,
        description="Max turns for the investigation agent.",
    )
    agentic_parsing_timeout: float = Field(
        default=120.0,
        ge=0.0,
        description="Timeout in seconds for the investigation agent.",
    )
    agentic_parsing_materialize_trace: bool = Field(
        default=False,
        description=(
            "Write the answering agent trace to a file for Stage 7b's "
            "investigation agent. This is for the coding-task agentic "
            "parsing path (Stage 7b). For scenario handover-level trace "
            "materialization, use the 'transcript_materialize' handover "
            "strategy on ScenarioEdge instead."
        ),
    )
    agentic_parsing_persist_trace: bool = Field(
        default=False,
        description=(
            "When True, the materialized trace file is kept after "
            "extraction. When False (default), it is cleaned up in a "
            "finally block after the stage runs. Ignored when "
            "agentic_parsing_materialize_trace is False."
        ),
    )

    # Agentic rubric evaluation
    agentic_rubric_strategy: Literal["individual", "shared"] = Field(
        "individual",
        description="How to evaluate agentic rubric traits. "
        "'individual': one agent session per trait (robust, isolated). "
        "'shared': one agent session for all traits (efficient, shared context).",
    )
    agentic_rubric_parallel: bool = Field(
        False,
        description="Enable parallel evaluation of agentic rubric traits. "
        "Only applies to 'individual' strategy. Each trait gets a concurrent agent session.",
    )

    # Workspace (workspace_root lives on Benchmark, not here)
    workspace_copy: bool = Field(
        default=True,
        description=(
            "When True, pre-existing question workspaces are copied to a "
            "sibling working directory before execution, protecting the "
            "original for re-runs. When False, the pipeline works directly "
            "in the original directory (destructive)."
        ),
    )
    workspace_cleanup: bool = Field(
        default=True,
        description=(
            "Whether to delete working copies after the run. Only applies to "
            "copied or auto-created workspaces, never to original source "
            "directories."
        ),
    )
    workspace_output_mode: Literal["none", "full", "produced"] = Field(
        default="none",
        description=(
            "Whether to capture runtime workspaces as sidecar artifacts. "
            "'none' preserves existing behavior. 'full' copies the complete "
            "final workspace. 'produced' copies new or content-modified files "
            "relative to the prepared workspace baseline."
        ),
    )
    workspace_output_dir: Path | None = Field(
        default=None,
        description="Directory where captured workspaces are written when workspace_output_mode is not 'none'.",
    )
    workspace_output_exclude_patterns: list[str] = Field(
        default_factory=list,
        description="Additional fnmatch-style exclude patterns for workspace capture.",
    )

    # Database storage settings
    db_config: Any | None = None  # DBConfig instance for automatic result persistence

    # HTTP request timeout (seconds) for all LLM calls in the pipeline.
    # Applies to answer generation, parsing, rubric evaluation, and guardrail calls.
    # Set to None to disable (use provider SDK defaults).
    request_timeout: float = 120.0

    # Scenario execution settings
    scenario_turn_limit: int = Field(default=20, ge=1)  # Max turns before forced termination in scenario execution

    # Replay layer (see karenina/replay).
    # replay_store is not serialized: it can hold large captured traces
    # and Pydantic cannot natively serialize an arbitrary in-memory store.
    replay_store: Any = Field(default=None, exclude=True)
    replay_parse_on_hydration_mismatch: Literal["fall_through", "strict"] = "fall_through"

    # Internal plumbing for Benchmark.extend_template: set of
    # (question_id, answering canonical_key, parsing canonical_key, replicate)
    # tuples already covered by prior_results. The batch runner skips these
    # during task-queue expansion so prior rows pass through verbatim.
    # Not part of the public surface; always None for standard verification runs.
    skip_triples: frozenset[tuple[str, str, str, int | None]] | None = Field(default=None, exclude=True, repr=False)

    @model_validator(mode="after")
    def _validate_custom_mode_has_config(self) -> "VerificationConfig":
        """Validate that custom mode has the required config.

        Raises:
            ValueError: If deep_judgment_rubric_mode is 'custom' but
                deep_judgment_rubric_config is None.
        """
        if self.deep_judgment_rubric_mode == "custom" and self.deep_judgment_rubric_config is None:
            raise ValueError("deep_judgment_rubric_config is required when deep_judgment_rubric_mode is 'custom'")
        return self

    @field_validator("db_config", mode="before")
    @classmethod
    def _validate_db_config(cls, v: Any) -> Any:
        """Validate that db_config is a DBConfig instance or None.

        Uses runtime import to avoid circular dependency with karenina.storage.

        Raises:
            TypeError: If value is not None and not a DBConfig instance.
        """
        if v is None:
            return v
        from karenina.storage.db_config import DBConfig

        if not isinstance(v, DBConfig):
            raise TypeError(f"db_config must be a DBConfig instance or None, got {type(v).__name__}")
        return v

    # Retry settings for transient errors and executor requeue
    retry_policy: RetryPolicy = Field(
        default_factory=RetryPolicy,
        description="Per-category retry budgets for transient LLM errors.",
    )
    custom_error_patterns: list[ErrorPatternConfig] = Field(
        default_factory=list,
        description="User-defined error patterns for the ErrorRegistry.",
    )
    max_requeue_count: int = Field(
        default=5,
        ge=1,
        description="Maximum times a task can be requeued in the parallel executor's "
        "IN_PROGRESS cache loop before generating the answer fresh.",
    )

    def __init__(self, **data: Any) -> None:
        """
        Initialize with environment variable support and default system prompts.

        Configuration precedence (highest to lowest):
        1. Explicit arguments (including preset values)
        2. Environment variables (only if set)
        3. Field defaults
        """
        # Read environment variables for embedding check settings (only if not explicitly provided AND env var is set)
        if "embedding_check_enabled" not in data:
            env_val = os.getenv("EMBEDDING_CHECK")
            if env_val is not None:
                data["embedding_check_enabled"] = env_val.lower() in ("true", "1", "yes")
            # else: let Pydantic use field default (False)

        if "embedding_check_model" not in data:
            env_val = os.getenv("EMBEDDING_CHECK_MODEL")
            if env_val is not None:
                data["embedding_check_model"] = env_val
            # else: let Pydantic use field default (DEFAULT_EMBEDDING_MODEL)

        if "embedding_check_threshold" not in data:
            env_val = os.getenv("EMBEDDING_CHECK_THRESHOLD")
            if env_val is not None:
                # Invalid env var value will let Pydantic use field default (0.85)
                with contextlib.suppress(ValueError):
                    data["embedding_check_threshold"] = float(env_val)
            # else: let Pydantic use field default (DEFAULT_EMBEDDING_THRESHOLD)

        # Read environment variables for async execution settings (only if not explicitly provided AND env var is set)
        if "async_enabled" not in data:
            env_val = os.getenv("KARENINA_ASYNC_ENABLED")
            if env_val is not None:
                data["async_enabled"] = env_val.lower() in ("true", "1", "yes")
            # else: let Pydantic use field default (DEFAULT_ASYNC_ENABLED)

        if "async_max_workers" not in data:
            env_val = os.getenv("KARENINA_ASYNC_MAX_WORKERS")
            if env_val is not None:
                # Invalid env var value will let Pydantic use field default (2)
                with contextlib.suppress(ValueError):
                    data["async_max_workers"] = int(env_val)
            # else: let Pydantic use field default (DEFAULT_ASYNC_MAX_WORKERS)

        if "max_concurrent_requests" not in data:
            env_val = os.getenv("KARENINA_MAX_CONCURRENT_LLM_REQUESTS")
            if env_val is not None:
                with contextlib.suppress(ValueError):
                    data["max_concurrent_requests"] = int(env_val)

        # Apply default system prompts to models that don't have one.
        # Deep-copy ModelConfig instances to avoid mutating shared objects.
        if "answering_models" in data:
            data["answering_models"] = [
                m.model_copy(update={"system_prompt": DEFAULT_ANSWERING_SYSTEM_PROMPT})
                if isinstance(m, ModelConfig) and not m.system_prompt
                else (
                    {**m, "system_prompt": DEFAULT_ANSWERING_SYSTEM_PROMPT}
                    if isinstance(m, dict) and not m.get("system_prompt")
                    else m
                )
                for m in data["answering_models"]
            ]

        # Strip rubric_enabled from input: now derived from evaluation_mode
        data.pop("rubric_enabled", None)

        # Strip deep_judgment_rubric_search_enabled: not a declared field,
        # but injected by from_overrides() and some CLI callers.
        data.pop("deep_judgment_rubric_search_enabled", None)

        # Wire instruction shortcuts into prompt_config (pop before super().__init__)
        shortcut_mapping = {
            "generation": data.pop("generation_instructions", None),
            "parsing": data.pop("parsing_instructions", None),
            "abstention_detection": data.pop("abstention_detection_instructions", None),
            "sufficiency_detection": data.pop("sufficiency_detection_instructions", None),
            "rubric_evaluation": data.pop("rubric_evaluation_instructions", None),
            "agentic_parsing": data.pop("agentic_parsing_instructions", None),
            "deep_judgment": data.pop("deep_judgment_instructions", None),
        }
        active_shortcuts = {k: v for k, v in shortcut_mapping.items() if v is not None}
        if active_shortcuts:
            pc = data.get("prompt_config") or PromptConfig()
            if isinstance(pc, dict):
                pc = PromptConfig(**pc)
            updates = {k: v for k, v in active_shortcuts.items() if getattr(pc, k) is None}
            if updates:
                pc = pc.model_copy(update=updates)
            data["prompt_config"] = pc

        super().__init__(**data)

        # Validate configuration after initialization
        self._validate_config()

    def _validate_config(self) -> None:
        """
        Validate configuration, especially for rubric-enabled scenarios.

        Validates that:
        - At least one parsing model is configured
        - At least one answering model is configured (unless parsing_only=True)
        - Required fields are present for each model
        - Model provider is provided for interfaces that require it
        - Rubric-specific requirements are met when enabled

        Raises:
            ValueError: If any validation rule fails
        """
        # Check that we have at least one parsing model (always required)
        if not self.parsing_models:
            raise ValueError("At least one parsing model must be configured")

        # Check answering models only if not in parsing-only mode
        if not self.parsing_only and not self.answering_models:
            raise ValueError("At least one answering model must be configured (unless parsing_only=True)")

        # Validate model configurations
        # Note: Basic model validation (model_name, model_provider) is also done by
        # the adapter factory at runtime, but we validate here too for early failure.
        from karenina.adapters.registry import AdapterRegistry

        for model in self.answering_models + self.parsing_models:
            if not model.model_name:
                raise ValueError(f"Model name is required in model configuration (model: {model.id})")
            # Model provider requirement is defined per-adapter via AdapterSpec.requires_provider
            spec = AdapterRegistry.get_spec(model.interface)
            if spec is not None and spec.requires_provider and not model.model_provider:
                raise ValueError(f"Model provider is required for interface '{model.interface}'. (model: {model.id})")

        # System prompt is required for answering models (not validated by factory).
        # Parsing models do not require a system_prompt; their prompt is assembled
        # by the pipeline at runtime.
        for model in self.answering_models:
            if not model.system_prompt:
                raise ValueError(f"System prompt is required for answering model {model.id}")

        # Additional validation for rubric-enabled scenarios
        if self.rubric_enabled and not self.parsing_models:
            raise ValueError("Parsing models are required when rubric evaluation is enabled")

        # Additional validation for few-shot prompting scenarios
        if self.few_shot_config is not None and self.few_shot_config.source != "disabled":
            if self.few_shot_config.pool_mode == "k-shot" and self.few_shot_config.pool_k < 1:
                raise ValueError("Pool few-shot k value must be at least 1 when using k-shot mode")

            # Validate question-specific k values
            for question_id, question_config in self.few_shot_config.question_configs.items():
                if question_config.mode == "k-shot" and question_config.k is not None and question_config.k < 1:
                    raise ValueError(
                        f"Question {question_id} few-shot k value must be at least 1 when using k-shot mode"
                    )

        # Validate incompatible deep-judgment combinations
        if self.deep_judgment_mode == "reasoning_only" and self.deep_judgment_search_enabled:
            raise ValueError(
                "deep_judgment_search_enabled=True is incompatible with "
                "deep_judgment_mode='reasoning_only'. Search requires excerpt "
                "extraction. Use deep_judgment_mode='full' for search."
            )

        # Additional validation for search-enhanced deep-judgment
        if self.deep_judgment_search_enabled:
            # Validate search tool
            if isinstance(self.deep_judgment_search_tool, str):
                # Check if it's a supported built-in tool
                supported_tools = ["tavily"]
                if self.deep_judgment_search_tool.lower() not in supported_tools:
                    raise ValueError(
                        f"Unknown search tool: '{self.deep_judgment_search_tool}'. Supported tools: {supported_tools}"
                    )
            elif not callable(self.deep_judgment_search_tool):
                raise ValueError(
                    "Search tool must be either a supported tool name string "
                    "or a callable with signature (str | list[str]) -> (str | list[str])"
                )

        # Agentic parsing validation
        if self.agentic_parsing_trigger == "dynamic" and not self.agentic_parsing:
            raise ValueError("agentic_parsing_trigger='dynamic' requires agentic_parsing=True")

        if self.agentic_parsing:
            # Check parsing model interface supports AgentPort
            from karenina.adapters.registry import AdapterRegistry

            for pm in self.parsing_models:
                spec = AdapterRegistry.get_spec(pm.interface)
                if spec is None or spec.agent_tier != "deep_agent":
                    tier = spec.agent_tier if spec else "unknown"
                    raise ValueError(
                        f"agentic_parsing=True requires an interface with "
                        f"agent_tier='deep_agent', but '{pm.interface}' has "
                        f"agent_tier='{tier}'. Use 'claude_agent_sdk' or "
                        f"'langchain_deep_agents' instead."
                    )

            # Agentic parsing is not supported in rubric_only mode
            if self.evaluation_mode == "rubric_only":
                raise ValueError(
                    "agentic_parsing=True is not supported with "
                    "evaluation_mode='rubric_only'. Use 'template_only' or "
                    "'template_and_rubric'."
                )

            # Warn about trace_only being equivalent to Stage 7a
            if self.agentic_judge_context == "trace_only":
                logger.warning(
                    "agentic_parsing=True with agentic_judge_context='trace_only' "
                    "is equivalent to classical parsing (Stage 7a)."
                )

            if self.agentic_parsing_trigger == "dynamic" and self.agentic_judge_context == "trace_only":
                logger.warning(
                    "agentic_parsing_trigger='dynamic' with agentic_judge_context='trace_only' "
                    "can only escalate to the same trace context that was already judged insufficient. "
                    "Use agentic_judge_context='workspace_only' or 'trace_and_workspace' for workspace recovery."
                )

            # materialize_trace needs a trace in the context
            if self.agentic_parsing_materialize_trace and self.agentic_judge_context == "workspace_only":
                raise ValueError(
                    "agentic_parsing_materialize_trace=True requires a trace "
                    "to materialize, but agentic_judge_context='workspace_only' "
                    "excludes the trace. Use 'trace_only' or 'trace_and_workspace'."
                )

    def __repr__(self) -> str:
        """
        Return detailed string representation for debugging/inspection.

        Shows key configuration settings including models, execution parameters,
        and enabled features in a human-readable multi-line format.
        """
        lines = ["VerificationConfig("]

        # === MODELS ===
        lines.append("  === MODELS ===")

        # Answering models
        if self.answering_models:
            lines.append(f"  Answering ({len(self.answering_models)}):")
            for model in self.answering_models:
                provider = model.model_provider or "none"
                lines.append(
                    f"    - {model.model_name} ({provider}) [temp={model.temperature}, interface={model.interface}]"
                )
        else:
            lines.append("  Answering: none")

        # Parsing models
        lines.append(f"  Parsing ({len(self.parsing_models)}):")
        for model in self.parsing_models:
            provider = model.model_provider or "none"
            lines.append(
                f"    - {model.model_name} ({provider}) [temp={model.temperature}, interface={model.interface}]"
            )

        # === EXECUTION ===
        lines.append("")
        lines.append("  === EXECUTION ===")
        lines.append(f"  Replicates: {self.replicate_count}")
        lines.append(f"  Async: {self.async_enabled}")
        if self.async_enabled:
            lines.append(f"    └─ workers: {self.async_max_workers}")
        if self.parsing_only:
            lines.append("  Parsing Only: True")
        lines.append(f"  Evaluation Mode: {self.evaluation_mode}")
        lines.append(f"  Rubric Evaluation Strategy: {self.rubric_evaluation_strategy}")

        # === FEATURES ===
        lines.append("")
        lines.append("  === FEATURES ===")
        features_shown = False

        # Rubric - just enabled/disabled status with optional trait selection
        if self.rubric_enabled:
            features_shown = True
            trait_info = ""
            if self.rubric_trait_names:
                trait_info = f" ({len(self.rubric_trait_names)} traits selected)"
            lines.append(f"  Rubric: enabled{trait_info}")
        else:
            lines.append("  Rubric: disabled")

        # Deep Judgment - Template
        if self.deep_judgment_mode != "disabled":
            features_shown = True
            lines.append(
                f"  Deep Judgment (Template): mode={self.deep_judgment_mode}, "
                f"max_excerpts={self.deep_judgment_max_excerpts_per_attribute}, "
                f"fuzzy_threshold={self.deep_judgment_fuzzy_match_threshold}"
            )
            if self.deep_judgment_search_enabled:
                search_tool = self.deep_judgment_search_tool
                if callable(search_tool):
                    search_tool = "<custom_callable>"
                lines.append(f"    └─ search: {search_tool}")

        # Deep Judgment - Rubric
        if self.deep_judgment_rubric_mode != "disabled":
            features_shown = True
            lines.append(
                f"  Deep Judgment (Rubric): mode={self.deep_judgment_rubric_mode}, "
                f"global_excerpts={self.deep_judgment_rubric_global_excerpts}"
            )
            # Warning about sequential evaluation
            lines.append("    ⚠️  Deep judgment traits are ALWAYS evaluated sequentially (one-by-one)")
            if self.deep_judgment_rubric_mode == "custom" and self.deep_judgment_rubric_config:
                global_traits = self.deep_judgment_rubric_config.global_traits
                question_configs = self.deep_judgment_rubric_config.question_specific
                lines.append(f"    └─ {len(global_traits)} global traits, {len(question_configs)} question configs")

        # Abstention
        if self.abstention_enabled:
            features_shown = True
            lines.append("  Abstention: enabled")

        # Sufficiency
        if self.sufficiency_enabled:
            features_shown = True
            lines.append("  Sufficiency: enabled")

        # Embedding Check
        if self.embedding_check_enabled:
            features_shown = True
            lines.append(
                f"  Embedding Check: model={self.embedding_check_model}, threshold={self.embedding_check_threshold}"
            )

        # Few-Shot
        few_shot_config = self.get_few_shot_config()
        if few_shot_config and few_shot_config.source != "disabled":
            features_shown = True
            lines.append(f"  Few-Shot: source={few_shot_config.source}, pool_mode={few_shot_config.pool_mode}")
            if few_shot_config.pool_mode == "k-shot":
                lines.append(f"    └─ pool_k={few_shot_config.pool_k}")
            if few_shot_config.question_configs:
                lines.append(f"    └─ {len(few_shot_config.question_configs)} question configs")

        if not features_shown:
            lines.append("  (none enabled)")

        lines.append(")")

        return "\n".join(lines)

    def __str__(self) -> str:
        """String representation (same as repr for developer-friendly output)."""
        return self.__repr__()

    def get_few_shot_config(self) -> FewShotConfig | None:
        """
        Get the FewShotConfig for this verification run.

        Returns:
            The FewShotConfig to use, or None if few-shot is disabled
        """
        return self.few_shot_config

    def is_few_shot_enabled(self) -> bool:
        """
        Check if few-shot prompting is enabled.

        Returns:
            True if few-shot is enabled
        """
        config = self.get_few_shot_config()
        return config is not None and config.source != "disabled"

    # ===== Preset Utility Class Methods =====
    # These methods delegate to config_presets module for backward compatibility.

    @classmethod
    def sanitize_model_config(cls, model: dict[str, Any]) -> dict[str, Any]:
        """Sanitize model configuration. Delegates to config_presets.sanitize_model_config."""
        return sanitize_model_config(model)

    @classmethod
    def sanitize_preset_name(cls, name: str) -> str:
        """Convert preset name to safe filename. Delegates to config_presets.sanitize_preset_name."""
        return sanitize_preset_name(name)

    @classmethod
    def validate_preset_metadata(cls, name: str, description: str | None = None) -> None:
        """Validate preset metadata. Delegates to config_presets.validate_preset_metadata."""
        return validate_preset_metadata(name, description)

    @classmethod
    def create_preset_structure(
        cls,
        preset_id: str,
        name: str,
        description: str | None,
        config_dict: dict[str, Any],
        created_at: str,
        updated_at: str,
    ) -> dict[str, Any]:
        """Create preset structure. Delegates to config_presets.create_preset_structure."""
        return create_preset_structure(preset_id, name, description, config_dict, created_at, updated_at)

    def save_preset(
        self,
        name: str,
        description: str | None = None,
        presets_dir: Path | None = None,
    ) -> dict[str, Any]:
        """Save this config as a preset file. Delegates to config_presets.save_preset."""
        return save_preset(self, name, description, presets_dir)

    @classmethod
    def from_preset(cls, filepath: Path) -> "VerificationConfig":
        """Load a VerificationConfig from a preset file. Delegates to config_presets.load_preset."""
        return load_preset(filepath)

    @classmethod
    def from_overrides(
        cls,
        base: "VerificationConfig | None" = None,
        *,
        # Model configuration
        answering_model: str | None = None,
        answering_provider: str | None = None,
        answering_id: str | None = None,
        answering_interface: str | None = None,
        parsing_model: str | None = None,
        parsing_provider: str | None = None,
        parsing_id: str | None = None,
        parsing_interface: str | None = None,
        temperature: float | None = None,
        manual_traces: Any | None = None,
        # Execution settings
        replicate_count: int | None = None,
        # Feature flags
        abstention: bool | None = None,
        sufficiency: bool | None = None,
        embedding_check: bool | None = None,
        deep_judgment_mode: str | None = None,
        # Evaluation settings
        evaluation_mode: str | None = None,
        embedding_threshold: float | None = None,
        embedding_model: str | None = None,
        async_execution: bool | None = None,
        async_workers: int | None = None,
        max_concurrent_requests: int | None = None,
        # Trace filtering
        use_full_trace_for_template: bool | None = None,
        use_full_trace_for_rubric: bool | None = None,
        # Deep judgment rubric settings
        deep_judgment_rubric_mode: str | None = None,
        deep_judgment_rubric_excerpts: bool | None = None,
        deep_judgment_rubric_max_excerpts: int | None = None,
        deep_judgment_rubric_fuzzy_threshold: float | None = None,
        deep_judgment_rubric_retry_attempts: int | None = None,
        deep_judgment_rubric_search: bool | None = None,
        deep_judgment_rubric_search_tool: str | None = None,
        deep_judgment_rubric_config: DeepJudgmentRubricCustomConfig | dict[str, Any] | None = None,
        workspace_output_mode: Literal["none", "full", "produced"] | None = None,
        workspace_output_dir: Path | None = None,
        workspace_output_exclude_patterns: list[str] | None = None,
    ) -> "VerificationConfig":
        """
        Create a VerificationConfig by applying overrides to an optional base config.

        Implements the hierarchy: overrides > base config > defaults.
        Parameters set to None are not applied (base or default value is preserved).

        This is the canonical way to construct a VerificationConfig with selective
        overrides, usable by CLI, server, and programmatic callers.

        Args:
            base: Optional base config (e.g., from a preset). If None, starts from defaults.
            answering_model: Override for the answering model name.
            answering_provider: Override for the answering model provider.
            answering_id: Override for the answering model identifier.
            answering_interface: Override for the answering adapter interface.
            parsing_model: Override for the parsing model name.
            parsing_provider: Override for the parsing model provider.
            parsing_id: Override for the parsing model identifier.
            parsing_interface: Override for the parsing adapter interface.
            temperature: Override for the LLM temperature.
            manual_traces: Override for manual traces data.
            replicate_count: Override for the number of replicates.
            abstention: Override for abstention detection flag.
            sufficiency: Override for sufficiency checking flag.
            embedding_check: Override for embedding check flag.
            deep_judgment_mode: Override for template deep judgment mode.
            evaluation_mode: Override for evaluation mode.
            embedding_threshold: Override for embedding similarity threshold.
            embedding_model: Override for embedding model name.
            async_execution: Override for async execution flag.
            async_workers: Override for number of async workers.
            max_concurrent_requests: Override for global concurrent LLM request cap.
            use_full_trace_for_template: Override for full trace template flag.
            use_full_trace_for_rubric: Override for full trace rubric flag.
            deep_judgment_rubric_mode: Override for deep judgment rubric mode.
            deep_judgment_rubric_excerpts: Override for rubric excerpts flag.
            deep_judgment_rubric_max_excerpts: Override for max rubric excerpts.
            deep_judgment_rubric_fuzzy_threshold: Override for rubric fuzzy threshold.
            deep_judgment_rubric_retry_attempts: Override for rubric retry attempts.
            deep_judgment_rubric_search: Override for rubric search flag.
            deep_judgment_rubric_search_tool: Override for rubric search tool.
            deep_judgment_rubric_config: Override for rubric config dict.
            workspace_output_mode: Override for workspace sidecar capture mode.
            workspace_output_dir: Override for workspace sidecar output directory.
            workspace_output_exclude_patterns: Additional workspace sidecar exclude patterns.

        Returns:
            A new VerificationConfig with overrides applied.
        """
        # Start with base config dump or empty dict
        config_dict: dict[str, Any] = base.model_dump() if base else {}

        # --- Scalar overrides (None = don't override) ---

        # Replicate count
        if replicate_count is not None:
            config_dict["replicate_count"] = replicate_count
        elif not base:
            config_dict["replicate_count"] = 1

        # Feature flags
        if abstention is not None:
            config_dict["abstention_enabled"] = abstention
        if sufficiency is not None:
            config_dict["sufficiency_enabled"] = sufficiency
        if embedding_check is not None:
            config_dict["embedding_check_enabled"] = embedding_check
        if deep_judgment_mode is not None:
            config_dict["deep_judgment_mode"] = deep_judgment_mode

        # Evaluation settings
        if evaluation_mode is not None:
            config_dict["evaluation_mode"] = evaluation_mode
        if embedding_threshold is not None:
            config_dict["embedding_check_threshold"] = embedding_threshold
        if embedding_model is not None:
            config_dict["embedding_check_model"] = embedding_model
        if async_execution is not None:
            config_dict["async_enabled"] = async_execution
        if async_workers is not None:
            config_dict["async_max_workers"] = async_workers
        if max_concurrent_requests is not None:
            config_dict["max_concurrent_requests"] = max_concurrent_requests

        # Trace filtering
        if use_full_trace_for_template is not None:
            config_dict["use_full_trace_for_template"] = use_full_trace_for_template
        if use_full_trace_for_rubric is not None:
            config_dict["use_full_trace_for_rubric"] = use_full_trace_for_rubric

        # Deep judgment rubric settings
        if deep_judgment_rubric_mode is not None:
            config_dict["deep_judgment_rubric_mode"] = deep_judgment_rubric_mode
        if deep_judgment_rubric_excerpts is not None:
            config_dict["deep_judgment_rubric_global_excerpts"] = deep_judgment_rubric_excerpts
        if deep_judgment_rubric_max_excerpts is not None:
            config_dict["deep_judgment_rubric_max_excerpts_default"] = deep_judgment_rubric_max_excerpts
        if deep_judgment_rubric_fuzzy_threshold is not None:
            config_dict["deep_judgment_rubric_fuzzy_match_threshold_default"] = deep_judgment_rubric_fuzzy_threshold
        if deep_judgment_rubric_retry_attempts is not None:
            config_dict["deep_judgment_rubric_excerpt_retry_attempts_default"] = deep_judgment_rubric_retry_attempts
        if deep_judgment_rubric_search is not None:
            config_dict["deep_judgment_rubric_search_enabled"] = deep_judgment_rubric_search
        if deep_judgment_rubric_search_tool is not None:
            config_dict["deep_judgment_rubric_search_tool"] = deep_judgment_rubric_search_tool
        if deep_judgment_rubric_config is not None:
            config_dict["deep_judgment_rubric_config"] = deep_judgment_rubric_config

        # Workspace output capture
        if workspace_output_mode is not None:
            config_dict["workspace_output_mode"] = workspace_output_mode
        if workspace_output_dir is not None:
            config_dict["workspace_output_dir"] = workspace_output_dir
        if workspace_output_exclude_patterns is not None:
            config_dict["workspace_output_exclude_patterns"] = workspace_output_exclude_patterns

        # --- Model configuration ---
        # Determine the unified interface (answering and parsing may differ)
        ans_interface = answering_interface
        par_interface = parsing_interface
        # If only a single 'interface' concept was provided via answering_interface,
        # it's already split by the caller. No implicit sharing here.

        answering_has_overrides = any(
            [
                answering_model is not None,
                answering_provider is not None,
                ans_interface is not None,
            ]
        )

        parsing_has_overrides = any(
            [
                parsing_model is not None,
                parsing_provider is not None,
                par_interface is not None,
            ]
        )

        if answering_has_overrides:
            config_dict["answering_models"] = [
                cls._build_model_config_dict(
                    base_models=base.answering_models if base else None,
                    model_name=answering_model,
                    provider=answering_provider,
                    model_id=answering_id,
                    temperature=temperature,
                    interface=ans_interface,
                    manual_traces=manual_traces,
                    default_model="gpt-4.1-mini",
                    default_provider="openai",
                    default_interface="langchain",
                )
            ]
        elif manual_traces is not None:
            # Manual interface requested via manual_traces without explicit model overrides
            config_dict["answering_models"] = [ModelConfig(interface="manual", manual_traces=manual_traces)]

        if parsing_has_overrides:
            config_dict["parsing_models"] = [
                cls._build_model_config_dict(
                    base_models=base.parsing_models if base else None,
                    model_name=parsing_model,
                    provider=parsing_provider,
                    model_id=parsing_id,
                    temperature=temperature,
                    interface=par_interface,
                    manual_traces=None,  # Parsing model never uses manual interface
                    default_model="gpt-4.1-mini",
                    default_provider="openai",
                    default_interface="langchain",
                )
            ]

        return cls(**config_dict)

    @classmethod
    def _build_model_config_dict(
        cls,
        *,
        base_models: list[ModelConfig] | None,
        model_name: str | None,
        provider: str | None,
        model_id: str | None,
        temperature: float | None,
        interface: str | None,
        manual_traces: Any | None,
        default_model: str,
        default_provider: str,
        default_interface: str,
    ) -> ModelConfig:
        """
        Build a ModelConfig by applying overrides to an optional base model.

        If base_models is provided, uses the first model as the starting point and
        applies only non-None overrides. If no base, constructs from scratch with defaults.

        Returns:
            A new ModelConfig instance.
        """
        if interface == "manual" and manual_traces is not None:
            return ModelConfig(interface="manual", manual_traces=manual_traces)

        if base_models:
            # Start from base model, apply overrides
            base_model = base_models[0].model_dump()
            if model_name is not None:
                base_model["model_name"] = model_name
            if provider is not None:
                base_model["model_provider"] = provider
            if model_id is not None:
                base_model["id"] = model_id
            if temperature is not None:
                base_model["temperature"] = temperature
            if interface is not None:
                base_model["interface"] = interface
            return ModelConfig(**base_model)

        # No base: build from scratch
        final_interface = interface or default_interface

        return ModelConfig(
            model_name=model_name or default_model,
            model_provider=provider or default_provider,
            interface=final_interface,
            temperature=temperature if temperature is not None else 0.1,
            id=model_id,
        )

Attributes

rubric_enabled property

rubric_enabled: bool

Whether rubric evaluation is enabled. Derived from evaluation_mode.

Functions

__init__

__init__(**data: Any) -> None

Configuration precedence (highest to lowest): 1. Explicit arguments (including preset values) 2. Environment variables (only if set) 3. Field defaults

Source code in src/karenina/schemas/verification/config.py

def __init__(self, **data: Any) -> None:
    """
    Initialize with environment variable support and default system prompts.

    Configuration precedence (highest to lowest):
    1. Explicit arguments (including preset values)
    2. Environment variables (only if set)
    3. Field defaults
    """
    # Read environment variables for embedding check settings (only if not explicitly provided AND env var is set)
    if "embedding_check_enabled" not in data:
        env_val = os.getenv("EMBEDDING_CHECK")
        if env_val is not None:
            data["embedding_check_enabled"] = env_val.lower() in ("true", "1", "yes")
        # else: let Pydantic use field default (False)

    if "embedding_check_model" not in data:
        env_val = os.getenv("EMBEDDING_CHECK_MODEL")
        if env_val is not None:
            data["embedding_check_model"] = env_val
        # else: let Pydantic use field default (DEFAULT_EMBEDDING_MODEL)

    if "embedding_check_threshold" not in data:
        env_val = os.getenv("EMBEDDING_CHECK_THRESHOLD")
        if env_val is not None:
            # Invalid env var value will let Pydantic use field default (0.85)
            with contextlib.suppress(ValueError):
                data["embedding_check_threshold"] = float(env_val)
        # else: let Pydantic use field default (DEFAULT_EMBEDDING_THRESHOLD)

    # Read environment variables for async execution settings (only if not explicitly provided AND env var is set)
    if "async_enabled" not in data:
        env_val = os.getenv("KARENINA_ASYNC_ENABLED")
        if env_val is not None:
            data["async_enabled"] = env_val.lower() in ("true", "1", "yes")
        # else: let Pydantic use field default (DEFAULT_ASYNC_ENABLED)

    if "async_max_workers" not in data:
        env_val = os.getenv("KARENINA_ASYNC_MAX_WORKERS")
        if env_val is not None:
            # Invalid env var value will let Pydantic use field default (2)
            with contextlib.suppress(ValueError):
                data["async_max_workers"] = int(env_val)
        # else: let Pydantic use field default (DEFAULT_ASYNC_MAX_WORKERS)

    if "max_concurrent_requests" not in data:
        env_val = os.getenv("KARENINA_MAX_CONCURRENT_LLM_REQUESTS")
        if env_val is not None:
            with contextlib.suppress(ValueError):
                data["max_concurrent_requests"] = int(env_val)

    # Apply default system prompts to models that don't have one.
    # Deep-copy ModelConfig instances to avoid mutating shared objects.
    if "answering_models" in data:
        data["answering_models"] = [
            m.model_copy(update={"system_prompt": DEFAULT_ANSWERING_SYSTEM_PROMPT})
            if isinstance(m, ModelConfig) and not m.system_prompt
            else (
                {**m, "system_prompt": DEFAULT_ANSWERING_SYSTEM_PROMPT}
                if isinstance(m, dict) and not m.get("system_prompt")
                else m
            )
            for m in data["answering_models"]
        ]

    # Strip rubric_enabled from input: now derived from evaluation_mode
    data.pop("rubric_enabled", None)

    # Strip deep_judgment_rubric_search_enabled: not a declared field,
    # but injected by from_overrides() and some CLI callers.
    data.pop("deep_judgment_rubric_search_enabled", None)

    # Wire instruction shortcuts into prompt_config (pop before super().__init__)
    shortcut_mapping = {
        "generation": data.pop("generation_instructions", None),
        "parsing": data.pop("parsing_instructions", None),
        "abstention_detection": data.pop("abstention_detection_instructions", None),
        "sufficiency_detection": data.pop("sufficiency_detection_instructions", None),
        "rubric_evaluation": data.pop("rubric_evaluation_instructions", None),
        "agentic_parsing": data.pop("agentic_parsing_instructions", None),
        "deep_judgment": data.pop("deep_judgment_instructions", None),
    }
    active_shortcuts = {k: v for k, v in shortcut_mapping.items() if v is not None}
    if active_shortcuts:
        pc = data.get("prompt_config") or PromptConfig()
        if isinstance(pc, dict):
            pc = PromptConfig(**pc)
        updates = {k: v for k, v in active_shortcuts.items() if getattr(pc, k) is None}
        if updates:
            pc = pc.model_copy(update=updates)
        data["prompt_config"] = pc

    super().__init__(**data)

    # Validate configuration after initialization
    self._validate_config()

create_preset_structure classmethod

create_preset_structure(
    preset_id: str,
    name: str,
    description: str | None,
    config_dict: dict[str, Any],
    created_at: str,
    updated_at: str,
) -> dict[str, Any]

Create preset structure. Delegates to config_presets.create_preset_structure.

Source code in src/karenina/schemas/verification/config.py

@classmethod
def create_preset_structure(
    cls,
    preset_id: str,
    name: str,
    description: str | None,
    config_dict: dict[str, Any],
    created_at: str,
    updated_at: str,
) -> dict[str, Any]:
    """Create preset structure. Delegates to config_presets.create_preset_structure."""
    return create_preset_structure(preset_id, name, description, config_dict, created_at, updated_at)

from_overrides classmethod

from_overrides(
    base: VerificationConfig | None = None,
    *,
    answering_model: str | None = None,
    answering_provider: str | None = None,
    answering_id: str | None = None,
    answering_interface: str | None = None,
    parsing_model: str | None = None,
    parsing_provider: str | None = None,
    parsing_id: str | None = None,
    parsing_interface: str | None = None,
    temperature: float | None = None,
    manual_traces: Any | None = None,
    replicate_count: int | None = None,
    abstention: bool | None = None,
    sufficiency: bool | None = None,
    embedding_check: bool | None = None,
    deep_judgment_mode: str | None = None,
    evaluation_mode: str | None = None,
    embedding_threshold: float | None = None,
    embedding_model: str | None = None,
    async_execution: bool | None = None,
    async_workers: int | None = None,
    max_concurrent_requests: int | None = None,
    use_full_trace_for_template: bool | None = None,
    use_full_trace_for_rubric: bool | None = None,
    deep_judgment_rubric_mode: str | None = None,
    deep_judgment_rubric_excerpts: bool | None = None,
    deep_judgment_rubric_max_excerpts: int | None = None,
    deep_judgment_rubric_fuzzy_threshold: float
    | None = None,
    deep_judgment_rubric_retry_attempts: int | None = None,
    deep_judgment_rubric_search: bool | None = None,
    deep_judgment_rubric_search_tool: str | None = None,
    deep_judgment_rubric_config: DeepJudgmentRubricCustomConfig
    | dict[str, Any]
    | None = None,
    workspace_output_mode: Literal[
        "none", "full", "produced"
    ]
    | None = None,
    workspace_output_dir: Path | None = None,
    workspace_output_exclude_patterns: list[str]
    | None = None,
) -> VerificationConfig

Create a VerificationConfig by applying overrides to an optional base config.

Implements the hierarchy: overrides > base config > defaults. Parameters set to None are not applied (base or default value is preserved).

This is the canonical way to construct a VerificationConfig with selective overrides, usable by CLI, server, and programmatic callers.

Parameters:

Name	Type	Description	Default
base	VerificationConfig | None	Optional base config (e.g., from a preset). If None, starts from defaults.	None
answering_model	str | None	Override for the answering model name.	None
answering_provider	str | None	Override for the answering model provider.	None
answering_id	str | None	Override for the answering model identifier.	None
answering_interface	str | None	Override for the answering adapter interface.	None
parsing_model	str | None	Override for the parsing model name.	None
parsing_provider	str | None	Override for the parsing model provider.	None
parsing_id	str | None	Override for the parsing model identifier.	None
parsing_interface	str | None	Override for the parsing adapter interface.	None
temperature	float | None	Override for the LLM temperature.	None
manual_traces	Any | None	Override for manual traces data.	None
replicate_count	int | None	Override for the number of replicates.	None
abstention	bool | None	Override for abstention detection flag.	None
sufficiency	bool | None	Override for sufficiency checking flag.	None
embedding_check	bool | None	Override for embedding check flag.	None
deep_judgment_mode	str | None	Override for template deep judgment mode.	None
evaluation_mode	str | None	Override for evaluation mode.	None
embedding_threshold	float | None	Override for embedding similarity threshold.	None
embedding_model	str | None	Override for embedding model name.	None
async_execution	bool | None	Override for async execution flag.	None
async_workers	int | None	Override for number of async workers.	None
max_concurrent_requests	int | None	Override for global concurrent LLM request cap.	None
use_full_trace_for_template	bool | None	Override for full trace template flag.	None
use_full_trace_for_rubric	bool | None	Override for full trace rubric flag.	None
deep_judgment_rubric_mode	str | None	Override for deep judgment rubric mode.	None
deep_judgment_rubric_excerpts	bool | None	Override for rubric excerpts flag.	None
deep_judgment_rubric_max_excerpts	int | None	Override for max rubric excerpts.	None
deep_judgment_rubric_fuzzy_threshold	float | None	Override for rubric fuzzy threshold.	None
deep_judgment_rubric_retry_attempts	int | None	Override for rubric retry attempts.	None
deep_judgment_rubric_search	bool | None	Override for rubric search flag.	None
deep_judgment_rubric_search_tool	str | None	Override for rubric search tool.	None
deep_judgment_rubric_config	DeepJudgmentRubricCustomConfig | dict[str, Any] | None	Override for rubric config dict.	None
workspace_output_mode	Literal['none', 'full', 'produced'] | None	Override for workspace sidecar capture mode.	None
workspace_output_dir	Path | None	Override for workspace sidecar output directory.	None
workspace_output_exclude_patterns	list[str] | None	Additional workspace sidecar exclude patterns.	None

Returns:

Type	Description
VerificationConfig	A new VerificationConfig with overrides applied.

Source code in src/karenina/schemas/verification/config.py

@classmethod
def from_overrides(
    cls,
    base: "VerificationConfig | None" = None,
    *,
    # Model configuration
    answering_model: str | None = None,
    answering_provider: str | None = None,
    answering_id: str | None = None,
    answering_interface: str | None = None,
    parsing_model: str | None = None,
    parsing_provider: str | None = None,
    parsing_id: str | None = None,
    parsing_interface: str | None = None,
    temperature: float | None = None,
    manual_traces: Any | None = None,
    # Execution settings
    replicate_count: int | None = None,
    # Feature flags
    abstention: bool | None = None,
    sufficiency: bool | None = None,
    embedding_check: bool | None = None,
    deep_judgment_mode: str | None = None,
    # Evaluation settings
    evaluation_mode: str | None = None,
    embedding_threshold: float | None = None,
    embedding_model: str | None = None,
    async_execution: bool | None = None,
    async_workers: int | None = None,
    max_concurrent_requests: int | None = None,
    # Trace filtering
    use_full_trace_for_template: bool | None = None,
    use_full_trace_for_rubric: bool | None = None,
    # Deep judgment rubric settings
    deep_judgment_rubric_mode: str | None = None,
    deep_judgment_rubric_excerpts: bool | None = None,
    deep_judgment_rubric_max_excerpts: int | None = None,
    deep_judgment_rubric_fuzzy_threshold: float | None = None,
    deep_judgment_rubric_retry_attempts: int | None = None,
    deep_judgment_rubric_search: bool | None = None,
    deep_judgment_rubric_search_tool: str | None = None,
    deep_judgment_rubric_config: DeepJudgmentRubricCustomConfig | dict[str, Any] | None = None,
    workspace_output_mode: Literal["none", "full", "produced"] | None = None,
    workspace_output_dir: Path | None = None,
    workspace_output_exclude_patterns: list[str] | None = None,
) -> "VerificationConfig":
    """
    Create a VerificationConfig by applying overrides to an optional base config.

    Implements the hierarchy: overrides > base config > defaults.
    Parameters set to None are not applied (base or default value is preserved).

    This is the canonical way to construct a VerificationConfig with selective
    overrides, usable by CLI, server, and programmatic callers.

    Args:
        base: Optional base config (e.g., from a preset). If None, starts from defaults.
        answering_model: Override for the answering model name.
        answering_provider: Override for the answering model provider.
        answering_id: Override for the answering model identifier.
        answering_interface: Override for the answering adapter interface.
        parsing_model: Override for the parsing model name.
        parsing_provider: Override for the parsing model provider.
        parsing_id: Override for the parsing model identifier.
        parsing_interface: Override for the parsing adapter interface.
        temperature: Override for the LLM temperature.
        manual_traces: Override for manual traces data.
        replicate_count: Override for the number of replicates.
        abstention: Override for abstention detection flag.
        sufficiency: Override for sufficiency checking flag.
        embedding_check: Override for embedding check flag.
        deep_judgment_mode: Override for template deep judgment mode.
        evaluation_mode: Override for evaluation mode.
        embedding_threshold: Override for embedding similarity threshold.
        embedding_model: Override for embedding model name.
        async_execution: Override for async execution flag.
        async_workers: Override for number of async workers.
        max_concurrent_requests: Override for global concurrent LLM request cap.
        use_full_trace_for_template: Override for full trace template flag.
        use_full_trace_for_rubric: Override for full trace rubric flag.
        deep_judgment_rubric_mode: Override for deep judgment rubric mode.
        deep_judgment_rubric_excerpts: Override for rubric excerpts flag.
        deep_judgment_rubric_max_excerpts: Override for max rubric excerpts.
        deep_judgment_rubric_fuzzy_threshold: Override for rubric fuzzy threshold.
        deep_judgment_rubric_retry_attempts: Override for rubric retry attempts.
        deep_judgment_rubric_search: Override for rubric search flag.
        deep_judgment_rubric_search_tool: Override for rubric search tool.
        deep_judgment_rubric_config: Override for rubric config dict.
        workspace_output_mode: Override for workspace sidecar capture mode.
        workspace_output_dir: Override for workspace sidecar output directory.
        workspace_output_exclude_patterns: Additional workspace sidecar exclude patterns.

    Returns:
        A new VerificationConfig with overrides applied.
    """
    # Start with base config dump or empty dict
    config_dict: dict[str, Any] = base.model_dump() if base else {}

    # --- Scalar overrides (None = don't override) ---

    # Replicate count
    if replicate_count is not None:
        config_dict["replicate_count"] = replicate_count
    elif not base:
        config_dict["replicate_count"] = 1

    # Feature flags
    if abstention is not None:
        config_dict["abstention_enabled"] = abstention
    if sufficiency is not None:
        config_dict["sufficiency_enabled"] = sufficiency
    if embedding_check is not None:
        config_dict["embedding_check_enabled"] = embedding_check
    if deep_judgment_mode is not None:
        config_dict["deep_judgment_mode"] = deep_judgment_mode

    # Evaluation settings
    if evaluation_mode is not None:
        config_dict["evaluation_mode"] = evaluation_mode
    if embedding_threshold is not None:
        config_dict["embedding_check_threshold"] = embedding_threshold
    if embedding_model is not None:
        config_dict["embedding_check_model"] = embedding_model
    if async_execution is not None:
        config_dict["async_enabled"] = async_execution
    if async_workers is not None:
        config_dict["async_max_workers"] = async_workers
    if max_concurrent_requests is not None:
        config_dict["max_concurrent_requests"] = max_concurrent_requests

    # Trace filtering
    if use_full_trace_for_template is not None:
        config_dict["use_full_trace_for_template"] = use_full_trace_for_template
    if use_full_trace_for_rubric is not None:
        config_dict["use_full_trace_for_rubric"] = use_full_trace_for_rubric

    # Deep judgment rubric settings
    if deep_judgment_rubric_mode is not None:
        config_dict["deep_judgment_rubric_mode"] = deep_judgment_rubric_mode
    if deep_judgment_rubric_excerpts is not None:
        config_dict["deep_judgment_rubric_global_excerpts"] = deep_judgment_rubric_excerpts
    if deep_judgment_rubric_max_excerpts is not None:
        config_dict["deep_judgment_rubric_max_excerpts_default"] = deep_judgment_rubric_max_excerpts
    if deep_judgment_rubric_fuzzy_threshold is not None:
        config_dict["deep_judgment_rubric_fuzzy_match_threshold_default"] = deep_judgment_rubric_fuzzy_threshold
    if deep_judgment_rubric_retry_attempts is not None:
        config_dict["deep_judgment_rubric_excerpt_retry_attempts_default"] = deep_judgment_rubric_retry_attempts
    if deep_judgment_rubric_search is not None:
        config_dict["deep_judgment_rubric_search_enabled"] = deep_judgment_rubric_search
    if deep_judgment_rubric_search_tool is not None:
        config_dict["deep_judgment_rubric_search_tool"] = deep_judgment_rubric_search_tool
    if deep_judgment_rubric_config is not None:
        config_dict["deep_judgment_rubric_config"] = deep_judgment_rubric_config

    # Workspace output capture
    if workspace_output_mode is not None:
        config_dict["workspace_output_mode"] = workspace_output_mode
    if workspace_output_dir is not None:
        config_dict["workspace_output_dir"] = workspace_output_dir
    if workspace_output_exclude_patterns is not None:
        config_dict["workspace_output_exclude_patterns"] = workspace_output_exclude_patterns

    # --- Model configuration ---
    # Determine the unified interface (answering and parsing may differ)
    ans_interface = answering_interface
    par_interface = parsing_interface
    # If only a single 'interface' concept was provided via answering_interface,
    # it's already split by the caller. No implicit sharing here.

    answering_has_overrides = any(
        [
            answering_model is not None,
            answering_provider is not None,
            ans_interface is not None,
        ]
    )

    parsing_has_overrides = any(
        [
            parsing_model is not None,
            parsing_provider is not None,
            par_interface is not None,
        ]
    )

    if answering_has_overrides:
        config_dict["answering_models"] = [
            cls._build_model_config_dict(
                base_models=base.answering_models if base else None,
                model_name=answering_model,
                provider=answering_provider,
                model_id=answering_id,
                temperature=temperature,
                interface=ans_interface,
                manual_traces=manual_traces,
                default_model="gpt-4.1-mini",
                default_provider="openai",
                default_interface="langchain",
            )
        ]
    elif manual_traces is not None:
        # Manual interface requested via manual_traces without explicit model overrides
        config_dict["answering_models"] = [ModelConfig(interface="manual", manual_traces=manual_traces)]

    if parsing_has_overrides:
        config_dict["parsing_models"] = [
            cls._build_model_config_dict(
                base_models=base.parsing_models if base else None,
                model_name=parsing_model,
                provider=parsing_provider,
                model_id=parsing_id,
                temperature=temperature,
                interface=par_interface,
                manual_traces=None,  # Parsing model never uses manual interface
                default_model="gpt-4.1-mini",
                default_provider="openai",
                default_interface="langchain",
            )
        ]

    return cls(**config_dict)

from_preset classmethod

from_preset(filepath: Path) -> VerificationConfig

Load a VerificationConfig from a preset file. Delegates to config_presets.load_preset.

Source code in src/karenina/schemas/verification/config.py

@classmethod
def from_preset(cls, filepath: Path) -> "VerificationConfig":
    """Load a VerificationConfig from a preset file. Delegates to config_presets.load_preset."""
    return load_preset(filepath)

get_few_shot_config

get_few_shot_config() -> FewShotConfig | None

Get the FewShotConfig for this verification run.

Returns:

Type	Description
FewShotConfig | None	The FewShotConfig to use, or None if few-shot is disabled

Source code in src/karenina/schemas/verification/config.py

def get_few_shot_config(self) -> FewShotConfig | None:
    """
    Get the FewShotConfig for this verification run.

    Returns:
        The FewShotConfig to use, or None if few-shot is disabled
    """
    return self.few_shot_config

is_few_shot_enabled

is_few_shot_enabled() -> bool

Check if few-shot prompting is enabled.

Returns:

Type	Description
bool	True if few-shot is enabled

Source code in src/karenina/schemas/verification/config.py

def is_few_shot_enabled(self) -> bool:
    """
    Check if few-shot prompting is enabled.

    Returns:
        True if few-shot is enabled
    """
    config = self.get_few_shot_config()
    return config is not None and config.source != "disabled"

sanitize_model_config classmethod

sanitize_model_config(
    model: dict[str, Any],
) -> dict[str, Any]

Sanitize model configuration. Delegates to config_presets.sanitize_model_config.

Source code in src/karenina/schemas/verification/config.py

@classmethod
def sanitize_model_config(cls, model: dict[str, Any]) -> dict[str, Any]:
    """Sanitize model configuration. Delegates to config_presets.sanitize_model_config."""
    return sanitize_model_config(model)

sanitize_preset_name classmethod

sanitize_preset_name(name: str) -> str

Convert preset name to safe filename. Delegates to config_presets.sanitize_preset_name.

Source code in src/karenina/schemas/verification/config.py

@classmethod
def sanitize_preset_name(cls, name: str) -> str:
    """Convert preset name to safe filename. Delegates to config_presets.sanitize_preset_name."""
    return sanitize_preset_name(name)

save_preset

save_preset(
    name: str,
    description: str | None = None,
    presets_dir: Path | None = None,
) -> dict[str, Any]

Save this config as a preset file. Delegates to config_presets.save_preset.

Source code in src/karenina/schemas/verification/config.py

def save_preset(
    self,
    name: str,
    description: str | None = None,
    presets_dir: Path | None = None,
) -> dict[str, Any]:
    """Save this config as a preset file. Delegates to config_presets.save_preset."""
    return save_preset(self, name, description, presets_dir)

validate_preset_metadata classmethod

validate_preset_metadata(
    name: str, description: str | None = None
) -> None

Validate preset metadata. Delegates to config_presets.validate_preset_metadata.

Source code in src/karenina/schemas/verification/config.py

@classmethod
def validate_preset_metadata(cls, name: str, description: str | None = None) -> None:
    """Validate preset metadata. Delegates to config_presets.validate_preset_metadata."""
    return validate_preset_metadata(name, description)

VerificationJob

Bases: BaseModel

Represents a verification job.

Source code in src/karenina/schemas/verification/job.py

class VerificationJob(BaseModel):
    """Represents a verification job."""

    model_config = ConfigDict(extra="forbid")

    job_id: str
    run_name: str  # User-defined or auto-generated run name
    status: Literal["pending", "running", "completed", "failed", "cancelled"]
    config: "VerificationConfig"

    # Database storage
    storage_url: str | None = None  # Database URL for auto-save functionality
    benchmark_name: str | None = None  # Benchmark name for auto-save functionality

    # Progress tracking
    total_questions: int
    processed_count: int = 0
    successful_count: int = 0
    failed_count: int = 0
    percentage: float = 0.0
    current_question: str = ""
    last_task_duration: float | None = None  # Execution time of last completed task

    # WebSocket streaming progress fields
    in_progress_questions: list[str] = Field(default_factory=list)

    # Task timing tracking (maps question_id to start time)
    task_start_times: dict[str, float] = Field(default_factory=dict)

    # Timing
    start_time: float | None = None
    end_time: float | None = None

    # Results
    result_set: "VerificationResultSet | None" = None  # Unified verification result container
    error_message: str | None = None

    @staticmethod
    def _make_task_key(question_id: str, replicate: int | None = None) -> str:
        """Create a unique task key from question_id and optional replicate.

        For single-replicate runs, returns just the question_id.
        For multi-replicate runs, appends _rep{N} to distinguish tasks.
        """
        if replicate is None:
            return question_id
        return f"{question_id}_rep{replicate}"

    def task_started(self, question_id: str, replicate: int | None = None) -> None:
        """Mark a task as started and record start time.

        Args:
            question_id: The question identifier
            replicate: Optional replicate number (for multi-replicate runs)
        """
        task_key = self._make_task_key(question_id, replicate)
        if task_key not in self.in_progress_questions:
            self.in_progress_questions.append(task_key)

        # Record task start time
        self.task_start_times[task_key] = time.time()

    def task_finished(self, question_id: str, success: bool, replicate: int | None = None) -> None:
        """Mark a task as finished, calculate duration, and update counts.

        Args:
            question_id: The question identifier
            success: Whether the task completed successfully
            replicate: Optional replicate number (for multi-replicate runs)
        """
        task_key = self._make_task_key(question_id, replicate)

        # Calculate task duration from recorded start time
        task_duration = 0.0
        if task_key in self.task_start_times:
            task_duration = time.time() - self.task_start_times[task_key]
            # Clean up start time
            del self.task_start_times[task_key]

        # Remove from in-progress list
        if task_key in self.in_progress_questions:
            self.in_progress_questions.remove(task_key)

        # Update counts
        self.processed_count += 1
        if success:
            self.successful_count += 1
        else:
            self.failed_count += 1

        # Update percentage
        self.percentage = (self.processed_count / self.total_questions) * 100 if self.total_questions > 0 else 0.0

        # Track last task duration
        self.last_task_duration = task_duration

    def to_dict(self) -> dict[str, Any]:
        """Convert job to dictionary for API response."""
        # Calculate duration if job has started
        duration = None
        if self.start_time:
            duration = self.end_time - self.start_time if self.end_time else time.time() - self.start_time

        return {
            "job_id": self.job_id,
            "run_name": self.run_name,
            "status": self.status,
            "total_questions": self.total_questions,
            "processed_count": self.processed_count,
            "successful_count": self.successful_count,
            "failed_count": self.failed_count,
            "percentage": self.percentage,
            "current_question": self.current_question,
            "duration_seconds": duration,
            "last_task_duration": self.last_task_duration,
            "error_message": self.error_message,
            "start_time": self.start_time,
            "end_time": self.end_time,
            "in_progress_questions": self.in_progress_questions,
        }

Functions

task_finished

task_finished(
    question_id: str,
    success: bool,
    replicate: int | None = None,
) -> None

Mark a task as finished, calculate duration, and update counts.

Parameters:

Name	Type	Description	Default
question_id	str	The question identifier	required
success	bool	Whether the task completed successfully	required
replicate	int | None	Optional replicate number (for multi-replicate runs)	None

Source code in src/karenina/schemas/verification/job.py

def task_finished(self, question_id: str, success: bool, replicate: int | None = None) -> None:
    """Mark a task as finished, calculate duration, and update counts.

    Args:
        question_id: The question identifier
        success: Whether the task completed successfully
        replicate: Optional replicate number (for multi-replicate runs)
    """
    task_key = self._make_task_key(question_id, replicate)

    # Calculate task duration from recorded start time
    task_duration = 0.0
    if task_key in self.task_start_times:
        task_duration = time.time() - self.task_start_times[task_key]
        # Clean up start time
        del self.task_start_times[task_key]

    # Remove from in-progress list
    if task_key in self.in_progress_questions:
        self.in_progress_questions.remove(task_key)

    # Update counts
    self.processed_count += 1
    if success:
        self.successful_count += 1
    else:
        self.failed_count += 1

    # Update percentage
    self.percentage = (self.processed_count / self.total_questions) * 100 if self.total_questions > 0 else 0.0

    # Track last task duration
    self.last_task_duration = task_duration

task_started

task_started(
    question_id: str, replicate: int | None = None
) -> None

Mark a task as started and record start time.

Parameters:

Name	Type	Description	Default
question_id	str	The question identifier	required
replicate	int | None	Optional replicate number (for multi-replicate runs)	None

Source code in src/karenina/schemas/verification/job.py

def task_started(self, question_id: str, replicate: int | None = None) -> None:
    """Mark a task as started and record start time.

    Args:
        question_id: The question identifier
        replicate: Optional replicate number (for multi-replicate runs)
    """
    task_key = self._make_task_key(question_id, replicate)
    if task_key not in self.in_progress_questions:
        self.in_progress_questions.append(task_key)

    # Record task start time
    self.task_start_times[task_key] = time.time()

to_dict

to_dict() -> dict[str, Any]

Convert job to dictionary for API response.

Source code in src/karenina/schemas/verification/job.py

def to_dict(self) -> dict[str, Any]:
    """Convert job to dictionary for API response."""
    # Calculate duration if job has started
    duration = None
    if self.start_time:
        duration = self.end_time - self.start_time if self.end_time else time.time() - self.start_time

    return {
        "job_id": self.job_id,
        "run_name": self.run_name,
        "status": self.status,
        "total_questions": self.total_questions,
        "processed_count": self.processed_count,
        "successful_count": self.successful_count,
        "failed_count": self.failed_count,
        "percentage": self.percentage,
        "current_question": self.current_question,
        "duration_seconds": duration,
        "last_task_duration": self.last_task_duration,
        "error_message": self.error_message,
        "start_time": self.start_time,
        "end_time": self.end_time,
        "in_progress_questions": self.in_progress_questions,
    }

VerificationResult

Bases: BaseModel

Result of verifying a single question.

Source code in src/karenina/schemas/verification/result.py

class VerificationResult(BaseModel):
    """Result of verifying a single question."""

    metadata: VerificationResultMetadata
    template: VerificationResultTemplate | None = None
    rubric: VerificationResultRubric | None = None
    deep_judgment: VerificationResultDeepJudgment | None = None
    deep_judgment_rubric: VerificationResultDeepJudgmentRubric | None = None

    # Shared trace filtering fields (for MCP agent responses)
    # These are at the root level because both template and rubric evaluation use the same input
    evaluation_input: str | None = None  # Input passed to evaluation (full trace or final AI message)
    used_full_trace: bool = True  # Whether full trace was used (True) or only final AI message (False)
    trace_extraction_error: str | None = None  # Error if final AI message extraction failed

Functions

abstention_detection_instruction

abstention_detection_instruction() -> str

Return Karenina's standard instruction for explicit abstention checks.

Source code in src/karenina/benchmark/trace.py

def abstention_detection_instruction() -> str:
    """Return Karenina's standard instruction for explicit abstention checks."""
    return ABSTENTION_DETECTION_SYS

count_deep_judgment_reasoning_tokens

count_deep_judgment_reasoning_tokens(
    answer: str,
    trait: LLMRubricTrait,
    *,
    question: str = "",
    task_eval_mode: bool = True,
    encoding_name: str = "o200k_base",
    parsing_model: ModelConfig | None = None,
    timeout: float = 10.0,
) -> int

Count the rendered first-stage deep-judgment reasoning prompt.

The count includes both system and user messages. It uses the stable offline encoding named by encoding_name and does not contact a model provider.

Parameters:

Name	Type	Description	Default
answer	str	Response or trace passed to the reasoning stage.	required
trait	LLMRubricTrait	Deep-judgment rubric trait.	required
question	str	Question rendered when TaskEval mode is disabled.	''
task_eval_mode	bool	Whether to use TaskEval prompt rendering.	True
encoding_name	str	Tiktoken encoding used for an offline count.	'o200k_base'
parsing_model	ModelConfig | None	Optional OpenAI-compatible model whose /tokenize endpoint provides an exact chat-template-aware count.	None
timeout	float	Endpoint timeout in seconds.	10.0

Returns:

Type	Description
int	Token count for the complete reasoning prompt.

Raises:

Type	Description
ValueError	If parsing_model is not an OpenAI endpoint model.
RuntimeError	If an exact endpoint count fails.

Source code in src/karenina/benchmark/verification/utils/token_budget.py

def count_deep_judgment_reasoning_tokens(
    answer: str,
    trait: LLMRubricTrait,
    *,
    question: str = "",
    task_eval_mode: bool = True,
    encoding_name: str = "o200k_base",
    parsing_model: ModelConfig | None = None,
    timeout: float = 10.0,
) -> int:
    """Count the rendered first-stage deep-judgment reasoning prompt.

    The count includes both system and user messages. It uses the stable
    offline encoding named by ``encoding_name`` and does not contact a model
    provider.

    Args:
        answer: Response or trace passed to the reasoning stage.
        trait: Deep-judgment rubric trait.
        question: Question rendered when TaskEval mode is disabled.
        task_eval_mode: Whether to use TaskEval prompt rendering.
        encoding_name: Tiktoken encoding used for an offline count.
        parsing_model: Optional OpenAI-compatible model whose ``/tokenize``
            endpoint provides an exact chat-template-aware count.
        timeout: Endpoint timeout in seconds.

    Returns:
        Token count for the complete reasoning prompt.

    Raises:
        ValueError: If ``parsing_model`` is not an OpenAI endpoint model.
        RuntimeError: If an exact endpoint count fails.
    """
    builder = DeepJudgmentPromptBuilder()
    system_prompt = builder.build_reasoning_system_prompt()
    user_prompt = builder.build_reasoning_user_prompt_without_excerpts(
        question=question,
        answer=answer,
        trait=trait,
        task_eval_mode=task_eval_mode,
    )
    if parsing_model is None:
        return count_tokens(f"{system_prompt}\n{user_prompt}", encoding_name)
    if parsing_model.interface != "openai_endpoint" or parsing_model.endpoint_base_url is None:
        raise ValueError("Exact token counting requires an openai_endpoint parsing model")

    base_url = str(parsing_model.endpoint_base_url).rstrip("/")
    if base_url.endswith("/v1"):
        base_url = base_url[: -len("/v1")]
    payload = json.dumps(
        {
            "model": parsing_model.model_name,
            "messages": [
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt},
            ],
        }
    ).encode("utf-8")
    request = urllib.request.Request(
        f"{base_url}/tokenize",
        data=payload,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    api_key = parsing_model.endpoint_api_key
    if api_key is not None:
        secret = api_key.get_secret_value()
        if secret and secret != "EMPTY":
            request.add_header("Authorization", f"Bearer {secret}")
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            body = json.loads(response.read().decode("utf-8"))
    except (
        urllib.error.HTTPError,
        urllib.error.URLError,
        TimeoutError,
        json.JSONDecodeError,
    ) as error:
        raise RuntimeError("The model endpoint could not count the reasoning prompt") from error
    count = body.get("count")
    if not isinstance(count, int):
        raise RuntimeError("The model endpoint returned no integer token count")
    return count

count_tokens

count_tokens(
    text: str, encoding_name: str = "o200k_base"
) -> int

Count tokens in text under the requested tiktoken encoding.

Source code in src/karenina/benchmark/verification/utils/token_budget.py

def count_tokens(text: str, encoding_name: str = "o200k_base") -> int:
    """Count tokens in text under the requested tiktoken encoding."""
    return len(_encoding(encoding_name).encode(text))

create_run_directory

create_run_directory(
    root: Path,
    label: str,
    *,
    benchmark_path: str | Path,
    config: VerificationConfig,
    run_name: str | None = None,
    selected_question_count: int | None = None,
    now: datetime | None = None,
    resume: bool = False,
) -> RunDirectory

Create a timestamped run directory and running manifest.

Parameters:

Name	Type	Description	Default
root	Path	Parent directory for verification runs.	required
label	str	Filesystem-safe reviewer-facing run label.	required
benchmark_path	str | Path	Benchmark source recorded for provenance.	required
config	VerificationConfig	Verification configuration to describe and sanitize.	required
run_name	str | None	Result metadata run name. Defaults to label.	None
selected_question_count	int | None	Number of questions selected for this run.	None
now	datetime | None	Timestamp override for deterministic callers and tests.	None
resume	bool	Reopen the timestamped directory when it already exists.	False

Returns:

Type	Description
RunDirectory	Newly created run directory descriptor.

Raises:

Type	Description
ValueError	If the label is unsafe.
FileExistsError	If the timestamped directory exists without resume.

Source code in src/karenina/benchmark/run_artifacts.py

def create_run_directory(
    root: Path,
    label: str,
    *,
    benchmark_path: str | Path,
    config: VerificationConfig,
    run_name: str | None = None,
    selected_question_count: int | None = None,
    now: datetime | None = None,
    resume: bool = False,
) -> RunDirectory:
    """Create a timestamped run directory and running manifest.

    Args:
        root: Parent directory for verification runs.
        label: Filesystem-safe reviewer-facing run label.
        benchmark_path: Benchmark source recorded for provenance.
        config: Verification configuration to describe and sanitize.
        run_name: Result metadata run name. Defaults to ``label``.
        selected_question_count: Number of questions selected for this run.
        now: Timestamp override for deterministic callers and tests.
        resume: Reopen the timestamped directory when it already exists.

    Returns:
        Newly created run directory descriptor.

    Raises:
        ValueError: If the label is unsafe.
        FileExistsError: If the timestamped directory exists without resume.
    """
    if not _SAFE_LABEL.fullmatch(label):
        raise ValueError("Run label must contain only letters, digits, dots, underscores, and hyphens")
    created = now or datetime.now(UTC)
    directory_name = f"{label}_{created.astimezone(UTC).strftime('%Y%m%dT%H%M%SZ')}"
    path = Path(root) / directory_name
    if path.exists():
        if not resume:
            raise FileExistsError(path)
        manifest_path = path / RUN_MANIFEST_FILENAME
        manifest = RunManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
        run_directory = RunDirectory(path=path, manifest=manifest)
        run_directory.traces_path.mkdir(exist_ok=True)
        run_directory.workspaces_path.mkdir(exist_ok=True)
        run_directory.update_status("running", now=created)
        return run_directory
    path.mkdir(parents=True, exist_ok=False)
    traces_path = path / "traces"
    workspaces_path = path / "workspaces"
    traces_path.mkdir()
    workspaces_path.mkdir()

    raw_configuration = config.model_dump(mode="json", exclude={"manual_traces": True})
    manifest = RunManifest(
        status="running",
        label=label,
        run_name=run_name or label,
        created_at=_format_time(created),
        updated_at=_format_time(created),
        benchmark_path=str(benchmark_path),
        selected_question_count=selected_question_count,
        replicate_count=config.replicate_count,
        answering_models=[ModelIdentity.from_model_config(model) for model in config.answering_models],
        parsing_models=[ModelIdentity.from_model_config(model, role="parsing") for model in config.parsing_models],
        configuration=mask_run_configuration(raw_configuration),
    )
    run_directory = RunDirectory(path=path, manifest=manifest)
    run_directory.write_manifest()
    return run_directory

evaluate_rubric_on_results

evaluate_rubric_on_results(
    results: Iterable[VerificationResult],
    rubric: Rubric,
    parsing_model: ModelConfig,
    *,
    text_selector: Callable[[VerificationResult], str]
    | None = None,
    row_context: Callable[[VerificationResult], RowContext]
    | None = None,
    row_filter: Callable[[VerificationResult], bool]
    | None = None,
    rubric_factory: Callable[
        [VerificationResult, Rubric], Rubric
    ]
    | None = None,
    sibling_identity: Callable[
        [VerificationResult], Hashable
    ]
    | None = None,
    collapse_parser_siblings: bool = True,
    max_workers: int = 4,
) -> Iterator[PostHocJudgment]

Judge stored result rows, optionally once per generated trace.

Parameters:

Name	Type	Description	Default
results	Iterable[VerificationResult]	Stored verification results.	required
rubric	Rubric	Traits to evaluate.	required
parsing_model	ModelConfig	Model used by traits that require an LLM.	required
text_selector	Callable[[VerificationResult], str] | None	Optional stored-text selector.	None
row_context	Callable[[VerificationResult], RowContext] | None	Optional question and reference-answer selector.	None
row_filter	Callable[[VerificationResult], bool] | None	Optional predicate used to exclude rows.	None
rubric_factory	Callable[[VerificationResult, Rubric], Rubric] | None	Optional factory for a row-specific rubric.	None
sibling_identity	Callable[[VerificationResult], Hashable] | None	Optional parser-independent grouping identity.	None
collapse_parser_siblings	bool	Judge rows sharing trace identity once.	True
max_workers	int	Maximum concurrent row evaluations.	4

Yields:

Type	Description
PostHocJudgment	Post-hoc judgments with every represented row key in sibling_keys.

Source code in src/karenina/benchmark/verification/post_hoc.py

def evaluate_rubric_on_results(
    results: Iterable[VerificationResult],
    rubric: Rubric,
    parsing_model: ModelConfig,
    *,
    text_selector: Callable[[VerificationResult], str] | None = None,
    row_context: Callable[[VerificationResult], RowContext] | None = None,
    row_filter: Callable[[VerificationResult], bool] | None = None,
    rubric_factory: Callable[[VerificationResult, Rubric], Rubric] | None = None,
    sibling_identity: Callable[[VerificationResult], Hashable] | None = None,
    collapse_parser_siblings: bool = True,
    max_workers: int = 4,
) -> Iterator[PostHocJudgment]:
    """Judge stored result rows, optionally once per generated trace.

    Args:
        results: Stored verification results.
        rubric: Traits to evaluate.
        parsing_model: Model used by traits that require an LLM.
        text_selector: Optional stored-text selector.
        row_context: Optional question and reference-answer selector.
        row_filter: Optional predicate used to exclude rows.
        rubric_factory: Optional factory for a row-specific rubric.
        sibling_identity: Optional parser-independent grouping identity.
        collapse_parser_siblings: Judge rows sharing trace identity once.
        max_workers: Maximum concurrent row evaluations.

    Yields:
        Post-hoc judgments with every represented row key in ``sibling_keys``.
    """
    select_text = text_selector or _default_text_selector
    select_context = row_context or (lambda _result: RowContext())

    prepared: list[PreparedPostHocItem] = []
    siblings: dict[Hashable, list[tuple[ResultRowKey, str]]] = {}
    judgment_groups: dict[str, Hashable] = {}
    for result in results:
        if row_filter is not None and not row_filter(result):
            continue
        key = ResultRowKey.from_result(result)
        result_id = result.metadata.result_id
        selected_rubric = rubric_factory(result, rubric) if rubric_factory else rubric
        if not collapse_parser_siblings:
            prepared.append(((key, select_text(result), select_context(result)), selected_rubric, result_id))
            continue
        trace_identity: Hashable = sibling_identity(result) if sibling_identity is not None else key.trace_identity
        if trace_identity in siblings:
            siblings[trace_identity].append((key, result_id))
            continue
        siblings[trace_identity] = [(key, result_id)]
        judgment_groups[result_id] = trace_identity
        prepared.append(((key, select_text(result), select_context(result)), selected_rubric, result_id))

    config = build_rubric_only_config(parsing_model)
    for judgment in _evaluate_prepared(prepared, config, max_workers=max_workers):
        if collapse_parser_siblings:
            representative_id = judgment.representative_result_id
            if representative_id is None:
                raise RuntimeError("Stored-result judgment has no representative result id")
            grouped = siblings[judgment_groups[representative_id]]
            yield replace(
                judgment,
                sibling_keys=tuple(key for key, _result_id in grouped),
                sibling_result_ids=tuple(result_id for _key, result_id in grouped),
            )
        else:
            yield judgment

evaluate_rubric_on_texts

evaluate_rubric_on_texts(
    items: Iterable[PostHocItem],
    rubric: Rubric,
    parsing_model: ModelConfig,
    *,
    max_workers: int = 4,
) -> Iterator[PostHocJudgment]

Judge prepared key, text, and context triples against a rubric.

Parameters:

Name	Type	Description	Default
items	Iterable[PostHocItem]	Prepared stored-text evaluation inputs.	required
rubric	Rubric	Traits to evaluate.	required
parsing_model	ModelConfig	Model used by traits that require an LLM.	required
max_workers	int	Maximum concurrent row evaluations.	4

Yields:

Type	Description
PostHocJudgment	One judgment per input, in completion order when concurrent.

Raises:

Type	Description
ValueError	If max_workers is less than one.

Source code in src/karenina/benchmark/verification/post_hoc.py

def evaluate_rubric_on_texts(
    items: Iterable[PostHocItem],
    rubric: Rubric,
    parsing_model: ModelConfig,
    *,
    max_workers: int = 4,
) -> Iterator[PostHocJudgment]:
    """Judge prepared key, text, and context triples against a rubric.

    Args:
        items: Prepared stored-text evaluation inputs.
        rubric: Traits to evaluate.
        parsing_model: Model used by traits that require an LLM.
        max_workers: Maximum concurrent row evaluations.

    Yields:
        One judgment per input, in completion order when concurrent.

    Raises:
        ValueError: If ``max_workers`` is less than one.
    """
    if max_workers < 1:
        raise ValueError("max_workers must be at least 1")

    config = build_rubric_only_config(parsing_model)
    prepared = ((item, rubric, None) for item in items)
    yield from _evaluate_prepared(prepared, config, max_workers=max_workers)

export_curation_workbook

export_curation_workbook(
    benchmark: Benchmark, output_path: str | Path
) -> CurationWorkbookSummary

Write a field-level workbook for benchmark curation and review.

Template source is parsed with :mod:ast and is never executed by this exporter.

Parameters:

Name	Type	Description	Default
benchmark	Benchmark	Benchmark to export.	required
output_path	str | Path	Destination .xlsx path.	required

Returns:

Type	Description
CurationWorkbookSummary	Counts and the resolved workbook path.

Raises:

Type	Description
ValueError	If the destination is not an Excel workbook path.
OSError	If the workbook cannot be written.

Source code in src/karenina/benchmark/curation_workbook.py

def export_curation_workbook(
    benchmark: Benchmark,
    output_path: str | Path,
) -> CurationWorkbookSummary:
    """Write a field-level workbook for benchmark curation and review.

    Template source is parsed with :mod:`ast` and is never executed by this
    exporter.

    Args:
        benchmark: Benchmark to export.
        output_path: Destination ``.xlsx`` path.

    Returns:
        Counts and the resolved workbook path.

    Raises:
        ValueError: If the destination is not an Excel workbook path.
        OSError: If the workbook cannot be written.
    """

    destination = Path(output_path).expanduser().resolve()
    if destination.suffix.lower() != ".xlsx":
        raise ValueError("Curation workbook output must use the .xlsx extension")

    rows, counts = _rows_for_benchmark(benchmark)
    workbook = Workbook()
    review = workbook.active
    review.title = "Curation Review"
    review.freeze_panes = "A2"
    review.auto_filter.ref = f"A1:{get_column_letter(len(_COLUMNS))}{len(rows) + 1}"

    header_fill = PatternFill(fill_type="solid", fgColor="2F5496")
    alternate_fill = PatternFill(fill_type="solid", fgColor="D9E8F5")
    header_alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
    body_alignment = Alignment(vertical="top", wrap_text=True)

    for column_number, (label, width) in enumerate(zip(_COLUMNS, _COLUMN_WIDTHS, strict=True), start=1):
        cell = review.cell(row=1, column=column_number, value=label)
        cell.font = Font(bold=True, color="FFFFFF")
        cell.fill = header_fill
        cell.alignment = header_alignment
        review.column_dimensions[get_column_letter(column_number)].width = width

    for row_number, values in enumerate(rows, start=2):
        item_number = int(values[0])
        for column_number, value in enumerate(values, start=1):
            cell = review.cell(row=row_number, column=column_number, value=value)
            cell.alignment = body_alignment
            if item_number % 2 == 0:
                cell.fill = alternate_fill

    summary = workbook.create_sheet("Summary")
    summary_rows = (
        ("Benchmark Name", benchmark.name),
        ("Questions", len(benchmark.get_question_ids())),
        ("Verified Fields", counts["fields"]),
        ("Draft Items", counts["Draft"]),
        ("Finished Items", counts["Finished"]),
        ("Template Parse Errors", counts["parse_errors"]),
    )
    for row_number, (label, value) in enumerate(summary_rows, start=1):
        summary.cell(row=row_number, column=1, value=label).font = Font(bold=True)
        summary.cell(row=row_number, column=2, value=value)
    summary.column_dimensions["A"].width = 28
    summary.column_dimensions["B"].width = 42

    destination.parent.mkdir(parents=True, exist_ok=True)
    try:
        workbook.save(destination)
    except OSError as exc:
        raise OSError(f"Unable to write curation workbook: {destination}") from exc

    return CurationWorkbookSummary(
        output_path=destination,
        question_count=len(benchmark.get_question_ids()),
        field_count=counts["fields"],
        draft_count=counts["Draft"],
        finished_count=counts["Finished"],
        template_parse_error_count=counts["parse_errors"],
    )

export_verification_results_csv

export_verification_results_csv(
    job: VerificationJob,
    results: VerificationResultSet,
    global_rubric: HasTraitNames | None = None,
) -> str

Export verification results to CSV format with rubric consolidation.

Parameters:

Name	Type	Description	Default
job	VerificationJob	The verification job	required
results	VerificationResultSet	VerificationResultSet containing all verification results	required
global_rubric	HasTraitNames | None	Optional global rubric object that implements HasTraitNames protocol for distinguishing global vs question-specific traits. If None, all rubric traits will be consolidated into question_specific_rubrics.	None

Returns:

Type	Description
str	CSV string with results. Global rubric traits appear as dedicated columns
str	(rubric_TraitName), while question-specific traits are consolidated into
str	a single JSON column (question_specific_rubrics).

Note

The function gracefully handles errors in trait name extraction and JSON serialization, logging warnings and continuing with fallback values.

Source code in src/karenina/benchmark/verification/stages/helpers/results_exporter.py

def export_verification_results_csv(
    job: VerificationJob, results: VerificationResultSet, global_rubric: HasTraitNames | None = None
) -> str:
    """
    Export verification results to CSV format with rubric consolidation.

    Args:
        job: The verification job
        results: VerificationResultSet containing all verification results
        global_rubric: Optional global rubric object that implements HasTraitNames protocol
                      for distinguishing global vs question-specific traits. If None,
                      all rubric traits will be consolidated into question_specific_rubrics.

    Returns:
        CSV string with results. Global rubric traits appear as dedicated columns
        (rubric_TraitName), while question-specific traits are consolidated into
        a single JSON column (question_specific_rubrics).

    Note:
        The function gracefully handles errors in trait name extraction and JSON
        serialization, logging warnings and continuing with fallback values.
    """
    # Input validation
    if not results or len(results) == 0:
        logger.warning("No results provided for CSV export. Generating empty CSV.")
        # Return minimal CSV with headers only
        output = StringIO()
        csv_writer = csv.writer(output)
        csv_writer.writerow(
            [
                "question_id",
                "success",
                "error",
                "question_text",
                "raw_llm_response",
                "keywords",
                "export_timestamp",
                "karenina_version",
                "job_id",
            ]
        )
        return output.getvalue()

    # Log export summary
    logger.info("Starting CSV export for %d results", len(results))

    output = StringIO()

    # Collect all unique rubric trait names across all results with validation
    all_rubric_traits: set[str] = set()
    invalid_trait_count = 0
    for result in results:
        if result.rubric:
            # Collect from all trait score dicts (llm, regex, callable, metric)
            for trait_dict in [
                result.rubric.llm_trait_scores,
                result.rubric.regex_trait_scores,
                result.rubric.callable_trait_scores,
                result.rubric.metric_trait_scores,
            ]:
                if trait_dict:
                    for trait_name in trait_dict:
                        if _validate_trait_name(trait_name):
                            all_rubric_traits.add(trait_name)
                        else:
                            invalid_trait_count += 1
                            logger.warning(
                                "Skipping invalid trait name '%s' in question %s",
                                trait_name,
                                result.metadata.question_id,
                            )

    if invalid_trait_count > 0:
        logger.info("Skipped %d invalid trait names during CSV export", invalid_trait_count)

    # Determine global vs question-specific rubrics
    global_trait_names: set[str] = set()
    if global_rubric:
        try:
            if hasattr(global_rubric, "get_trait_names") and callable(global_rubric.get_trait_names):
                trait_names = global_rubric.get_trait_names()
                if isinstance(trait_names, list):
                    # Validate each trait name from global rubric
                    valid_global_traits = []
                    for trait_name in trait_names:
                        if _validate_trait_name(trait_name):
                            valid_global_traits.append(trait_name)
                        else:
                            logger.warning("Skipping invalid global trait name '%s' from global_rubric", trait_name)
                    global_trait_names = set(valid_global_traits)

                    if len(valid_global_traits) != len(trait_names):
                        logger.info(
                            "Global rubric had %d traits, %d were valid for CSV export",
                            len(trait_names),
                            len(valid_global_traits),
                        )
                else:
                    logger.warning(
                        "Global rubric get_trait_names() returned %s instead of list. "
                        "All rubric traits will be treated as question-specific.",
                        type(trait_names).__name__,
                    )
            else:
                logger.warning(
                    "Global rubric object does not have a callable get_trait_names method. "
                    "All rubric traits will be treated as question-specific."
                )
        except (AttributeError, TypeError, ValueError) as e:
            logger.warning(
                "Error accessing global rubric trait names (%s: %s). "
                "All rubric traits will be treated as question-specific.",
                type(e).__name__,
                e,
            )
            # Continue with empty set - graceful degradation

    # Separate traits into global and question-specific (with performance optimization)
    global_traits = sorted(all_rubric_traits.intersection(global_trait_names))
    question_specific_traits = sorted(all_rubric_traits - global_trait_names)

    # Pre-compute set for faster lookups during row processing
    question_specific_traits_set = set(question_specific_traits)

    # Log export configuration
    logger.debug(
        "CSV export configuration: %d global traits, %d question-specific traits, %d total results",
        len(global_traits),
        len(question_specific_traits),
        len(results),
    )

    # Define CSV headers with all result fields + dynamic rubric columns
    headers = [
        "question_id",
        "success",
        "error",
        "question_text",
        "raw_llm_response",
        "parsed_gt_response",
        "parsed_llm_response",
        "template_verification_performed",
        "verify_result",
        "verify_granular_result",
        "rubric_evaluation_performed",
        "keywords",
    ]

    # Add global rubric trait columns (prefixed with 'rubric_')
    headers.extend([f"rubric_{trait}" for trait in global_traits])

    # Add single column for question-specific rubrics
    if question_specific_traits:
        headers.append("question_specific_rubrics")

    # Add remaining standard columns
    headers.extend(
        [
            "answering_model",
            "parsing_model",
            "replicate",
            "execution_time",
            "timestamp",
            "answering_system_prompt",
            "parsing_system_prompt",
            "run_name",
            "export_timestamp",
            "karenina_version",
            "job_id",
            # Embedding check fields
            "embedding_check_performed",
            "embedding_similarity_score",
            "embedding_override_applied",
            "embedding_model_used",
            # MCP server fields
            "answering_mcp_servers",
            # Deep-judgment fields
            "deep_judgment_mode",
            "deep_judgment_performed",
            "extracted_excerpts",
            "attribute_reasoning",
            "deep_judgment_stages_completed",
            "deep_judgment_model_calls",
            "deep_judgment_excerpt_retry_count",
            "attributes_without_excerpts",
            # Search-enhanced deep-judgment fields
            "deep_judgment_search_enabled",
            "hallucination_risk_assessment",
            # Deep-judgment rubric fields
            "deep_judgment_rubric_performed",
            "extracted_rubric_excerpts",
            "rubric_trait_reasoning",
            "deep_judgment_rubric_scores",
            "standard_rubric_scores",
            "trait_metadata",
            "traits_without_valid_excerpts",
            "rubric_hallucination_risk_assessment",
            "total_deep_judgment_model_calls",
            "total_traits_evaluated",
            "total_excerpt_retries",
            # Metric trait fields
            "metric_trait_confusion_lists",
            "metric_trait_metrics",
            # LLM usage tracking fields
            "usage_metadata",
            "agent_metrics",
        ]
    )

    writer: csv.DictWriter[str] = csv.DictWriter(output, fieldnames=headers)
    writer.writeheader()

    # Metadata for each row
    export_timestamp = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())
    karenina_version = get_karenina_version()

    # Write data rows
    for result in results:
        # Access fields from nested structure
        metadata = result.metadata
        template = result.template
        rubric = result.rubric
        deep_judgment = result.deep_judgment
        deep_judgment_rubric = result.deep_judgment_rubric

        row = {
            # Metadata fields
            "question_id": metadata.question_id,
            "success": metadata.failure is None,  # Header uses 'success'; derived from Failure presence
            "error": metadata.failure.reason if metadata.failure else "",
            "question_text": metadata.question_text,
            "keywords": _safe_json_serialize(metadata.keywords, metadata.question_id, "keywords"),
            "answering_model": metadata.answering_model,
            "parsing_model": metadata.parsing_model,
            "replicate": metadata.replicate or "",
            "execution_time": metadata.execution_time,
            "timestamp": metadata.timestamp,
            "run_name": metadata.run_name or "",
            # Template fields
            "raw_llm_response": template.raw_llm_response if template else "",
            "parsed_gt_response": _safe_json_serialize(
                template.parsed_gt_response if template else None, metadata.question_id, "parsed_gt_response"
            ),
            "parsed_llm_response": _safe_json_serialize(
                template.parsed_llm_response if template else None, metadata.question_id, "parsed_llm_response"
            ),
            "template_verification_performed": template.template_verification_performed if template else False,
            "verify_result": _serialize_verification_result(
                template.verify_result if template else None, metadata.question_id, "verify_result"
            ),
            "verify_granular_result": _serialize_verification_result(
                template.verify_granular_result if template else None, metadata.question_id, "verify_granular_result"
            ),
            "answering_system_prompt": metadata.answering_system_prompt or "",
            "parsing_system_prompt": metadata.parsing_system_prompt or "",
            "embedding_check_performed": template.embedding_check_performed if template else False,
            "embedding_similarity_score": template.embedding_similarity_score or "" if template else "",
            "embedding_override_applied": template.embedding_override_applied if template else False,
            "embedding_model_used": template.embedding_model_used or "" if template else "",
            "answering_mcp_servers": _safe_json_serialize(
                template.answering_mcp_servers if template else None, metadata.question_id, "answering_mcp_servers"
            ),
            "usage_metadata": _safe_json_serialize(
                template.usage_metadata if template else None, metadata.question_id, "usage_metadata"
            )
            if template and template.usage_metadata
            else "",
            "agent_metrics": _safe_json_serialize(
                template.agent_metrics if template else None, metadata.question_id, "agent_metrics"
            )
            if template and template.agent_metrics
            else "",
            # Rubric fields
            "rubric_evaluation_performed": rubric.rubric_evaluation_performed if rubric else False,
            "metric_trait_confusion_lists": _safe_json_serialize(
                rubric.metric_trait_confusion_lists if rubric else None,
                metadata.question_id,
                "metric_trait_confusion_lists",
            ),
            "metric_trait_metrics": _safe_json_serialize(
                rubric.metric_trait_scores if rubric else None, metadata.question_id, "metric_trait_metrics"
            ),
            # Deep-judgment fields
            "deep_judgment_mode": deep_judgment.deep_judgment_mode if deep_judgment else None,
            "deep_judgment_performed": deep_judgment.deep_judgment_performed if deep_judgment else False,
            "extracted_excerpts": _safe_json_serialize(
                deep_judgment.extracted_excerpts if deep_judgment else None, metadata.question_id, "extracted_excerpts"
            ),
            "attribute_reasoning": _safe_json_serialize(
                deep_judgment.attribute_reasoning if deep_judgment else None,
                metadata.question_id,
                "attribute_reasoning",
            ),
            "deep_judgment_stages_completed": _safe_json_serialize(
                deep_judgment.deep_judgment_stages_completed if deep_judgment else None,
                metadata.question_id,
                "deep_judgment_stages_completed",
            ),
            "deep_judgment_model_calls": deep_judgment.deep_judgment_model_calls if deep_judgment else 0,
            "deep_judgment_excerpt_retry_count": deep_judgment.deep_judgment_excerpt_retry_count
            if deep_judgment
            else 0,
            "attributes_without_excerpts": _safe_json_serialize(
                deep_judgment.attributes_without_excerpts if deep_judgment else None,
                metadata.question_id,
                "attributes_without_excerpts",
            ),
            "deep_judgment_search_enabled": deep_judgment.deep_judgment_search_enabled if deep_judgment else False,
            "hallucination_risk_assessment": _safe_json_serialize(
                deep_judgment.hallucination_risk_assessment if deep_judgment else None,
                metadata.question_id,
                "hallucination_risk_assessment",
            ),
            # Deep-judgment rubric fields
            "deep_judgment_rubric_performed": deep_judgment_rubric.deep_judgment_rubric_performed
            if deep_judgment_rubric
            else False,
            "extracted_rubric_excerpts": _safe_json_serialize(
                deep_judgment_rubric.extracted_rubric_excerpts if deep_judgment_rubric else None,
                metadata.question_id,
                "extracted_rubric_excerpts",
            ),
            "rubric_trait_reasoning": _safe_json_serialize(
                deep_judgment_rubric.rubric_trait_reasoning if deep_judgment_rubric else None,
                metadata.question_id,
                "rubric_trait_reasoning",
            ),
            "deep_judgment_rubric_scores": _safe_json_serialize(
                deep_judgment_rubric.deep_judgment_rubric_scores if deep_judgment_rubric else None,
                metadata.question_id,
                "deep_judgment_rubric_scores",
            ),
            "standard_rubric_scores": _safe_json_serialize(
                deep_judgment_rubric.standard_rubric_scores if deep_judgment_rubric else None,
                metadata.question_id,
                "standard_rubric_scores",
            ),
            "trait_metadata": _safe_json_serialize(
                deep_judgment_rubric.trait_metadata if deep_judgment_rubric else None,
                metadata.question_id,
                "trait_metadata",
            ),
            "traits_without_valid_excerpts": _safe_json_serialize(
                deep_judgment_rubric.traits_without_valid_excerpts if deep_judgment_rubric else None,
                metadata.question_id,
                "traits_without_valid_excerpts",
            ),
            "rubric_hallucination_risk_assessment": _safe_json_serialize(
                deep_judgment_rubric.rubric_hallucination_risk_assessment if deep_judgment_rubric else None,
                metadata.question_id,
                "rubric_hallucination_risk_assessment",
            ),
            "total_deep_judgment_model_calls": deep_judgment_rubric.total_deep_judgment_model_calls
            if deep_judgment_rubric
            else 0,
            "total_traits_evaluated": deep_judgment_rubric.total_traits_evaluated if deep_judgment_rubric else 0,
            "total_excerpt_retries": deep_judgment_rubric.total_excerpt_retries if deep_judgment_rubric else 0,
            # Export metadata
            "export_timestamp": export_timestamp,
            "karenina_version": karenina_version,
            "job_id": job.job_id,
        }

        # Add global rubric trait values from all trait score dicts
        if rubric:
            # Merge all trait scores into a unified dict for CSV export
            merged_traits: dict[str, Any] = {}
            if rubric.llm_trait_scores:
                merged_traits.update(rubric.llm_trait_scores)
            if rubric.regex_trait_scores:
                merged_traits.update(rubric.regex_trait_scores)
            if rubric.callable_trait_scores:
                merged_traits.update(rubric.callable_trait_scores)
            if rubric.metric_trait_scores:
                merged_traits.update(rubric.metric_trait_scores)

            # Use pre-computed set for faster membership testing
            for trait in global_traits:
                row[f"rubric_{trait}"] = str(merged_traits.get(trait, ""))
        else:
            # Set all global traits to empty when no rubric data
            for trait in global_traits:
                row[f"rubric_{trait}"] = ""

        # Add question-specific rubrics as JSON (optimized)
        if question_specific_traits_set:
            if rubric and merged_traits:
                # Use dictionary comprehension for better performance
                question_specific_rubrics = {
                    trait: merged_traits[trait] for trait in question_specific_traits_set if trait in merged_traits
                }
            else:
                question_specific_rubrics = {}

            # Safe JSON serialization with error handling
            serialized = _safe_json_serialize(
                question_specific_rubrics, metadata.question_id, "question_specific_rubrics"
            )
            row["question_specific_rubrics"] = serialized if serialized else "{}"

        writer.writerow(row)

    # Log completion summary
    result_count = len(results)
    logger.info("CSV export completed successfully for %d results", result_count)

    return output.getvalue()

export_verification_results_json_stream

export_verification_results_json_stream(
    job: VerificationJob,
    results_iter: Iterable[VerificationResult],
    global_rubric: HasTraitNames | None = None,
    *,
    is_complete: bool = False,
    scenario_results: Iterable[Any] | None = None,
    out_path: Path,
) -> None

Stream the v2.2 JSON export directly to out_path.

Writes to out_path.with_suffix(out_path.suffix + '.partial') and atomically renames it into place on success. On any exception raised by the iterator or I/O, removes the .partial file and reraises.

The on-disk schema matches the v2.2 export format byte-for-byte at the top-level keys and per-result fields; only whitespace differs (compact outer + one compact JSON line per result, separated by ",\n").

Parameters:

Name	Type	Description	Default
job	VerificationJob	The verification job whose metadata heads the export.	required
results_iter	Iterable[VerificationResult]	Single-pass iterable over VerificationResult. Lists, generators, and JSONL-backed iterators all work.	required
global_rubric	HasTraitNames | None	Optional rubric folded into shared_data. Must expose either a model_dump() method (Pydantic) or get_trait_names.	None
is_complete	bool	Written into job_summary.is_complete so readers can distinguish final exports from in-progress snapshots.	False
out_path	Path	Target path (required kwarg, no default).	required

Source code in src/karenina/benchmark/verification/stages/helpers/results_exporter.py

def export_verification_results_json_stream(
    job: VerificationJob,
    results_iter: Iterable[VerificationResult],
    global_rubric: HasTraitNames | None = None,
    *,
    is_complete: bool = False,
    scenario_results: Iterable[Any] | None = None,
    out_path: Path,
) -> None:
    """Stream the v2.2 JSON export directly to out_path.

    Writes to ``out_path.with_suffix(out_path.suffix + '.partial')`` and
    atomically renames it into place on success. On any exception raised
    by the iterator or I/O, removes the ``.partial`` file and reraises.

    The on-disk schema matches the v2.2 export format byte-for-byte at
    the top-level keys and per-result fields; only whitespace differs
    (compact outer + one compact JSON line per result, separated by
    ",\\n").

    Args:
        job: The verification job whose metadata heads the export.
        results_iter: Single-pass iterable over VerificationResult. Lists,
            generators, and JSONL-backed iterators all work.
        global_rubric: Optional rubric folded into shared_data. Must expose
            either a ``model_dump()`` method (Pydantic) or ``get_trait_names``.
        is_complete: Written into ``job_summary.is_complete`` so readers
            can distinguish final exports from in-progress snapshots.
        out_path: Target path (required kwarg, no default).

    Raises:
        Propagates any exception from ``results_iter`` or disk I/O after
        cleaning up the ``.partial`` sidecar.
    """
    rubric_definition = None
    if global_rubric is not None:
        if hasattr(global_rubric, "model_dump"):
            rubric_definition = global_rubric.model_dump(mode="json", exclude_unset=True)
        else:
            rubric_definition = {"trait_names": global_rubric.get_trait_names()}

    start_time = job.start_time
    end_time = job.end_time
    total_duration = end_time - start_time if (end_time is not None and start_time is not None) else None

    header: dict[str, Any] = {
        "format_version": "2.2",
        "metadata": {
            "export_timestamp": time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime()),
            "karenina_version": get_karenina_version(),
            "job_id": job.job_id,
            "verification_config": {
                "answering_models": [
                    {
                        "provider": m.model_provider,
                        "name": m.model_name,
                        "temperature": m.temperature,
                        "interface": m.interface,
                    }
                    for m in job.config.answering_models
                ],
                "parsing_models": [
                    {
                        "provider": m.model_provider,
                        "name": m.model_name,
                        "temperature": m.temperature,
                        "interface": m.interface,
                    }
                    for m in job.config.parsing_models
                ],
            },
            "job_summary": {
                "total_questions": job.total_questions,
                "successful_count": job.successful_count,
                "failed_count": job.failed_count,
                "start_time": start_time,
                "end_time": end_time,
                "total_duration": total_duration,
                "is_complete": is_complete,
            },
        },
        "shared_data": {
            "rubric_definition": rubric_definition,
        },
    }

    header_str = json.dumps(header, ensure_ascii=False)
    assert header_str.endswith("}"), "json.dumps of a dict must end with }"

    with atomic_writer(out_path) as f:
        f.write(header_str[:-1])  # drop trailing "}"
        f.write(',"results":[\n')
        first = True
        for result in results_iter:
            if not first:
                f.write(",\n")
            f.write(json.dumps(result.model_dump(mode="json"), ensure_ascii=False))
            first = False
        f.write("\n]")
        if scenario_results is not None:
            f.write(',"scenario_results":[\n')
            first = True
            for result in scenario_results:
                if not first:
                    f.write(",\n")
                payload = result.model_dump(mode="json") if hasattr(result, "model_dump") else result
                f.write(json.dumps(payload, ensure_ascii=False, default=str))
                first = False
            f.write("\n]")
        f.write("}\n")

format_trace_messages

format_trace_messages(
    messages: Iterable[Message | Mapping[str, Any]],
) -> str

Format stored message records using Karenina's canonical trace format.

Parameters:

Name	Type	Description	Default
messages	Iterable[Message | Mapping[str, Any]]	Live Message objects or dictionaries from a stored verification result's template.trace_messages field.	required

Returns:

Type	Description
str	The canonical raw trace, including assistant text, tool calls, and tool
str	results.

Source code in src/karenina/benchmark/trace.py

def format_trace_messages(messages: Iterable[Message | Mapping[str, Any]]) -> str:
    """Format stored message records using Karenina's canonical trace format.

    Args:
        messages: Live ``Message`` objects or dictionaries from a stored
            verification result's ``template.trace_messages`` field.

    Returns:
        The canonical raw trace, including assistant text, tool calls, and tool
        results.
    """
    hydrated = [message if isinstance(message, Message) else Message.from_dict(dict(message)) for message in messages]
    return messages_to_raw_trace(hydrated)

managed_run_directory

managed_run_directory(
    root: Path,
    label: str,
    *,
    benchmark_path: str | Path,
    config: VerificationConfig,
    run_name: str | None = None,
    selected_question_count: int | None = None,
    now: datetime | None = None,
    resume: bool = False,
) -> Iterator[RunDirectory]

Create a run directory and maintain its terminal manifest status.

Parameters:

Name	Type	Description	Default
root	Path	Parent directory for verification runs.	required
label	str	Filesystem-safe reviewer-facing run label.	required
benchmark_path	str | Path	Benchmark source recorded for provenance.	required
config	VerificationConfig	Verification configuration to describe and sanitize.	required
run_name	str | None	Result metadata run name. Defaults to label.	None
selected_question_count	int | None	Number of selected questions.	None
now	datetime | None	Creation timestamp override.	None
resume	bool	Reopen the timestamped directory when it already exists.	False

Yields:

Type	Description
RunDirectory	The running directory descriptor.

Source code in src/karenina/benchmark/run_artifacts.py

@contextmanager
def managed_run_directory(
    root: Path,
    label: str,
    *,
    benchmark_path: str | Path,
    config: VerificationConfig,
    run_name: str | None = None,
    selected_question_count: int | None = None,
    now: datetime | None = None,
    resume: bool = False,
) -> Iterator[RunDirectory]:
    """Create a run directory and maintain its terminal manifest status.

    Args:
        root: Parent directory for verification runs.
        label: Filesystem-safe reviewer-facing run label.
        benchmark_path: Benchmark source recorded for provenance.
        config: Verification configuration to describe and sanitize.
        run_name: Result metadata run name. Defaults to ``label``.
        selected_question_count: Number of selected questions.
        now: Creation timestamp override.
        resume: Reopen the timestamped directory when it already exists.

    Yields:
        The running directory descriptor.
    """
    run = create_run_directory(
        root,
        label,
        benchmark_path=benchmark_path,
        config=config,
        run_name=run_name,
        selected_question_count=selected_question_count,
        now=now,
        resume=resume,
    )
    try:
        yield run
    except KeyboardInterrupt:
        run.update_status("interrupted")
        raise
    except BaseException as exc:
        run.update_status("failed", error=exc)
        raise
    else:
        run.update_status("completed")

mask_graphql_schema_messages

mask_graphql_schema_messages(trace: str) -> MaskStats

Mask GraphQL schema introspection responses in tool messages.

Source code in src/karenina/benchmark/verification/utils/trace_masking.py

def mask_graphql_schema_messages(trace: str) -> MaskStats:
    """Mask GraphQL schema introspection responses in tool messages."""
    return mask_tool_messages(
        trace,
        should_mask=is_graphql_schema_payload,
        describe=_describe_graphql_schema,
    )

mask_run_configuration

mask_run_configuration(
    configuration: Mapping[str, JsonValue],
) -> dict[str, JsonValue]

Return a JSON configuration snapshot with credential fields masked.

Parameters:

Name	Type	Description	Default
configuration	Mapping[str, JsonValue]	Configuration mapping to sanitize.	required

Returns:

Type	Description
dict[str, JsonValue]	A deep copy safe to write into a run manifest.

Source code in src/karenina/benchmark/run_artifacts.py

def mask_run_configuration(configuration: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    """Return a JSON configuration snapshot with credential fields masked.

    Args:
        configuration: Configuration mapping to sanitize.

    Returns:
        A deep copy safe to write into a run manifest.
    """
    return {str(key): _mask_configuration(value, key=str(key)) for key, value in configuration.items()}

repair_results_export

repair_results_export(
    benchmark: _RepairBenchmark,
    source_path: Path,
    config: VerificationConfig,
    selection: RepairSelection
    | Callable[[VerificationResult], bool],
    *,
    mode: RepairMode = "replay",
    output_path: Path | None = None,
    dry_run: bool = False,
    now: datetime | None = None,
) -> RepairOutcome

Rerun and splice selected rows of a completed JSON export.

Parameters:

Name	Type	Description	Default
benchmark	_RepairBenchmark	Loaded Benchmark exposing run_verification.	required
source_path	Path	Completed result export to load with ResultsIOManager.	required
config	VerificationConfig	Full model configuration corresponding to the source run.	required
selection	RepairSelection | Callable[[VerificationResult], bool]	Declarative filters or a Python predicate.	required
mode	RepairMode	replay reuses answer traces. live regenerates answers.	'replay'
output_path	Path | None	Destination. Defaults to replacing source_path safely.	None
dry_run	bool	Select and report rows without model calls or file changes.	False
now	datetime | None	Timestamp override for backup and provenance names.	None

Returns:

Type	Description
RepairOutcome	Repair paths, counts, mode, and selected identities.

Source code in src/karenina/benchmark/verification/repair.py

def repair_results_export(
    benchmark: _RepairBenchmark,
    source_path: Path,
    config: VerificationConfig,
    selection: RepairSelection | Callable[[VerificationResult], bool],
    *,
    mode: RepairMode = "replay",
    output_path: Path | None = None,
    dry_run: bool = False,
    now: datetime | None = None,
) -> RepairOutcome:
    """Rerun and splice selected rows of a completed JSON export.

    Args:
        benchmark: Loaded ``Benchmark`` exposing ``run_verification``.
        source_path: Completed result export to load with ``ResultsIOManager``.
        config: Full model configuration corresponding to the source run.
        selection: Declarative filters or a Python predicate.
        mode: ``replay`` reuses answer traces. ``live`` regenerates answers.
        output_path: Destination. Defaults to replacing ``source_path`` safely.
        dry_run: Select and report rows without model calls or file changes.
        now: Timestamp override for backup and provenance names.

    Returns:
        Repair paths, counts, mode, and selected identities.
    """
    source_path = Path(source_path)
    original_iter = cast(Iterator[VerificationResult], ResultsIOManager.iter_from_json(source_path))
    original = list(original_iter)
    _index_unique(original)
    selected = select_repair_rows(original, selection)
    if not selected:
        raise ValueError("Repair selection matched no result rows")
    selected_keys = [_key_string(ResultRowKey.from_result(row)) for row in selected]
    destination = Path(output_path) if output_path is not None else source_path
    if dry_run:
        return RepairOutcome(
            source_path=source_path,
            output_path=destination,
            backup_path=None,
            provenance_path=None,
            selected_count=len(selected),
            replaced_count=0,
            mode=mode,
            dry_run=True,
            selected_keys=selected_keys,
        )

    run_names = {row.metadata.run_name for row in selected}
    if len(run_names) != 1:
        raise ValueError("Selected repair rows must share one run name")
    run_name = next(iter(run_names)) or "repaired_run"
    targeted_config = _targeted_config(config, selected, mode)
    question_ids = sorted({row.metadata.question_id for row in selected})
    repaired = benchmark.run_verification(
        config=targeted_config,
        question_ids=question_ids,
        run_name=run_name,
    )
    repaired_rows = list(repaired.results)
    if len(repaired_rows) != len(selected):
        raise ValueError(f"Repair produced {len(repaired_rows)} rows for {len(selected)} selected rows")
    merged = splice_repaired_rows(original, repaired_rows)

    timestamp = (now or datetime.now(UTC)).astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
    backup_path: Path | None = None
    if destination.resolve() == source_path.resolve():
        backup_path = source_path.with_name(f"{source_path.name}.backup_{timestamp}")
        if backup_path.exists():
            raise FileExistsError(backup_path)
        shutil.copy2(source_path, backup_path)
    destination.parent.mkdir(parents=True, exist_ok=True)
    source_checksum = _sha256(source_path)
    _export(destination, merged, config, run_name)
    provenance_path = destination.with_name(f"{destination.name}.repair_{timestamp}.json")
    provenance = {
        "schema_version": "1.0",
        "timestamp": timestamp,
        "mode": mode,
        "source_path": str(source_path),
        "source_sha256": source_checksum,
        "output_path": str(destination),
        "backup_path": str(backup_path) if backup_path is not None else None,
        "selected_count": len(selected),
        "replaced_count": len(repaired_rows),
        "selected_keys": selected_keys,
    }
    atomic_write(provenance_path, json.dumps(provenance, indent=2, sort_keys=True) + "\n")
    return RepairOutcome(
        source_path=source_path,
        output_path=destination,
        backup_path=backup_path,
        provenance_path=provenance_path,
        selected_count=len(selected),
        replaced_count=len(repaired_rows),
        mode=mode,
        dry_run=False,
        selected_keys=selected_keys,
    )

run_question_verification

run_question_verification(
    question_id: str,
    question_text: str,
    template_code: str,
    answering_model: ModelConfig,
    parsing_model: ModelConfig,
    run_name: str | None = None,
    replicate: int | None = None,
    rubric: Rubric | None = None,
    dynamic_rubric: DynamicRubric | None = None,
    keywords: list[str] | None = None,
    raw_answer: str | None = None,
    few_shot_examples: list[dict[str, str]] | None = None,
    few_shot_enabled: bool = False,
    abstention_enabled: bool = False,
    sufficiency_enabled: bool = False,
    deep_judgment_mode: str = "disabled",
    rubric_evaluation_strategy: str = "batch",
    deep_judgment_max_excerpts_per_attribute: int = DEFAULT_DEEP_JUDGMENT_MAX_EXCERPTS,
    deep_judgment_fuzzy_match_threshold: float = DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD,
    deep_judgment_excerpt_retry_attempts: int = DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS,
    deep_judgment_search_enabled: bool = False,
    deep_judgment_search_tool: str | Any = "tavily",
    deep_judgment_rubric_mode: str = "disabled",
    deep_judgment_rubric_global_excerpts: bool = True,
    deep_judgment_rubric_config: DeepJudgmentRubricCustomConfig
    | None = None,
    deep_judgment_rubric_max_excerpts_default: int = DEFAULT_RUBRIC_MAX_EXCERPTS,
    deep_judgment_rubric_fuzzy_match_threshold_default: float = DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD,
    deep_judgment_rubric_excerpt_retry_attempts_default: int = DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS,
    deep_judgment_rubric_search_enabled: bool = False,
    deep_judgment_rubric_search_tool: str | Any = "tavily",
    evaluation_mode: str = "template_only",
    cached_answer_data: dict[str, Any] | None = None,
    prompt_config: PromptConfig | None = None,
    use_full_trace_for_template: bool = False,
    use_full_trace_for_rubric: bool = True,
    allow_partial_trace_scoring: bool = True,
    agentic_parsing: bool = False,
    agentic_parsing_trigger: str = "always",
    agentic_judge_context: str = "workspace_only",
    agentic_parsing_max_turns: int = 15,
    agentic_parsing_timeout: float = 120.0,
    agentic_parsing_materialize_trace: bool = False,
    agentic_parsing_persist_trace: bool = False,
    workspace_root: Path | None = None,
    workspace_copy: bool = True,
    workspace_cleanup: bool = True,
    workspace_output_mode: str = "none",
    workspace_output_dir: Path | None = None,
    workspace_output_exclude_patterns: list[str]
    | None = None,
    question_workspace_path: str | None = None,
    agentic_rubric_strategy: str = "individual",
    agentic_rubric_parallel: bool = False,
    embedding_check_enabled: bool = False,
    embedding_check_model: str | None = None,
    embedding_check_threshold: float | None = None,
    trait_provenance: dict[str, str] | None = None,
    custom_error_patterns: list[ErrorPatternConfig]
    | None = None,
    replay_store: Any = None,
    replay_parse_on_hydration_mismatch: str = "fall_through",
    task_eval_mode: bool = False,
    callable_registry: dict[str, RubricCallable]
    | None = None,
) -> VerificationResult

Run verification for a single question with specific answering and parsing models.

This function uses a stage-based pipeline architecture for modularity and testability. Each verification step (validation, generation, parsing, verification, etc.) is implemented as a discrete stage that can be independently tested and configured.

Parameters:

Name	Type	Description	Default
question_id	str	Unique identifier for the question. For manual interface, this MUST be a 32-character hexadecimal MD5 hash (generated during question extraction).	required
question_text	str	The question to ask the LLM	required
template_code	str	Python code defining the Answer class	required
answering_model	ModelConfig	Configuration for the answering model	required
parsing_model	ModelConfig	Configuration for the parsing model	required
run_name	str | None	Optional run name for tracking	None
replicate	int | None	Optional replicate number for repeated runs of the same question	None
rubric	Rubric | None	Optional rubric for qualitative evaluation	None
keywords	list[str] | None	Optional keywords associated with the question	None
few_shot_examples	list[dict[str, str]] | None	Optional list of question-answer pairs for few-shot prompting	None
few_shot_enabled	bool	Whether to use few-shot prompting (disabled by default)	False
abstention_enabled	bool	Whether to enable abstention detection	False
sufficiency_enabled	bool	Whether to enable trace sufficiency detection	False
deep_judgment_mode	str	Template deep-judgment mode ("disabled", "reasoning_only", "full")	'disabled'
rubric_evaluation_strategy	str	Strategy for evaluating LLM rubric traits: - "batch": All traits evaluated in single LLM call (default, efficient) - "sequential": Traits evaluated one-by-one (reliable, more expensive)	'batch'
callable_registry	dict[str, RubricCallable] | None	Runtime callable overrides keyed by callable trait name.	None
deep_judgment_max_excerpts_per_attribute	int	Max excerpts per attribute (deep-judgment)	DEFAULT_DEEP_JUDGMENT_MAX_EXCERPTS
deep_judgment_fuzzy_match_threshold	float	Similarity threshold for excerpts (deep-judgment)	DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD
deep_judgment_excerpt_retry_attempts	int	Retry attempts for excerpt validation (deep-judgment)	DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS
deep_judgment_search_enabled	bool	Whether to enable search enhancement (deep-judgment)	False
deep_judgment_search_tool	str | Any	Search tool name or callable (deep-judgment)	'tavily'
evaluation_mode	str	Evaluation mode determining which stages run: - "template_only": Template verification only (default) - "template_and_rubric": Template verification + rubric evaluation - "rubric_only": Skip template, only evaluate rubrics on raw response	'template_only'
cached_answer_data	dict[str, Any] | None	Optional cached answer data from previous generation. If provided, the GenerateAnswerStage will skip LLM invocation and use this cached data. Used to share answers across multiple judges.	None

Returns:

Type	Description
VerificationResult	VerificationResult with all details and optional rubric scores

Raises:

Type	Description
ValueError	If question_id is not a valid MD5 hash when using manual interface
RuntimeError	If stage orchestration fails critically

Source code in src/karenina/benchmark/verification/runner.py

def run_single_model_verification(
    question_id: str,
    question_text: str,
    template_code: str,
    answering_model: ModelConfig,
    parsing_model: ModelConfig,
    run_name: str | None = None,
    replicate: int | None = None,
    rubric: Rubric | None = None,
    dynamic_rubric: DynamicRubric | None = None,
    keywords: list[str] | None = None,
    raw_answer: str | None = None,
    few_shot_examples: list[dict[str, str]] | None = None,
    few_shot_enabled: bool = False,
    abstention_enabled: bool = False,
    sufficiency_enabled: bool = False,
    deep_judgment_mode: str = "disabled",
    rubric_evaluation_strategy: str = "batch",
    deep_judgment_max_excerpts_per_attribute: int = DEFAULT_DEEP_JUDGMENT_MAX_EXCERPTS,
    deep_judgment_fuzzy_match_threshold: float = DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD,
    deep_judgment_excerpt_retry_attempts: int = DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS,
    deep_judgment_search_enabled: bool = False,
    deep_judgment_search_tool: str | Any = "tavily",
    # Deep-judgment rubric configuration (NEW)
    deep_judgment_rubric_mode: str = "disabled",
    deep_judgment_rubric_global_excerpts: bool = True,
    deep_judgment_rubric_config: DeepJudgmentRubricCustomConfig | None = None,
    deep_judgment_rubric_max_excerpts_default: int = DEFAULT_RUBRIC_MAX_EXCERPTS,
    deep_judgment_rubric_fuzzy_match_threshold_default: float = DEFAULT_DEEP_JUDGMENT_FUZZY_THRESHOLD,
    deep_judgment_rubric_excerpt_retry_attempts_default: int = DEFAULT_DEEP_JUDGMENT_RETRY_ATTEMPTS,
    deep_judgment_rubric_search_enabled: bool = False,
    deep_judgment_rubric_search_tool: str | Any = "tavily",
    evaluation_mode: str = "template_only",
    cached_answer_data: dict[str, Any] | None = None,
    # Prompt configuration
    prompt_config: PromptConfig | None = None,
    # Trace filtering configuration (MCP Agent Evaluation)
    use_full_trace_for_template: bool = False,
    use_full_trace_for_rubric: bool = True,
    allow_partial_trace_scoring: bool = True,
    # Agentic parsing configuration
    agentic_parsing: bool = False,
    agentic_parsing_trigger: str = "always",
    agentic_judge_context: str = "workspace_only",
    agentic_parsing_max_turns: int = 15,
    agentic_parsing_timeout: float = 120.0,
    agentic_parsing_materialize_trace: bool = False,
    agentic_parsing_persist_trace: bool = False,
    workspace_root: Path | None = None,
    workspace_copy: bool = True,
    workspace_cleanup: bool = True,
    workspace_output_mode: str = "none",
    workspace_output_dir: Path | None = None,
    workspace_output_exclude_patterns: list[str] | None = None,
    question_workspace_path: str | None = None,
    # Agentic rubric evaluation configuration
    agentic_rubric_strategy: str = "individual",
    agentic_rubric_parallel: bool = False,
    # Embedding check configuration
    embedding_check_enabled: bool = False,
    embedding_check_model: str | None = None,
    embedding_check_threshold: float | None = None,
    # Trait provenance
    trait_provenance: dict[str, str] | None = None,
    # Error classification
    custom_error_patterns: list[ErrorPatternConfig] | None = None,
    # Replay layer
    replay_store: Any = None,
    replay_parse_on_hydration_mismatch: str = "fall_through",
    # TaskEval signal (set by TaskEval entry; suppresses QUESTION slot in rubric prompts)
    task_eval_mode: bool = False,
    # Runtime callable overrides (used by TaskEval)
    callable_registry: dict[str, RubricCallable] | None = None,
) -> VerificationResult:
    """
    Run verification for a single question with specific answering and parsing models.

    This function uses a stage-based pipeline architecture for modularity and testability.
    Each verification step (validation, generation, parsing, verification, etc.) is
    implemented as a discrete stage that can be independently tested and configured.

    Args:
        question_id: Unique identifier for the question. For manual interface, this MUST be
                    a 32-character hexadecimal MD5 hash (generated during question extraction).
        question_text: The question to ask the LLM
        template_code: Python code defining the Answer class
        answering_model: Configuration for the answering model
        parsing_model: Configuration for the parsing model
        run_name: Optional run name for tracking
        replicate: Optional replicate number for repeated runs of the same question
        rubric: Optional rubric for qualitative evaluation
        keywords: Optional keywords associated with the question
        few_shot_examples: Optional list of question-answer pairs for few-shot prompting
        few_shot_enabled: Whether to use few-shot prompting (disabled by default)
        abstention_enabled: Whether to enable abstention detection
        sufficiency_enabled: Whether to enable trace sufficiency detection
        deep_judgment_mode: Template deep-judgment mode ("disabled", "reasoning_only", "full")
        rubric_evaluation_strategy: Strategy for evaluating LLM rubric traits:
            - "batch": All traits evaluated in single LLM call (default, efficient)
            - "sequential": Traits evaluated one-by-one (reliable, more expensive)
        callable_registry: Runtime callable overrides keyed by callable trait name.
        deep_judgment_max_excerpts_per_attribute: Max excerpts per attribute (deep-judgment)
        deep_judgment_fuzzy_match_threshold: Similarity threshold for excerpts (deep-judgment)
        deep_judgment_excerpt_retry_attempts: Retry attempts for excerpt validation (deep-judgment)
        deep_judgment_search_enabled: Whether to enable search enhancement (deep-judgment)
        deep_judgment_search_tool: Search tool name or callable (deep-judgment)
        evaluation_mode: Evaluation mode determining which stages run:
            - "template_only": Template verification only (default)
            - "template_and_rubric": Template verification + rubric evaluation
            - "rubric_only": Skip template, only evaluate rubrics on raw response
        cached_answer_data: Optional cached answer data from previous generation.
            If provided, the GenerateAnswerStage will skip LLM invocation and use
            this cached data. Used to share answers across multiple judges.

    Returns:
        VerificationResult with all details and optional rubric scores

    Raises:
        ValueError: If question_id is not a valid MD5 hash when using manual interface
        RuntimeError: If stage orchestration fails critically
    """
    # Compute template_id from template_code (composite key component)
    template_id = generate_template_id(template_code)

    # Initialize verification context with all parameters
    context = VerificationContext(
        # Identity & Metadata
        question_id=question_id,
        template_id=template_id,
        question_text=question_text,
        template_code=template_code,
        # Configuration
        answering_model=answering_model,
        parsing_model=parsing_model,
        rubric=rubric,
        dynamic_rubric=dynamic_rubric,
        keywords=keywords,
        raw_answer=raw_answer,
        # Run Metadata
        run_name=run_name,
        replicate=replicate,
        # Feature Flags
        few_shot_enabled=few_shot_enabled,
        abstention_enabled=abstention_enabled,
        sufficiency_enabled=sufficiency_enabled,
        deep_judgment_mode=deep_judgment_mode,
        # Rubric Configuration
        rubric_evaluation_strategy=rubric_evaluation_strategy,
        callable_registry=callable_registry,
        # Deep-Judgment Configuration
        deep_judgment_max_excerpts_per_attribute=deep_judgment_max_excerpts_per_attribute,
        deep_judgment_fuzzy_match_threshold=deep_judgment_fuzzy_match_threshold,
        deep_judgment_excerpt_retry_attempts=deep_judgment_excerpt_retry_attempts,
        deep_judgment_search_enabled=deep_judgment_search_enabled,
        deep_judgment_search_tool=deep_judgment_search_tool,
        # Deep-Judgment Rubric Configuration (NEW)
        deep_judgment_rubric_mode=deep_judgment_rubric_mode,
        deep_judgment_rubric_global_excerpts=deep_judgment_rubric_global_excerpts,
        deep_judgment_rubric_config=deep_judgment_rubric_config,
        deep_judgment_rubric_max_excerpts_default=deep_judgment_rubric_max_excerpts_default,
        deep_judgment_rubric_fuzzy_match_threshold_default=deep_judgment_rubric_fuzzy_match_threshold_default,
        deep_judgment_rubric_excerpt_retry_attempts_default=deep_judgment_rubric_excerpt_retry_attempts_default,
        deep_judgment_rubric_search_enabled=deep_judgment_rubric_search_enabled,
        deep_judgment_rubric_search_tool=deep_judgment_rubric_search_tool,
        # Few-Shot Configuration
        few_shot_examples=few_shot_examples,
        # Prompt Configuration
        prompt_config=prompt_config,
        # Trace Filtering Configuration (MCP Agent Evaluation)
        use_full_trace_for_template=use_full_trace_for_template,
        use_full_trace_for_rubric=use_full_trace_for_rubric,
        allow_partial_trace_scoring=allow_partial_trace_scoring,
        # Answer Caching
        cached_answer_data=cached_answer_data,
        # Agentic Parsing
        agentic_parsing=agentic_parsing,
        agentic_parsing_trigger=agentic_parsing_trigger,
        agentic_judge_context=agentic_judge_context,
        agentic_parsing_max_turns=agentic_parsing_max_turns,
        agentic_parsing_timeout=agentic_parsing_timeout,
        agentic_parsing_materialize_trace=agentic_parsing_materialize_trace,
        agentic_parsing_persist_trace=agentic_parsing_persist_trace,
        question_workspace_path=question_workspace_path,
        workspace_root=workspace_root,
        workspace_copy=workspace_copy,
        workspace_cleanup=workspace_cleanup,
        workspace_output_mode=workspace_output_mode,
        workspace_output_dir=workspace_output_dir,
        workspace_output_exclude_patterns=workspace_output_exclude_patterns or [],
        # Agentic Rubric
        agentic_rubric_strategy=agentic_rubric_strategy,
        agentic_rubric_parallel=agentic_rubric_parallel,
        # Embedding Check
        embedding_check_enabled=embedding_check_enabled,
        embedding_check_model=embedding_check_model,
        embedding_check_threshold=embedding_check_threshold,
        # Trait Provenance
        trait_provenance=trait_provenance,
        # Error Classification
        error_registry=_build_error_registry(custom_error_patterns or []),
        # Replay layer
        replay_store=replay_store,
        replay_parse_on_hydration_mismatch=replay_parse_on_hydration_mismatch,
        # TaskEval signal
        task_eval_mode=task_eval_mode,
    )

    # Build ModelIdentity objects for pipeline use (needed even if validation fails)
    from karenina.schemas.verification.model_identity import ModelIdentity

    answering_identity = ModelIdentity.from_model_config(answering_model, role="answering")
    parsing_identity = ModelIdentity.from_model_config(parsing_model, role="parsing")

    # Store ModelIdentity objects in context for downstream stages (e.g., finalize_result)
    context.set_artifact("answering_model_identity", answering_identity)
    context.set_artifact("parsing_model_identity", parsing_identity)

    # Store MCP server names as result field for VerificationResultTemplate
    answering_mcp_servers = list(answering_model.mcp_urls_dict.keys()) if answering_model.mcp_urls_dict else None
    context.set_result_field("answering_mcp_servers", answering_mcp_servers)

    # Warn if rubric traits are provided but evaluation_mode won't use them.
    _has_rubric_traits = rubric and (
        rubric.llm_traits
        or rubric.regex_traits
        or rubric.callable_traits
        or rubric.metric_traits
        or rubric.agentic_traits
    )
    _has_dynamic_rubric_traits = dynamic_rubric is not None and not dynamic_rubric.is_empty()
    if (_has_rubric_traits or _has_dynamic_rubric_traits) and evaluation_mode == "template_only":
        logger.warning(
            "Rubric traits were provided but evaluation_mode='template_only'. "
            "Rubric evaluation will be skipped. Set evaluation_mode='template_and_rubric' "
            "to evaluate rubric traits."
        )

    if evaluation_mode == "rubric_only" and not _has_rubric_traits and not _has_dynamic_rubric_traits:
        logger.warning(
            "evaluation_mode='rubric_only' but no rubric traits provided. Rubric evaluation will produce no scores."
        )

    # Build stage orchestrator from configuration
    orchestrator = StageOrchestrator.from_config(
        rubric=rubric,
        dynamic_rubric=dynamic_rubric,
        abstention_enabled=abstention_enabled,
        sufficiency_enabled=sufficiency_enabled,
        deep_judgment_enabled=deep_judgment_mode != "disabled",
        evaluation_mode=evaluation_mode,
        agentic_parsing=agentic_parsing,
        agentic_parsing_trigger=agentic_parsing_trigger,
    )

    # Execute verification pipeline
    result = orchestrator.execute(context)

    return result

select_repair_rows

select_repair_rows(
    results: Iterable[VerificationResult],
    selection: RepairSelection
    | Callable[[VerificationResult], bool],
) -> list[VerificationResult]

Select completed rows for repair.

Parameters:

Name	Type	Description	Default
results	Iterable[VerificationResult]	Validated verification results.	required
selection	RepairSelection | Callable[[VerificationResult], bool]	Declarative filters or a Python predicate.	required

Returns:

Type	Description
list[VerificationResult]	Selected rows in source order.

Source code in src/karenina/benchmark/verification/repair.py

def select_repair_rows(
    results: Iterable[VerificationResult],
    selection: RepairSelection | Callable[[VerificationResult], bool],
) -> list[VerificationResult]:
    """Select completed rows for repair.

    Args:
        results: Validated verification results.
        selection: Declarative filters or a Python predicate.

    Returns:
        Selected rows in source order.
    """
    predicate = selection.matches if isinstance(selection, RepairSelection) else selection
    return [result for result in results if predicate(result)]

splice_repaired_rows

splice_repaired_rows(
    original: Iterable[VerificationResult],
    replacements: Iterable[VerificationResult],
) -> list[VerificationResult]

Replace exact row identities while preserving source order.

Parameters:

Name	Type	Description	Default
original	Iterable[VerificationResult]	Complete original result rows.	required
replacements	Iterable[VerificationResult]	Newly evaluated rows with matching identities.	required

Returns:

Type	Description
list[VerificationResult]	Original rows with exact matching replacements spliced in.

Raises:

Type	Description
ValueError	If either input contains duplicates or a replacement has no corresponding original row.

Source code in src/karenina/benchmark/verification/repair.py

def splice_repaired_rows(
    original: Iterable[VerificationResult],
    replacements: Iterable[VerificationResult],
) -> list[VerificationResult]:
    """Replace exact row identities while preserving source order.

    Args:
        original: Complete original result rows.
        replacements: Newly evaluated rows with matching identities.

    Returns:
        Original rows with exact matching replacements spliced in.

    Raises:
        ValueError: If either input contains duplicates or a replacement has no
            corresponding original row.
    """
    original_rows = list(original)
    original_index = _index_unique(original_rows)
    replacement_index = _index_unique(replacements)
    unknown = set(replacement_index) - set(original_index)
    if unknown:
        key = sorted(unknown, key=_key_string)[0]
        raise ValueError(f"Replacement has no original row: {_key_string(key)}")
    return [replacement_index.get(ResultRowKey.from_result(row), row) for row in original_rows]

truncate_to_token_budget

truncate_to_token_budget(
    text: str,
    max_tokens: int,
    encoding_name: str = "o200k_base",
) -> tuple[str, bool]

Middle-truncate text so its encoded form fits a token budget.

The head and tail are retained around an omission marker so both the beginning of a trace and its final messages remain available to a judge.

Parameters:

Name	Type	Description	Default
text	str	Text to constrain.	required
max_tokens	int	Positive maximum number of encoded tokens.	required
encoding_name	str	Tiktoken encoding used to count and truncate.	'o200k_base'

Returns:

Type	Description
tuple[str, bool]	The possibly truncated text and whether truncation occurred.

Raises:

Type	Description
ValueError	If max_tokens is not positive.

Source code in src/karenina/benchmark/verification/utils/token_budget.py

def truncate_to_token_budget(
    text: str,
    max_tokens: int,
    encoding_name: str = "o200k_base",
) -> tuple[str, bool]:
    """Middle-truncate text so its encoded form fits a token budget.

    The head and tail are retained around an omission marker so both the
    beginning of a trace and its final messages remain available to a judge.

    Args:
        text: Text to constrain.
        max_tokens: Positive maximum number of encoded tokens.
        encoding_name: Tiktoken encoding used to count and truncate.

    Returns:
        The possibly truncated text and whether truncation occurred.

    Raises:
        ValueError: If ``max_tokens`` is not positive.
    """
    if max_tokens <= 0:
        raise ValueError("max_tokens must be positive")

    encoding = _encoding(encoding_name)
    tokens = encoding.encode(text)
    if len(tokens) <= max_tokens:
        return text, False

    omitted = len(tokens)
    marker_tokens: list[int] = []
    keep_count = 0
    for _ in range(4):
        marker = f"\n[truncated: {omitted} tokens omitted]\n"
        marker_tokens = encoding.encode(marker)
        keep_count = max_tokens - len(marker_tokens)
        if keep_count <= 0:
            return encoding.decode(marker_tokens[:max_tokens]), True
        updated_omitted = len(tokens) - keep_count
        if updated_omitted == omitted:
            break
        omitted = updated_omitted

    head_count = (keep_count + 1) // 2
    tail_count = keep_count - head_count
    result_tokens = tokens[:head_count] + marker_tokens
    if tail_count:
        result_tokens += tokens[-tail_count:]
    return encoding.decode(result_tokens), True

validate_answer_template

validate_answer_template(
    template_code: str,
) -> tuple[bool, str | None, type | None]

Validate that template code defines a proper Answer class.

Discovers the answer class by scanning for the leaf BaseAnswer subclass, supporting custom class names (not just "Answer").

Parameters:

Name	Type	Description	Default
template_code	str	Python source code defining a BaseAnswer subclass.	required

Returns:

Type	Description
tuple[bool, str | None, type | None]	Tuple of (is_valid, error_message, Answer_class).

Source code in src/karenina/benchmark/verification/utils/template_validation.py

def validate_answer_template(template_code: str) -> tuple[bool, str | None, type | None]:
    """Validate that template code defines a proper Answer class.

    Discovers the answer class by scanning for the leaf BaseAnswer subclass,
    supporting custom class names (not just "Answer").

    Args:
        template_code: Python source code defining a BaseAnswer subclass.

    Returns:
        Tuple of (is_valid, error_message, Answer_class).
    """
    try:
        global_ns = _build_exec_namespace()
        local_ns: dict[str, Any] = {}

        exec(template_code, global_ns, local_ns)

        # Discover the answer class (supports custom names)
        try:
            Answer = find_answer_class(local_ns)
        except ValueError as e:
            return False, str(e), None

        # Store the template code for exec-created classes
        # (since inspect.getsource() won't work for them)
        Answer._source_code = template_code  # type: ignore[attr-defined]

        if not inspect.isclass(Answer):
            return False, "Answer is not a class", None

        if not issubclass(Answer, BaseAnswer):
            return False, "Answer class must inherit from BaseAnswer", None

        # Check if it has a verify method (not required for regex-only or VerifiedField templates)
        from .template_parsing_helpers import is_regex_only_template

        has_verified_fields = bool(Answer._get_verified_fields())
        if not is_regex_only_template(Answer) and not has_verified_fields:
            if not hasattr(Answer, "verify"):
                return False, "does not have a 'verify' method", None
            if not callable(getattr(Answer, "verify", None)):
                return False, "verify must be a callable method", None

        # The 'correct' field is optional, but if present via ground_truth/model_post_init, it must be a dict
        has_init = "model_post_init" in Answer.__dict__ or "ground_truth" in Answer.__dict__
        if has_init:
            try:
                from .template_parsing_helpers import create_test_instance_from_answer_class

                test_instance, ground_truth = create_test_instance_from_answer_class(Answer)
                if ground_truth is not None and not isinstance(ground_truth, dict):
                    return False, "ground_truth/model_post_init must assign 'self.correct' as a dictionary", None
            except Exception as e:
                return False, f"Error testing ground_truth/model_post_init: {e}", None

        return True, None, Answer

    except SyntaxError as e:
        return False, f"Error executing template code: {e}", None
    except Exception as e:
        return False, f"Error executing template code: {e}", None