import type { RunSampleQuery, RunSampleSource } from './sample-source.js';
import type { RunScore, ScoreStore } from './score-store.js';
import type { ScorableRun, ScoreResult, Scorer } from './types.js';
/** One scorer blowing up on one run. Collected, never thrown — see {@link runEvaluation}. */
export interface ScorerFailure {
    runId: string;
    scorer: string;
    message: string;
}
/** What a batch did. `runsRead - runsScored` is the runs no scorer had anything to say about. */
export interface EvaluationSummary {
    /** Runs the source returned for the query. */
    runsRead: number;
    /** Distinct runs that produced at least one score. */
    runsScored: number;
    /** `(run, scorer)` pairs skipped because that scorer had already covered that run. */
    pairsSkipped: number;
    scoresRecorded: number;
    failures: ScorerFailure[];
}
export interface EvaluationOptions {
    source: RunSampleSource;
    scorers: Scorer[];
    store: ScoreStore;
    query: RunSampleQuery;
    /**
     * Score runs a scorer has already covered. Default `false`, which makes a batch resumable: an
     * interrupted backfill restarted with the same query costs nothing for the part it finished — and
     * for a model-graded scorer, bills nothing.
     */
    rescore?: boolean;
    /** Clock for {@link RunScore.scoredAt}; injectable so a test can pin it. */
    now?: () => Date;
}
/**
 * Score a batch of stored runs and persist the results — the offline path, and the one to reach for.
 *
 * A scorer that throws is recorded as a {@link ScorerFailure} and the batch carries on. One
 * malformed run, one judge that returned prose instead of a verdict, one adapter row a scorer did
 * not expect: none of those should cost you the other 499 runs of a backfill, and a failure that
 * silently became a score of 0 would be worse still — it would read as the agent's fault.
 */
export declare function runEvaluation(options: EvaluationOptions): Promise<EvaluationSummary>;
/** Assemble the persisted row: the scorer's verdict plus the provenance a summary groups by. */
export declare function toRunScore(args: {
    run: ScorableRun;
    scorer: Scorer;
    result: ScoreResult;
    scoredAt: Date;
}): RunScore;
//# sourceMappingURL=evaluate.d.ts.map