/** * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets. */ export declare const EVALUATION_FRAMEWORKS: { readonly exgentic: { readonly name: "exgentic"; readonly description: "Exgentic is an open evaluation framework for general-purpose AI agents across diverse domains and benchmarks."; readonly url: "https://github.com/Exgentic/exgentic"; }; readonly "inspect-ai": { readonly name: "inspect-ai"; readonly description: "Inspect AI is an open-source framework for large language model evaluations."; readonly url: "https://inspect.aisi.org.uk/"; }; readonly "math-arena": { readonly name: "math-arena"; readonly description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads."; readonly url: "https://github.com/eth-sri/matharena"; }; readonly mteb: { readonly name: "mteb"; readonly description: "Multimodal toolbox for evaluating embeddings and retrieval systems."; readonly url: "https://github.com/embeddings-benchmark/mteb"; }; readonly "olmocr-bench": { readonly name: "olmocr-bench"; readonly description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools."; readonly url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench"; }; readonly harbor: { readonly name: "harbor"; readonly description: "Harbor is a framework for evaluating and optimizing agents and language models."; readonly url: "https://github.com/laude-institute/harbor"; }; readonly ifstruct: { readonly name: "ifstruct"; readonly description: "IFStruct is a benchmark for structured-output compliance: whether a model produces valid JSON/YAML that follows a requested schema, scored without constrained decoding."; readonly url: "https://github.com/Liquid4All/ifstruct"; }; readonly pier: { readonly name: "pier"; readonly description: "Pier is a Harbor fork built for DeepSWE, with stronger support for CLI agents in no-internet tasks and more faithful, consistent agent trajectories."; readonly url: "https://github.com/datacurve-ai/pier"; }; readonly "redline-bench": { readonly name: "redline-bench"; readonly description: "RedlineBench measures multi-turn contract redlining: agents produce tracked-change .docx edits that are graded against attorney-authored weighted rubrics by an LLM judge panel across five dimensions. Report: https://intelligence.crosby.ai/benchmark/"; readonly url: "https://github.com/crosbylegal/redline-bench"; }; readonly archipelago: { readonly name: "archipelago"; readonly description: "Archipelago is a system for running and evaluating AI agents against MCP applications."; readonly url: "https://github.com/Mercor-Intelligence/archipelago"; }; readonly benchflow: { readonly name: "benchflow"; readonly description: "BenchFlow is an evaluation framework for AI agents on professional, skill-aware workflows. It powers SkillsBench and runs containerized agent trials with paired with-skills / without-skills configurations."; readonly url: "https://github.com/benchflow-ai/benchflow"; }; readonly "apex-evals": { readonly name: "apex-evals"; readonly description: "APEX Evals is a benchmark suite and evaluation harness for evaluating large language models."; readonly url: "https://github.com/Mercor-Intelligence/apex-evals"; }; readonly "screenspot-pro": { readonly name: "screenspot-pro"; readonly description: "ScreenSpot-Pro is a GUI grounding benchmark designed to evaluate how well AI agents can locate and identify UI elements across professional software applications in high-resolution screenshots, covering 1,585 annotated images from 26 professional tools."; readonly url: "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding"; }; readonly "swe-bench": { readonly name: "swe-bench"; readonly description: "SWE Bench is a framework for evaluating the performance of LLMs on software engineering tasks."; readonly url: "https://github.com/swe-bench/swe-bench"; }; readonly "swe-bench-pro": { readonly name: "swe-bench-pro"; readonly description: "SWE-Bench Pro is a challenging benchmark evaluating LLMs/Agents on long-horizon software engineering tasks."; readonly url: "https://github.com/scaleapi/SWE-bench_Pro-os"; }; readonly "nemo-evaluator": { readonly name: "nemo-evaluator"; readonly description: "NeMo Evaluator is an open-source platform for robust, reproducible, and scalable evaluation of Large Language Models across 100+ benchmarks."; readonly url: "https://github.com/NVIDIA-NeMo/Evaluator"; }; readonly "yc-bench": { readonly name: "yc-bench"; readonly description: "YC Bench is a long-horizon deterministic benchmark for LLM agents. The agent plays CEO of an AI startup over a simulated 1–3 year run."; readonly url: "https://github.com/collinear-ai/yc-bench"; }; readonly "open-asr-leaderboard": { readonly name: "open-asr-leaderboard"; readonly description: "The Open ASR Leaderboard ranks and evaluates speech recognition models."; readonly url: "https://github.com/huggingface/open_asr_leaderboard"; }; readonly mdpbench: { readonly name: "mdpbench"; readonly description: "MDPBench is a benchmark for evaluating multilingual document parsing across digital, photographed, Latin, and non-Latin document subsets."; readonly url: "https://github.com/Yuliang-Liu/MultimodalOCR"; }; readonly parsebench: { readonly name: "parsebench"; readonly description: "ParseBench is a benchmark for evaluating document parsing systems on real-world enterprise documents across tables, charts, content faithfulness, semantic formatting, and visual grounding."; readonly url: "https://github.com/run-llama/ParseBench"; }; readonly "video-mme-v2": { readonly name: "video-mme-v2"; readonly description: "Video-MME-v2 is a benchmark for evaluating the next stage of video understanding capabilities of multimodal large language models."; readonly url: "https://github.com/MME-Benchmarks/Video-MME-v2"; }; readonly "claw-eval": { readonly name: "claw-eval"; readonly description: "CLAW-Eval is an evaluation framework for assessing LLMs as autonomous agents across 300 human-verified tasks covering communication, finance, and productivity domains."; readonly url: "https://github.com/claw-eval/claw-eval"; }; readonly researchclawbench: { readonly name: "researchclawbench"; readonly description: "ResearchClawBench is a benchmark for evaluating AI agents on end-to-end scientific research tasks, from reading data and related work to producing code, figures, and publication-style reports."; readonly url: "https://github.com/InternScience/ResearchClawBench"; }; readonly pbench: { readonly name: "pbench"; readonly description: "PBench is a multi-level referring expression segmentation benchmark for evaluating vision-language perception across a structured hierarchy of skills."; readonly url: "https://github.com/tiiuae/Falcon-Perception"; }; readonly wildclawbench: { readonly name: "wildclawbench"; readonly description: "WildClawBench is an in-the-wild benchmark for evaluating AI agents in the OpenClaw environment across 60 hand-built, end-to-end tasks spanning productivity, code intelligence, social interaction, search, creative synthesis, and safety domains."; readonly url: "https://github.com/InternLM/WildClawBench"; }; readonly wbench: { readonly name: "wbench"; readonly description: "WBench is a comprehensive multi-turn benchmark for interactive video world model evaluation, assessing models across 5 dimensions (video quality, setting adherence, interaction adherence, consistency, physics compliance) and 22 metrics over 289 multi-turn interaction cases."; readonly url: "https://github.com/meituan-longcat/WBench"; }; readonly nanofold: { readonly name: "nanofold"; readonly description: "nanoFold is a data-efficiency benchmark for protein structure prediction. Its goal is to evaluate models on scenarios with scarce data."; readonly url: "https://github.com/ChrisHayduk/nanoFold-Competition"; }; readonly mmmu: { readonly name: "mmmu"; readonly description: "MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning."; readonly url: "https://mmmu-benchmark.github.io/"; }; }; //# sourceMappingURL=eval.d.ts.map