# Evals Framework Configuration
# Standard evaluation criteria for AI coding tools

version = "1.0.0"
last_updated = "2025-10-04"

# This file defines the standard evaluation criteria used across all tool evaluations.
# These criteria are built into the CriteriaManager and are loaded automatically.
# Custom X-factor criteria can be added via x-factor.toml

# Standard Criteria Summary:
# 1. Authentication & Setup - API key flow, initial setup complexity
# 2. Invocation Interface - CLI ergonomics, command structure
# 3. Model/Provider Support - LLM support, multi-provider capability
# 4. Extensibility - Plugin system, customization options
# 5. User Experience - Conversation flow, error handling
# 6. Privacy & Security - Data handling, local-first, telemetry
# 7. Documentation Quality - Completeness, examples, API docs
# 8. Community & Support - GitHub activity, response time
# 9. Licensing - Open source status, restrictions
# 10. Performance - Speed, resource usage, streaming
# 11. Integration Capabilities - Git, IDE, workflow tools
# 12. Unique Differentiators - Killer features, innovation
# 13. Cost Structure - Free tier, pricing model

# Note: The actual criterion definitions are in the Rust code
# (src/evals/evals_criteria.rs) to ensure type safety and consistency.
# This file serves as documentation and version tracking.

[metadata]
framework_version = "1.0.0"
schema_version = "1.0.0"
evaluation_guide_url = "https://github.com/BA-CalderonMorales/terminal-jarvis/docs/evals"

[scoring]
# Scoring scale: 0.0 to 10.0
min_score = 0.0
max_score = 10.0

# Rating thresholds
[scoring.ratings]
excellent = 9.0  # 9.0-10.0
good = 7.0       # 7.0-8.9
adequate = 5.0   # 5.0-6.9
poor = 3.0       # 3.0-4.9
inadequate = 0.0 # 0.0-2.9

# Criterion weights (for weighted overall score calculation)
[weights]
auth_setup = 1.0
invocation = 1.2
model_support = 1.1
extensibility = 0.9
user_experience = 1.3
privacy_security = 1.4
documentation = 1.0
community_support = 0.8
licensing = 0.7
performance = 1.0
integration = 0.9
differentiators = 1.1
cost = 0.8

[evaluation_guidelines]
# Guidelines for evaluators
comprehensive_testing = true
evidence_required = true
citation_format = "Tool documentation, GitHub repo, or testing notes"
version_tracking = true
regular_updates = true

# Evaluation frequency recommendations
update_frequency_days = 90 # Re-evaluate tools every 3 months
