/** * Type definitions for Variably LLM SDK features */ export type LLMProvider = 'openai' | 'anthropic' | 'google' | 'azure' | 'custom'; export interface LLMConfig { /** LLM provider to use */ provider: LLMProvider; /** Model identifier (e.g., 'gpt-4', 'claude-3-opus') */ model: string; /** Provider API key (optional if using server-side configuration) */ apiKey?: string; /** Provider endpoint URL (for custom or Azure providers) */ endpoint?: string; /** Maximum tokens in response */ maxTokens?: number; /** Temperature for response generation (0-2) */ temperature?: number; /** Top-p sampling parameter */ topP?: number; /** Frequency penalty (-2.0 to 2.0) */ frequencyPenalty?: number; /** Presence penalty (-2.0 to 2.0) */ presencePenalty?: number; /** Stop sequences */ stopSequences?: string[]; /** Request timeout in milliseconds */ timeout?: number; /** Enable streaming responses */ streaming?: boolean; /** Custom headers for provider requests */ customHeaders?: Record; } export interface PromptExecutionRequest { /** Experiment ID for the prompt experiment */ experimentId: string; /** User context for variant selection */ userContext: LLMUserContext; /** Input variables for the prompt template */ variables?: Record; /** Override LLM configuration */ llmConfig?: Partial; /** Include evaluation in response */ includeEvaluation?: boolean; /** Custom evaluation criteria */ evaluationCriteria?: EvaluationCriteria; /** Enable automatic metric tracking */ trackMetrics?: boolean; /** Session ID for conversation continuity */ sessionId?: string; /** Previous conversation history */ conversationHistory?: ConversationMessage[]; /** Metadata for tracking */ metadata?: Record; } export interface PromptExecutionResponse { /** Unique execution ID */ executionId: string; /** Experiment ID */ experimentId: string; /** Selected variant */ variant: PromptVariant; /** Generated response */ response: LLMResponse; /** Evaluation results (if requested) */ evaluation?: EvaluationResult; /** Performance metrics */ metrics: PerformanceMetrics; /** Tracking information */ tracking: TrackingInfo; /** Error details (if any) */ error?: LLMError; } export interface LLMUserContext { /** Unique user identifier */ userId: string; /** User attributes for targeting */ attributes?: Record; /** User segments */ segments?: string[]; /** Geographic information */ geo?: { country?: string; region?: string; city?: string; }; /** Device/platform information */ device?: { type?: 'desktop' | 'mobile' | 'tablet'; platform?: string; browser?: string; }; /** User preferences */ preferences?: { language?: string; responseStyle?: 'concise' | 'detailed' | 'technical' | 'casual'; outputFormat?: 'text' | 'markdown' | 'json' | 'html'; }; } export interface PromptVariant { /** Variant ID */ variantId: string; /** Variant name */ name: string; /** Prompt template */ promptTemplate: string; /** Compiled prompt with variables */ compiledPrompt?: string; /** System message (if applicable) */ systemMessage?: string; /** LLM configuration for this variant */ llmConfig: LLMConfig; /** Traffic weight for this variant (used in allocation) */ trafficWeight?: number; /** Variant metadata */ metadata?: Record; } export interface LLMResponse { /** Generated text content */ content: string; /** Token usage information */ usage: TokenUsage; /** Model used for generation */ model: string; /** Provider used */ provider: LLMProvider; /** Finish reason */ finishReason: 'stop' | 'length' | 'content_filter' | 'error'; /** Response metadata from provider */ providerMetadata?: Record; /** Generated at timestamp */ generatedAt: Date; /** Response latency in milliseconds */ latencyMs: number; } export interface TokenUsage { /** Tokens in the prompt */ promptTokens: number; /** Tokens in the completion */ completionTokens: number; /** Total tokens used */ totalTokens: number; /** Estimated cost in USD */ estimatedCost?: number; } export interface EvaluationCriteria { /** Quality dimensions to evaluate */ dimensions: QualityDimension[]; /** Custom evaluation rules */ customRules?: EvaluationRule[]; /** Minimum acceptable scores */ thresholds?: Record; /** Use human review */ requireHumanReview?: boolean; /** Evaluation provider (for automated evaluation) */ evaluationProvider?: 'openai' | 'anthropic' | 'custom'; /** Evaluation model to use */ evaluationModel?: string; } export interface QualityDimension { /** Dimension name */ name: string; /** Description of what to evaluate */ description?: string; /** Weight in overall score (0-1) */ weight?: number; /** Evaluation method */ method: 'automated' | 'human' | 'hybrid'; /** Custom evaluation prompt (for automated) */ evaluationPrompt?: string; } export interface EvaluationRule { /** Rule ID */ ruleId: string; /** Rule type */ type: 'regex' | 'contains' | 'length' | 'custom'; /** Rule configuration */ config: Record; /** Score impact */ scoreImpact: number; /** Error message if rule fails */ errorMessage?: string; } export interface EvaluationResult { /** Overall quality score (0-1) */ overallScore: number; /** Individual dimension scores */ dimensionScores: Record; /** Rule evaluation results */ ruleResults?: RuleResult[]; /** Evaluation metadata */ metadata: { evaluationId: string; evaluatedAt: Date; evaluationMethod: string; evaluationModel?: string; }; /** Human review status */ humanReview?: { required: boolean; completed: boolean; reviewerId?: string; reviewedAt?: Date; feedback?: string; overrideScore?: number; }; /** Suggestions for improvement */ suggestions?: string[]; } export interface RuleResult { /** Rule ID */ ruleId: string; /** Whether the rule passed */ passed: boolean; /** Score impact applied */ scoreImpact: number; /** Error message (if failed) */ errorMessage?: string; /** Rule execution details */ details?: Record; } export interface ResponseEvaluationRequest { /** Response to evaluate */ response: string; /** Original prompt */ prompt?: string; /** Expected output (for comparison) */ expectedOutput?: string; /** Evaluation criteria */ criteria: EvaluationCriteria; /** Context for evaluation */ context?: Record; /** User context */ userContext?: LLMUserContext; /** Execution ID (for linking to prompt execution) */ executionId?: string; } export interface ResponseEvaluationResponse { /** Evaluation result */ evaluation: EvaluationResult; /** Comparison with expected output */ comparison?: ComparisonResult; /** Improvement recommendations */ recommendations?: ImprovementRecommendation[]; /** Evaluation metadata */ metadata: { evaluationId: string; evaluatedAt: Date; processingTimeMs: number; }; } export interface ComparisonResult { /** Similarity score (0-1) */ similarityScore: number; /** Differences found */ differences: Difference[]; /** Comparison method used */ method: 'semantic' | 'exact' | 'fuzzy'; } export interface Difference { /** Type of difference */ type: 'missing' | 'extra' | 'incorrect' | 'formatting'; /** Description of the difference */ description: string; /** Expected value */ expected?: string; /** Actual value */ actual?: string; /** Severity of the difference */ severity: 'low' | 'medium' | 'high'; } export interface ImprovementRecommendation { /** Area of improvement */ area: string; /** Specific recommendation */ recommendation: string; /** Priority level */ priority: 'low' | 'medium' | 'high'; /** Example of improved output */ example?: string; } export interface StreamingOptions { /** Enable streaming */ enabled: boolean; /** Chunk processing callback */ onChunk?: (chunk: StreamChunk) => void; /** Stream complete callback */ onComplete?: (response: LLMResponse) => void; /** Error callback */ onError?: (error: LLMError) => void; /** Buffer size for chunks */ bufferSize?: number; } export interface StreamChunk { /** Chunk content */ content: string; /** Chunk index */ index: number; /** Is final chunk */ isFinal: boolean; /** Timestamp */ timestamp: Date; } export interface ConversationMessage { /** Message role */ role: 'system' | 'user' | 'assistant'; /** Message content */ content: string; /** Message metadata */ metadata?: Record; /** Message timestamp */ timestamp?: Date; } export interface ConversationContext { /** Session ID */ sessionId: string; /** Conversation history */ messages: ConversationMessage[]; /** Context window size */ maxMessages?: number; /** Total tokens in context */ totalTokens?: number; /** Conversation metadata */ metadata?: Record; } export interface LLMCacheConfig { /** Enable response caching */ enabled: boolean; /** Cache TTL in seconds */ ttl?: number; /** Maximum cache size */ maxSize?: number; /** Cache key strategy */ keyStrategy?: 'prompt' | 'prompt+config' | 'custom'; /** Custom cache key generator */ keyGenerator?: (request: PromptExecutionRequest) => string; /** Cache storage backend */ storage?: 'memory' | 'localStorage' | 'custom'; } export interface CachedResponse { /** Cached response */ response: LLMResponse; /** Cache key */ key: string; /** Cached at timestamp */ cachedAt: Date; /** Cache hit count */ hitCount: number; /** Cache expiry */ expiresAt: Date; } export interface LLMError { /** Error code */ code: string; /** Error message */ message: string; /** Error type */ type: 'provider' | 'network' | 'validation' | 'evaluation' | 'rate_limit' | 'quota'; /** Provider-specific error details */ providerError?: any; /** Retry information */ retry?: { shouldRetry: boolean; retryAfter?: number; attemptNumber?: number; maxAttempts?: number; }; /** Error timestamp */ timestamp: Date; } export interface PerformanceMetrics { /** Total latency in milliseconds */ latencyMs: number; /** Time to first token (for streaming) */ timeToFirstTokenMs?: number; /** Tokens per second */ tokensPerSecond?: number; /** Provider latency */ providerLatencyMs?: number; /** SDK overhead */ sdkOverheadMs?: number; /** Cache hit */ cacheHit: boolean; /** Network retries */ retries: number; } export interface TrackingInfo { /** Execution ID */ executionId: string; /** Experiment ID */ experimentId: string; /** Variant ID */ variantId: string; /** User ID */ userId: string; /** Session ID */ sessionId?: string; /** Tracked at timestamp */ trackedAt: Date; /** Custom event properties */ properties?: Record; } export interface LLMSDKMetrics { /** Total prompt executions */ promptExecutions: number; /** Total response evaluations */ responseEvaluations: number; /** Average quality score */ averageQualityScore: number; /** Total tokens consumed */ totalTokens: number; /** Total cost incurred */ totalCost: number; /** Provider-specific metrics */ providerMetrics: Record; /** Cache metrics */ cacheMetrics: { hits: number; misses: number; evictions: number; hitRate: number; }; /** Error breakdown */ errorBreakdown: Record; } export interface BatchPromptRequest { /** Multiple prompt execution requests */ requests: PromptExecutionRequest[]; /** Batch processing options */ options?: { /** Process in parallel */ parallel?: boolean; /** Maximum concurrent requests */ maxConcurrency?: number; /** Stop on first error */ stopOnError?: boolean; /** Batch timeout */ timeoutMs?: number; }; } export interface BatchPromptResponse { /** Batch ID */ batchId: string; /** Individual responses */ responses: (PromptExecutionResponse | LLMError)[]; /** Batch statistics */ statistics: { total: number; successful: number; failed: number; averageLatencyMs: number; totalTokens: number; totalCost: number; }; } export interface ExportFormat { /** Format type */ type: 'json' | 'csv' | 'markdown'; /** Include metadata */ includeMetadata?: boolean; /** Include evaluation results */ includeEvaluation?: boolean; /** Custom fields to export */ fields?: string[]; } export interface ExportResult { /** Exported data */ data: string | object; /** Export format */ format: ExportFormat; /** Number of items exported */ count: number; /** Export timestamp */ exportedAt: Date; } export interface LogParams { /** The prompt sent to the LLM */ prompt: string; /** The LLM's response text */ response: string; /** Prior conversation turns for cross-turn coherence scoring */ conversationHistory?: ConversationMessage[]; /** Retrieved RAG chunks for grounding/hallucination scoring */ referenceMaterials?: Array<{ id: string; content: string; source?: string; type?: string; relevanceScore?: number; }>; /** The query sent to the retriever */ retrievalQuery?: string; /** Tags for grouping observations (e.g. ["production", "chatbot"]) */ tags?: string[]; /** End-user ID for per-user analytics */ userId?: string; /** Session ID for multi-turn tracking */ sessionId?: string; /** LLM provider name (e.g. "openai", "anthropic") */ provider?: string; /** Model name (e.g. "gpt-4", "claude-3-opus") */ model?: string; /** LLM call latency in milliseconds */ latencyMs?: number; /** Number of input tokens */ promptTokens?: number; /** Number of output tokens */ completionTokens?: number; /** Arbitrary key-value metadata */ metadata?: Record; } export interface LogResult { /** Unique observation ID */ observationId: string; /** Auto-created experiment ID */ experimentId: string; /** Status — always "queued" (scoring is async) */ status: string; } export type BusinessOutcomeType = 'conversion' | 'revenue' | 'user_satisfaction' | 'task_completion' | 'engagement' | 'retention' | 'custom'; export interface BusinessOutcome { /** Type of business outcome */ type: BusinessOutcomeType; /** Outcome name (e.g., 'purchase_completed', 'signup_completed') */ name: string; /** Numeric value associated with the outcome */ value?: number; /** Currency code for revenue outcomes */ currency?: string; /** User ID who triggered the outcome */ userId: string; /** Execution ID that led to this outcome */ executionId?: string; /** Experiment ID associated with this outcome */ experimentId?: string; /** Variant ID that was shown */ variantId?: string; /** Session ID */ sessionId?: string; /** Additional outcome properties */ properties?: Record; /** Timestamp of the outcome */ timestamp?: Date; } export interface UserFeedback { /** Execution ID being rated */ executionId: string; /** User ID providing feedback */ userId: string; /** Numeric rating (1-5, or custom scale) */ rating?: number; /** Thumbs up/down */ sentiment?: 'positive' | 'negative' | 'neutral'; /** Free-form feedback text */ comment?: string; /** Specific feedback category */ category?: 'accuracy' | 'helpfulness' | 'relevance' | 'quality' | 'other'; /** Additional feedback properties */ properties?: Record; /** Feedback timestamp */ timestamp?: Date; } export interface ConversionEvent { /** Event name (e.g., 'add_to_cart', 'purchase', 'signup') */ eventName: string; /** User ID */ userId: string; /** Execution ID that influenced this conversion */ executionId?: string; /** Experiment ID */ experimentId?: string; /** Variant ID */ variantId?: string; /** Revenue amount if applicable */ revenue?: number; /** Currency code */ currency?: string; /** Event properties */ properties?: Record; /** Funnel step (for funnel analysis) */ funnelStep?: string; /** Event timestamp */ timestamp?: Date; } export interface BusinessMetricsRequest { /** Experiment ID */ experimentId: string; /** Time range start */ startDate?: Date; /** Time range end */ endDate?: Date; /** Group by variant */ groupByVariant?: boolean; /** Include conversion rates */ includeConversions?: boolean; /** Include revenue metrics */ includeRevenue?: boolean; /** Include satisfaction scores */ includeSatisfaction?: boolean; } export interface BusinessMetricsResponse { /** Experiment ID */ experimentId: string; /** Time period covered */ period: { start: Date; end: Date; }; /** Overall business metrics */ overall: { /** Total conversions */ conversions: number; /** Conversion rate */ conversionRate: number; /** Total revenue */ totalRevenue?: number; /** Average revenue per user */ averageRevenuePerUser?: number; /** Average satisfaction score */ averageSatisfaction?: number; /** Task completion rate */ taskCompletionRate?: number; /** Engagement rate */ engagementRate?: number; }; /** Metrics by variant */ byVariant?: Record; /** Winner variant ID (if statistically significant) */ winnerVariantId?: string; } //# sourceMappingURL=llm-types.d.ts.map