// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.cloud.ces.v1beta; import "google/api/field_behavior.proto"; import "google/api/resource.proto"; import "google/cloud/ces/v1beta/app.proto"; import "google/cloud/ces/v1beta/common.proto"; import "google/cloud/ces/v1beta/example.proto"; import "google/cloud/ces/v1beta/fakes.proto"; import "google/cloud/ces/v1beta/golden_run.proto"; import "google/cloud/ces/v1beta/session_service.proto"; import "google/cloud/ces/v1beta/toolset_tool.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; import "google/rpc/status.proto"; option go_package = "cloud.google.com/go/ces/apiv1beta/cespb;cespb"; option java_multiple_files = true; option java_outer_classname = "EvaluationProto"; option java_package = "com.google.cloud.ces.v1beta"; // Aggregated metrics for an evaluation or evaluation dataset. message AggregatedMetrics { // Metrics for a single tool. message ToolMetrics { // Output only. The name of the tool. string tool = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" } ]; // Output only. The number of times the tool passed. int32 pass_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The number of times the tool failed. int32 fail_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metrics for turn latency. message TurnLatencyMetrics { // Output only. The average latency of the turns. google.protobuf.Duration average_latency = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metrics for tool call latency. message ToolCallLatencyMetrics { // Output only. The name of the tool. string tool = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" } ]; // Output only. The average latency of the tool calls. google.protobuf.Duration average_latency = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metrics for semantic similarity results. message SemanticSimilarityMetrics { // Output only. The average semantic similarity score (0-4). float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metrics for hallucination results. message HallucinationMetrics { // Output only. The average hallucination score (0 to 1). float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metrics aggregated per app version. message MetricsByAppVersion { // Output only. The app version ID. string app_version_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for each tool within this app version. repeated ToolMetrics tool_metrics = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for semantic similarity within this app version. repeated SemanticSimilarityMetrics semantic_similarity_metrics = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for hallucination within this app version. repeated HallucinationMetrics hallucination_metrics = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for tool call latency within this app version. repeated ToolCallLatencyMetrics tool_call_latency_metrics = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for turn latency within this app version. repeated TurnLatencyMetrics turn_latency_metrics = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The number of times the evaluation passed. int32 pass_count = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The number of times the evaluation failed. int32 fail_count = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics aggregated per turn within this app version. repeated MetricsByTurn metrics_by_turn = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metrics aggregated per turn. message MetricsByTurn { // Output only. The turn index (0-based). int32 turn_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for each tool within this turn. repeated ToolMetrics tool_metrics = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for semantic similarity within this turn. repeated SemanticSimilarityMetrics semantic_similarity_metrics = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for hallucination within this turn. repeated HallucinationMetrics hallucination_metrics = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for tool call latency within this turn. repeated ToolCallLatencyMetrics tool_call_latency_metrics = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Metrics for turn latency within this turn. repeated TurnLatencyMetrics turn_latency_metrics = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. Aggregated metrics, grouped by app version ID. repeated MetricsByAppVersion metrics_by_app_version = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // An evaluation represents all of the information needed to simulate and // evaluate an agent. message Evaluation { option (google.api.resource) = { type: "ces.googleapis.com/Evaluation" pattern: "projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}" plural: "evaluations" singular: "evaluation" }; // Represents a single, checkable requirement. message GoldenExpectation { // The actual check to perform. oneof condition { // Optional. Check that a specific tool was called with the parameters. ToolCall tool_call = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Check that a specific tool had the expected response. ToolResponse tool_response = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Check that the agent responded with the correct response. The // role "agent" is implied. Message agent_response = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. Check that the agent transferred the conversation to a // different agent. AgentTransfer agent_transfer = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. Check that the agent updated the session variables to the // expected values. Used to also capture agent variable updates for golden // evals. google.protobuf.Struct updated_variables = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. The tool response to mock, with the parameters of interest // specified. Any parameters not specified will be hallucinated by the // LLM. ToolResponse mock_tool_response = 7 [(google.api.field_behavior) = OPTIONAL]; } // Optional. A note for this requirement, useful in reporting when specific // checks fail. E.g., "Check_Payment_Tool_Called". string note = 1 [(google.api.field_behavior) = OPTIONAL]; } // A step defines a singular action to happen during the evaluation. message Step { // The step to perform. oneof step { // Optional. User input for the conversation. SessionInput user_input = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Transfer the conversation to a different agent. AgentTransfer agent_transfer = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Executes an expectation on the current turn. GoldenExpectation expectation = 3 [(google.api.field_behavior) = OPTIONAL]; } } // A golden turn defines a single turn in a golden conversation. message GoldenTurn { // Required. The steps required to replay a golden conversation. repeated Step steps = 1 [(google.api.field_behavior) = REQUIRED]; // Optional. The root span of the golden turn for processing and maintaining // audio information. Span root_span = 2 [(google.api.field_behavior) = OPTIONAL]; } // The steps required to replay a golden conversation. message Golden { // Required. The golden turns required to replay a golden conversation. repeated GoldenTurn turns = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. The evaluation expectations to evaluate the replayed // conversation against. Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}` repeated string evaluation_expectations = 3 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationExpectation" } ]; } // The expectation to evaluate the conversation produced by the simulation. message ScenarioExpectation { // The tool call and response pair to be evaluated. message ToolExpectation { // Required. The expected tool call, with the parameters of interest // specified. Any parameters not specified will be hallucinated by the // LLM. ToolCall expected_tool_call = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The tool response to mock, with the parameters of interest // specified. Any parameters not specified will be hallucinated by the // LLM. ToolResponse mock_tool_response = 2 [(google.api.field_behavior) = REQUIRED]; } // The expectation to evaluate the conversation produced by the simulation. oneof expectation { // Optional. The tool call and response pair to be evaluated. ToolExpectation tool_expectation = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The agent response to be evaluated. Message agent_response = 2 [(google.api.field_behavior) = OPTIONAL]; } } // The config for a scenario message Scenario { // Facts about the user as a key value pair. message UserFact { // Required. The name of the user fact. string name = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The value of the user fact. string value = 2 [(google.api.field_behavior) = REQUIRED]; } // The expected behavior of the user task. This is used to determine whether // the scenario is successful. enum TaskCompletionBehavior { // Behavior unspecified. Will default to TASK_SATISFIED. TASK_COMPLETION_BEHAVIOR_UNSPECIFIED = 0; // The user task should be completed successfully. TASK_SATISFIED = 1; // The user task should be rejected. TASK_REJECTED = 2; } // The expected behavior of the user goal. This is used to determine whether // the scenario is successful. enum UserGoalBehavior { // Behavior unspecified. Will default to USER_GOAL_SATISFIED. USER_GOAL_BEHAVIOR_UNSPECIFIED = 0; // The user goal should be completed successfully. USER_GOAL_SATISFIED = 1; // The user goal should be rejected. USER_GOAL_REJECTED = 2; // Ignore the user goal status. USER_GOAL_IGNORED = 3; } // Required. The task to be targeted by the scenario. string task = 1 [(google.api.field_behavior) = REQUIRED]; // Optional. The user facts to be used by the scenario. repeated UserFact user_facts = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. The maximum number of turns to simulate. If not specified, the // simulation will continue until the task is complete. int32 max_turns = 5 [(google.api.field_behavior) = OPTIONAL]; // Required. The rubrics to score the scenario against. repeated string rubrics = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The ScenarioExpectations to evaluate the conversation produced // by the user simulation. repeated ScenarioExpectation scenario_expectations = 3 [(google.api.field_behavior) = REQUIRED]; // Optional. Variables / Session Parameters as context for the session, // keyed by variable names. Members of this struct will override any default // values set by the system. // // Note, these are different from user facts, which are facts known to the // user. Variables are parameters known to the agent: i.e. MDN (phone // number) passed by the telephony system. google.protobuf.Struct variable_overrides = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. Deprecated. Use user_goal_behavior instead. TaskCompletionBehavior task_completion_behavior = 7 [deprecated = true, (google.api.field_behavior) = OPTIONAL]; // Optional. The expected behavior of the user goal. UserGoalBehavior user_goal_behavior = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. The evaluation expectations to evaluate the conversation // produced by the simulation against. Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}` repeated string evaluation_expectations = 10 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationExpectation" } ]; } // The inputs for the evaluation oneof inputs { // Optional. The golden steps to be evaluated. Golden golden = 11 [(google.api.field_behavior) = OPTIONAL]; // Optional. The config for a scenario. Scenario scenario = 12 [(google.api.field_behavior) = OPTIONAL]; } // Identifier. The unique identifier of this evaluation. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}` string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Required. User-defined display name of the evaluation. Unique within an // App. string display_name = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. User-defined description of the evaluation. string description = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. User defined tags to categorize the evaluation. repeated string tags = 4 [(google.api.field_behavior) = OPTIONAL]; // Output only. List of evaluation datasets the evaluation belongs to. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}` repeated string evaluation_datasets = 5 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationDataset" } ]; // Output only. Timestamp when the evaluation was created. google.protobuf.Timestamp create_time = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who created the evaluation. string created_by = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Timestamp when the evaluation was last updated. google.protobuf.Timestamp update_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who last updated the evaluation. string last_updated_by = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The EvaluationRuns that this Evaluation is associated with. repeated string evaluation_runs = 15 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationRun" } ]; // Output only. Etag used to ensure the object hasn't changed during a // read-modify-write operation. If the etag is empty, the update will // overwrite any concurrent changes. string etag = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The aggregated metrics for this evaluation across all runs. AggregatedMetrics aggregated_metrics = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latest evaluation result for this evaluation. EvaluationResult last_completed_result = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Whether the evaluation is invalid. This can happen if an // evaluation is referencing a tool, toolset, or agent that has since been // deleted. bool invalid = 18 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The last 10 evaluation results for this evaluation. This is // only populated if include_last_ten_results is set to true in the // ListEvaluationsRequest or GetEvaluationRequest. repeated EvaluationResult last_ten_results = 19 [(google.api.field_behavior) = OUTPUT_ONLY]; } // An evaluation dataset represents a set of evaluations that are grouped // together basaed on shared tags. message EvaluationDataset { option (google.api.resource) = { type: "ces.googleapis.com/EvaluationDataset" pattern: "projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluation_dataset}" plural: "evaluationDatasets" singular: "evaluationDataset" }; // Identifier. The unique identifier of this evaluation dataset. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}` string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Required. User-defined display name of the evaluation dataset. Unique // within an App. string display_name = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. Evaluations that are included in this dataset. repeated string evaluations = 3 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/Evaluation" } ]; // Output only. Timestamp when the evaluation dataset was created. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Timestamp when the evaluation dataset was last updated. google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Etag used to ensure the object hasn't changed during a // read-modify-write operation. If the etag is empty, the update will // overwrite any concurrent changes. string etag = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who created the evaluation dataset. string created_by = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who last updated the evaluation dataset. string last_updated_by = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The aggregated metrics for this evaluation dataset across all // runs. AggregatedMetrics aggregated_metrics = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; } // An evaluation result represents the output of running an Evaluation. message EvaluationResult { option (google.api.resource) = { type: "ces.googleapis.com/EvaluationResult" pattern: "projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{evaluation_result}" plural: "evaluationResults" singular: "evaluationResult" }; // Specifies the expectation and the result of that expectation. message GoldenExpectationOutcome { // The result of the tool invocation check. message ToolInvocationResult { // Output only. The tool invocation parameter correctness score. This // indicates the percent of parameters from the expected tool call that // were also present in the actual tool call. optional float parameter_correctness_score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the tool invocation check. This is // determined by comparing the parameter_correctness_score to the // threshold. If the score is equal to or above the threshold, the outcome // will be PASS. Otherwise, the outcome will be FAIL. Outcome outcome = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. A free text explanation for the tool invocation result. string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of the expectation. oneof result { // Output only. The result of the tool call expectation. ToolCall observed_tool_call = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the tool response expectation. ToolResponse observed_tool_response = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the agent response expectation. Message observed_agent_response = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the agent transfer expectation. AgentTransfer observed_agent_transfer = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. The expectation that was evaluated. Evaluation.GoldenExpectation expectation = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the expectation. Outcome outcome = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the semantic similarity check. SemanticSimilarityResult semantic_similarity_result = 7 [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the tool invocation check. ToolInvocationResult tool_invocation_result = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of a single evaluation expectation. message EvaluationExpectationResult { // Output only. The evaluation expectation. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}` string evaluation_expectation = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationExpectation" } ]; // Output only. The prompt that was used for the evaluation. string prompt = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the evaluation expectation. Outcome outcome = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The explanation for the result. string explanation = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of a golden evaluation. message GoldenResult { // The result of running a single turn of the golden conversation. message TurnReplayResult { // Output only. The conversation that was generated for this turn. string conversation = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Conversation" } ]; // Output only. The outcome of each expectation. repeated GoldenExpectationOutcome expectation_outcome = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the hallucination check. HallucinationResult hallucination_result = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Deprecated. Use OverallToolInvocationResult instead. float tool_invocation_score = 4 [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The overall tool ordered invocation score for this turn. // This indicates the overall percent of tools from the expected turn that // were actually invoked in the expected order. optional float tool_ordered_invocation_score = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Duration of the turn. google.protobuf.Duration turn_latency = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency of each tool call in the turn. repeated ToolCallLatency tool_call_latencies = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the semantic similarity check. SemanticSimilarityResult semantic_similarity_result = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the overall tool invocation check. OverallToolInvocationResult overall_tool_invocation_result = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Information about the error that occurred during this // turn. EvaluationErrorInfo error_info = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency of spans in the turn. repeated SpanLatency span_latencies = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. The result of running each turn of the golden conversation. repeated TurnReplayResult turn_replay_results = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The results of the evaluation expectations. repeated EvaluationExpectationResult evaluation_expectation_results = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The outcome of the evaluation against the rubric. message ScenarioRubricOutcome { // Output only. The rubric that was used to evaluate the conversation. string rubric = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The score of the conversation against the rubric. optional float score = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The rater's response to the rubric. string score_explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The outcome of a scenario expectation. message ScenarioExpectationOutcome { // The observed tool call and response. message ObservedToolCall { // Output only. The observed tool call. ToolCall tool_call = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The observed tool response. ToolResponse tool_response = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of the expectation. oneof result { // Output only. The observed tool call. ObservedToolCall observed_tool_call = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The observed agent response. Message observed_agent_response = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. The expectation that was evaluated. Evaluation.ScenarioExpectation expectation = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the ScenarioExpectation. Outcome outcome = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The outcome of a scenario evaluation. message ScenarioResult { // Output only. The conversation that was generated in the scenario. string conversation = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Conversation" } ]; // Output only. The task that was used when running the scenario for this // result. string task = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user facts that were used by the scenario for this // result. repeated Evaluation.Scenario.UserFact user_facts = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of each expectation. repeated ScenarioExpectationOutcome expectation_outcomes = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the rubric. repeated ScenarioRubricOutcome rubric_outcomes = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the hallucination check. // There will be one hallucination result for each turn in the conversation. repeated HallucinationResult hallucination_result = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the task completion check. TaskCompletionResult task_completion_result = 5 [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency of each tool call execution in the conversation. repeated ToolCallLatency tool_call_latencies = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The result of the user goal satisfaction check. UserGoalSatisfactionResult user_goal_satisfaction_result = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Whether all expectations were satisfied for this turn. optional bool all_expectations_satisfied = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Whether the task was completed for this turn. This is a // composite of all expectations satisfied, no hallucinations, and user goal // satisfaction. optional bool task_completed = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency of spans in the conversation. repeated SpanLatency span_latencies = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The results of the evaluation expectations. repeated EvaluationExpectationResult evaluation_expectation_results = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The latency of a span execution. message SpanLatency { // The type of span. // Additional values may be added in the future. enum Type { // Default value. This value is unused. TYPE_UNSPECIFIED = 0; // Tool call span. TOOL = 1; // User callback span. USER_CALLBACK = 2; // Guardrail span. GUARDRAIL = 3; // LLM span. LLM = 4; } // The identifier of the specific item based on its type. oneof identifier { // Output only. The resource name of the guardrail or tool spans. string resource = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The toolset tool identifier. ToolsetTool toolset = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The name of the LLM span. string model = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The name of the user callback span. string callback = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. The type of span. Type type = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The display name of the span. Applicable to tool and // guardrail spans. string display_name = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The start time of span. google.protobuf.Timestamp start_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The end time of span. google.protobuf.Timestamp end_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency of span. google.protobuf.Duration execution_latency = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The latency of a tool call execution. message ToolCallLatency { // Output only. The name of the tool that got executed. // Format: // `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`. string tool = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" } ]; // Output only. The display name of the tool. string display_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The start time of the tool call execution. google.protobuf.Timestamp start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The end time of the tool call execution. google.protobuf.Timestamp end_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency of the tool call execution. google.protobuf.Duration execution_latency = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of the hallucination check for a single turn. message HallucinationResult { // Output only. The hallucination score. Can be -1, 0, 1. optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The label associated with each score. // Score 1: Justified // Score 0: Not Justified // Score -1: No Claim To Assess string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The explanation for the hallucination score. string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of a user goal satisfaction check for a conversation. message UserGoalSatisfactionResult { // Output only. The user task satisfaction score. Can be -1, 0, 1. optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The label associated with each score. // Score 1: User Task Satisfied // Score 0: User Task Not Satisfied // Score -1: User Task Unspecified string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The explanation for the user task satisfaction score. string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of the semantic similarity check. message SemanticSimilarityResult { // Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4. optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The label associated with each score. // Score 4: Fully Consistent // Score 3: Mostly Consistent // Score 2: Partially Consistent (Minor Omissions) // Score 1: Largely Inconsistent (Major Omissions) // Score 0: Completely Inconsistent / Contradictory string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The explanation for the semantic similarity score. string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the semantic similarity check. This is // determined by comparing the score to the // semantic_similarity_success_threshold. If the score is equal to or above // the threshold, the outcome will be PASS. Otherwise, the outcome will be // FAIL. Outcome outcome = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of the overall tool invocation check. message OverallToolInvocationResult { // The overall tool invocation score for this turn. This indicates the // overall percent of tools from the expected turn that were actually // invoked. optional float tool_invocation_score = 1; // Output only. The outcome of the tool invocation check. This is determined // by comparing the tool_invocation_score to the // overall_tool_invocation_correctness_threshold. If the // score is equal to or above the threshold, the outcome will be PASS. // Otherwise, the outcome will be FAIL. Outcome outcome = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The result of the task completion check for the conversation. message TaskCompletionResult { // Output only. The task completion score. Can be -1, 0, 1 optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The label associated with each score. // Score 1: Task Completed // Score 0: Task Not Completed // Score -1: User Goal Undefined string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The explanation for the task completion score. string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The outcome of the evaluation or expectation. enum Outcome { // Evaluation outcome is not specified. OUTCOME_UNSPECIFIED = 0; // Evaluation/Expectation passed. In the case of an evaluation, this means // that all expectations were met. PASS = 1; // Evaluation/Expectation failed. In the case of an evaluation, this means // that at least one expectation was not met. FAIL = 2; // Evaluation/Expectation was skipped. SKIPPED = 3; } // The state of the evaluation result execution. enum ExecutionState { // Evaluation result execution state is not specified. EXECUTION_STATE_UNSPECIFIED = 0; // Evaluation result execution is running. RUNNING = 1; // Evaluation result execution has completed. COMPLETED = 2; // Evaluation result execution failed due to an internal error. ERROR = 3; } // The result of the evaluation. Only populated when the execution_state is // COMPLETED. oneof result { // Output only. The outcome of a golden evaluation. GoldenResult golden_result = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of a scenario evaluation. ScenarioResult scenario_result = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Identifier. The unique identifier of the evaluation result. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{result}` string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Required. Display name of the Evaluation Result. Unique within an // Evaluation. By default, it has the following format: // " result // - ". string display_name = 2 [(google.api.field_behavior) = REQUIRED]; // Output only. Timestamp when the evaluation result was created. google.protobuf.Timestamp create_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The outcome of the evaluation. Only populated if // execution_state is COMPLETE. Outcome evaluation_status = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The evaluation run that produced this result. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}` string evaluation_run = 6 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationRun" } ]; // Output only. The persona used to generate the conversation for the // evaluation result. EvaluationPersona persona = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Error information for the evaluation result. EvaluationErrorInfo error_info = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Deprecated: Use `error_info` instead. Errors encountered // during execution. google.rpc.Status error = 14 [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who initiated the evaluation run that resulted in // this result. string initiated_by = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The app version used to generate the conversation that // resulted in this result. Format: // `projects/{project}/locations/{location}/apps/{app}/versions/{version}` string app_version = 10 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/AppVersion" } ]; // Output only. The display name of the `app_version` that the evaluation ran // against. string app_version_display_name = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The changelog of the app version that the evaluation ran // against. This is populated if user runs evaluation on latest/draft. string changelog = 21 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Changelog" } ]; // Output only. The create time of the changelog of the app version that the // evaluation ran against. This is populated if user runs evaluation on // latest/draft. google.protobuf.Timestamp changelog_create_time = 22 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The state of the evaluation result execution. ExecutionState execution_state = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The evaluation thresholds for the result. EvaluationMetricsThresholds evaluation_metrics_thresholds = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The configuration used in the evaluation run that resulted in // this result. EvaluationConfig config = 18 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The method used to run the golden evaluation. GoldenRunMethod golden_run_method = 19 [(google.api.field_behavior) = OUTPUT_ONLY]; } // An evaluation run represents an all the evaluation results from an // evaluation execution. message EvaluationRun { option (google.api.resource) = { type: "ces.googleapis.com/EvaluationRun" pattern: "projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluation_run}" plural: "evaluationRuns" singular: "evaluationRun" }; // The progress of the evaluation run. message Progress { // Output only. Total number of evaluation results in this run. int32 total_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Number of completed evaluation results with an outcome of // FAIL. (EvaluationResult.execution_state is COMPLETED and // EvaluationResult.evaluation_status is FAIL). int32 failed_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Number of evaluation results that failed to execute. // (EvaluationResult.execution_state is ERROR). int32 error_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Number of evaluation results that finished successfully. // (EvaluationResult.execution_state is COMPLETED). int32 completed_count = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Number of completed evaluation results with an outcome of // PASS. (EvaluationResult.execution_state is COMPLETED and // EvaluationResult.evaluation_status is PASS). int32 passed_count = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Contains the summary of passed and failed result counts for a specific // evaluation in an evaluation run. message EvaluationRunSummary { // Output only. Number of passed results for the associated Evaluation in // this run. int32 passed_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Number of failed results for the associated Evaluation in // this run. int32 failed_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Number of error results for the associated Evaluation in // this run. int32 error_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The type of the evaluations in this run. // Additional values may be added in the future. enum EvaluationType { // Evaluation type is not specified. EVALUATION_TYPE_UNSPECIFIED = 0; // Golden evaluation. GOLDEN = 1; // Scenario evaluation. SCENARIO = 2; // Indicates the run includes a mix of golden and scenario evaluations. MIXED = 3; } // The state of the evaluation run. enum EvaluationRunState { // Evaluation run state is not specified. EVALUATION_RUN_STATE_UNSPECIFIED = 0; // Evaluation run is running. RUNNING = 1; // Evaluation run has completed. COMPLETED = 2; // The evaluation run has an error. ERROR = 3; } // Identifier. The unique identifier of the evaluation run. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}` string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Optional. User-defined display name of the evaluation run. // default: " run - ". string display_name = 2 [(google.api.field_behavior) = OPTIONAL]; // Output only. The evaluation results that are part of this run. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{result}` repeated string evaluation_results = 3 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationResult" } ]; // Output only. Timestamp when the evaluation run was created. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who initiated the evaluation run. string initiated_by = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The app version to evaluate. // Format: // `projects/{project}/locations/{location}/apps/{app}/versions/{version}` string app_version = 6 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/AppVersion" } ]; // Output only. The display name of the `app_version` that the evaluation ran // against. string app_version_display_name = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The changelog of the app version that the evaluation ran // against. This is populated if user runs evaluation on latest/draft. string changelog = 22 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Changelog" } ]; // Output only. The create time of the changelog of the app version that the // evaluation ran against. This is populated if user runs evaluation on // latest/draft. google.protobuf.Timestamp changelog_create_time = 23 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The evaluations that are part of this run. The list may // contain evaluations of either type. This field is mutually exclusive with // `evaluation_dataset`. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}` repeated string evaluations = 7 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Evaluation" } ]; // Output only. The evaluation dataset that this run is associated with. This // field is mutually exclusive with `evaluations`. Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}` string evaluation_dataset = 8 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationDataset" } ]; // Output only. The type of the evaluations in this run. EvaluationType evaluation_type = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The state of the evaluation run. EvaluationRunState state = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The progress of the evaluation run. Progress progress = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The configuration used in the run. EvaluationConfig config = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Deprecated: Use error_info instead. Errors encountered during // execution. google.rpc.Status error = 14 [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Error information for the evaluation run. EvaluationErrorInfo error_info = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Map of evaluation name to EvaluationRunSummary. map evaluation_run_summaries = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Latency report for the evaluation run. LatencyReport latency_report = 25 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The number of times the evaluations inside the run were run. int32 run_count = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The configuration to use for the run per persona. repeated PersonaRunConfig persona_run_configs = 18 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. Configuration for running the optimization step after the // evaluation run. If not set, the optimization step will not be run. OptimizationConfig optimization_config = 19 [(google.api.field_behavior) = OPTIONAL]; // Output only. The scheduled evaluation run resource name that created this // evaluation run. This field is only set if the evaluation run was created // by a scheduled evaluation run. // Format: // `projects/{project}/locations/{location}/apps/{app}/scheduledEvaluationRuns/{scheduled_evaluation_run}` string scheduled_evaluation_run = 20 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/ScheduledEvaluationRun" } ]; // Output only. The method used to run the evaluation. GoldenRunMethod golden_run_method = 21 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Latency report for the evaluation run. message LatencyReport { // Latency metrics for a component. message LatencyMetrics { // Output only. The 50th percentile latency. google.protobuf.Duration p50_latency = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The 90th percentile latency. google.protobuf.Duration p90_latency = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The 99th percentile latency. google.protobuf.Duration p99_latency = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The number of times the resource was called. int32 call_count = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Latency metrics for a single tool. message ToolLatency { // The identifier of the tool. oneof tool_identifier { // Output only. Format: // `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`. string tool = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" } ]; // Output only. The toolset tool identifier. ToolsetTool toolset_tool = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. The display name of the tool. string tool_display_name = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency metrics for the tool. LatencyMetrics latency_metrics = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Latency metrics for a single callback. message CallbackLatency { // Output only. The stage of the callback. string stage = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency metrics for the callback. LatencyMetrics latency_metrics = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Latency metrics for a single guardrail. message GuardrailLatency { // Output only. The name of the guardrail. // Format: // `projects/{project}/locations/{location}/apps/{app}/guardrails/{guardrail}`. string guardrail = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/Guardrail" } ]; // Output only. The display name of the guardrail. string guardrail_display_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency metrics for the guardrail. LatencyMetrics latency_metrics = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Latency metrics for a single LLM call. message LlmCallLatency { // Output only. The name of the model. string model = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The latency metrics for the LLM call. LatencyMetrics latency_metrics = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Output only. Unordered list. Latency metrics for each tool. repeated ToolLatency tool_latencies = 1 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = UNORDERED_LIST ]; // Output only. Unordered list. Latency metrics for each callback. repeated CallbackLatency callback_latencies = 2 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = UNORDERED_LIST ]; // Output only. Unordered list. Latency metrics for each guardrail. repeated GuardrailLatency guardrail_latencies = 3 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = UNORDERED_LIST ]; // Output only. Unordered list. Latency metrics for each LLM call. repeated LlmCallLatency llm_call_latencies = 4 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = UNORDERED_LIST ]; // Output only. The total number of sessions considered in the latency report. int32 session_count = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; } // An evaluation expectation represents a specific criteria to evaluate against. message EvaluationExpectation { option (google.api.resource) = { type: "ces.googleapis.com/EvaluationExpectation" pattern: "projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}" plural: "evaluationExpectations" singular: "evaluationExpectation" }; // Configuration for LLM-based evaluation criteria. message LlmCriteria { // Required. The prompt/instructions provided to the LLM judge. string prompt = 1 [(google.api.field_behavior) = REQUIRED]; } // The definition of the criteria used for evaluation. oneof criteria { // Optional. Evaluation criteria based on an LLM prompt. LlmCriteria llm_criteria = 3 [(google.api.field_behavior) = OPTIONAL]; } // Identifier. The unique identifier of this evaluation expectation. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}` string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Required. User-defined display name. Must be unique within the app. string display_name = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. User-defined tags for expectations. Can be used to filter // expectations. repeated string tags = 8 [(google.api.field_behavior) = OPTIONAL]; // Output only. Timestamp when the evaluation expectation was created. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Timestamp when the evaluation expectation was last updated. google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Etag used to ensure the object hasn't changed during a // read-modify-write operation. If the etag is empty, the update will // overwrite any concurrent changes. string etag = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; } // EvaluationConfig configures settings for running the evaluation. message EvaluationConfig { // The channel to evaluate. enum EvaluationChannel { // Unspecified evaluation channel. EVALUATION_CHANNEL_UNSPECIFIED = 0; // Text-only evaluation channel. TEXT = 1; // Audio evaluation channel. AUDIO = 2; } // Optional. Configuration for processing the input audio. InputAudioConfig input_audio_config = 1 [deprecated = true, (google.api.field_behavior) = OPTIONAL]; // Optional. Configuration for generating the output audio. OutputAudioConfig output_audio_config = 2 [deprecated = true, (google.api.field_behavior) = OPTIONAL]; // Optional. The channel to evaluate. EvaluationChannel evaluation_channel = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Specifies whether the evaluation should use real tool calls or // fake tools. EvaluationToolCallBehaviour tool_call_behaviour = 4 [(google.api.field_behavior) = OPTIONAL]; } // Information about an error encountered during an evaluation execution. message EvaluationErrorInfo { // The type of error enum ErrorType { // Unspecified error type. ERROR_TYPE_UNSPECIFIED = 0; // Failure during runtime execution. RUNTIME_FAILURE = 1; // Failure to retrieve conversation from CES Runtime. CONVERSATION_RETRIEVAL_FAILURE = 2; // Failure to calculate a metric / outcome. METRIC_CALCULATION_FAILURE = 3; // Failure to update the evaluation. EVALUATION_UPDATE_FAILURE = 4; // Ran out of quota. QUOTA_EXHAUSTED = 5; // Failure during user simulation. USER_SIMULATION_FAILURE = 6; } // Output only. The type of error. ErrorType error_type = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The error message. string error_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The session ID for the conversation that caused the error. string session_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Request message for // [EvaluationService.RunEvaluation][google.cloud.ces.v1beta.EvaluationService.RunEvaluation]. message RunEvaluationRequest { // Required. The app to evaluate. // Format: // `projects/{project}/locations/{location}/apps/{app}` string app = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "ces.googleapis.com/App" } ]; // Optional. List of evaluations to run. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}` repeated string evaluations = 2 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/Evaluation" } ]; // Optional. An evaluation dataset to run. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}` string evaluation_dataset = 3 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationDataset" } ]; // Optional. The display name of the evaluation run. string display_name = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. The app version to evaluate. // Format: // `projects/{project}/locations/{location}/apps/{app}/versions/{version}` string app_version = 6 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/AppVersion" } ]; // Optional. The configuration to use for the run. EvaluationConfig config = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. The number of times to run the evaluation. If not set, the // default value is 1 per golden, and 5 per scenario. optional int32 run_count = 9 [(google.api.field_behavior) = OPTIONAL]; // Optional. The configuration to use for the run per persona. repeated PersonaRunConfig persona_run_configs = 10 [(google.api.field_behavior) = OPTIONAL]; // Optional. Configuration for running the optimization step after the // evaluation run. If not set, the optimization step will not be run. OptimizationConfig optimization_config = 11 [(google.api.field_behavior) = OPTIONAL]; // Optional. The resource name of the `ScheduledEvaluationRun` that is // triggering this evaluation run. // // If this field is set, the `scheduled_evaluation_run` field on the created // `EvaluationRun` resource will be populated from this value. // Format: // `projects/{project}/locations/{location}/apps/{app}/scheduledEvaluationRuns/{scheduled_evaluation_run}` string scheduled_evaluation_run = 12 [ (google.api.field_behavior) = OPTIONAL, (google.api.resource_reference) = { type: "ces.googleapis.com/ScheduledEvaluationRun" } ]; // Optional. The method to run the evaluation if it is a golden evaluation. If // not set, default to STABLE. GoldenRunMethod golden_run_method = 13 [(google.api.field_behavior) = OPTIONAL]; // Optional. Whether to generate a latency report for the evaluation run. bool generate_latency_report = 14 [(google.api.field_behavior) = OPTIONAL]; } // Represents a scheduled evaluation run configuration. message ScheduledEvaluationRun { option (google.api.resource) = { type: "ces.googleapis.com/ScheduledEvaluationRun" pattern: "projects/{project}/locations/{location}/apps/{app}/scheduledEvaluationRuns/{scheduled_evaluation_run}" plural: "scheduledEvaluationRuns" singular: "scheduledEvaluationRun" }; // Eval scheduling configuration details message SchedulingConfig { // The frequencies evals can be run at enum Frequency { // The frequency is unspecified. FREQUENCY_UNSPECIFIED = 0; // Indicates a one-time scheduled run that should not repeat NONE = 1; // Run the evaluation daily. DAILY = 2; // Run the evaluation weekly. WEEKLY = 3; // Run the evaluation biweekly. BIWEEKLY = 4; } // Required. The frequency with which to run the eval Frequency frequency = 1 [(google.api.field_behavior) = REQUIRED]; // Required. Timestamp when the eval should start. google.protobuf.Timestamp start_time = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. The days of the week to run the eval. Applicable only for // Weekly and Biweekly frequencies. 1 is Monday, 2 is Tuesday, ..., 7 is // Sunday. repeated int32 days_of_week = 3 [(google.api.field_behavior) = OPTIONAL]; } // Identifier. The unique identifier of the scheduled evaluation run config. // Format: // projects/{projectId}/locations/{locationId}/apps/{appId}/scheduledEvaluationRuns/{scheduledEvaluationRunId} string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Required. User-defined display name of the scheduled evaluation run config. string display_name = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The RunEvaluationRequest to schedule RunEvaluationRequest request = 3 [(google.api.field_behavior) = REQUIRED]; // Optional. User-defined description of the scheduled evaluation run. string description = 4 [(google.api.field_behavior) = OPTIONAL]; // Required. Configuration for the timing and frequency with which to execute // the evaluations. SchedulingConfig scheduling_config = 5 [(google.api.field_behavior) = REQUIRED]; // Optional. Whether this config is active bool active = 6 [(google.api.field_behavior) = OPTIONAL]; // Output only. The last successful EvaluationRun of this scheduled execution. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}` string last_completed_run = 7 [ (google.api.field_behavior) = OUTPUT_ONLY, (google.api.resource_reference) = { type: "ces.googleapis.com/EvaluationRun" } ]; // Output only. The total number of times this run has been executed int32 total_executions = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The next time this is scheduled to execute google.protobuf.Timestamp next_scheduled_execution_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Timestamp when the scheduled evaluation run was created. google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who created the scheduled evaluation run. string created_by = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Timestamp when the evaluation was last updated. google.protobuf.Timestamp update_time = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The user who last updated the evaluation. string last_updated_by = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Etag used to ensure the object hasn't changed during a // read-modify-write operation. If the etag is empty, the update will // overwrite any concurrent changes. string etag = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Configuration for running an evaluation for a specific persona. message PersonaRunConfig { // Optional. The persona to use for the evaluation. // Format: // `projects/{project}/locations/{location}/apps/{app}/evaluationPersonas/{evaluationPersona}` string persona = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The number of tasks to run for the persona. int32 task_count = 2 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for running the optimization step after the evaluation run. message OptimizationConfig { // The status of the optimization run. enum OptimizationStatus { // Optimization status is not specified. OPTIMIZATION_STATUS_UNSPECIFIED = 0; // Optimization is running. RUNNING = 1; // Optimization has completed. COMPLETED = 2; // Optimization failed due to an internal error. ERROR = 3; } // Optional. Whether to generate a loss report. bool generate_loss_report = 1 [(google.api.field_behavior) = OPTIONAL]; // Output only. The assistant session to use for the optimization based on // this evaluation run. Format: // `projects/{project}/locations/{location}/apps/{app}/assistantSessions/{assistantSession}` string assistant_session = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The summary of the loss report. string report_summary = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Whether to suggest a fix for the losses. bool should_suggest_fix = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The status of the optimization run. OptimizationStatus status = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The error message if the optimization run failed. string error_message = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The generated loss report. google.protobuf.Struct loss_report = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; }