// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.dialogflow.v2;

import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/dialogflow/v2/generator.proto";
import "google/cloud/dialogflow/v2/operations.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option csharp_namespace = "Google.Cloud.Dialogflow.V2";
option go_package = "cloud.google.com/go/dialogflow/apiv2/dialogflowpb;dialogflowpb";
option java_multiple_files = true;
option java_outer_classname = "GeneratorEvaluationProto";
option java_package = "com.google.cloud.dialogflow.v2";
option objc_class_prefix = "DF";

// Service for managing generator evaluations.
service GeneratorEvaluations {
  option (google.api.default_host) = "dialogflow.googleapis.com";
  option (google.api.oauth_scopes) =
      "https://www.googleapis.com/auth/cloud-platform,"
      "https://www.googleapis.com/auth/dialogflow";

  // Creates evaluation of a generator.
  rpc CreateGeneratorEvaluation(CreateGeneratorEvaluationRequest)
      returns (google.longrunning.Operation) {
    option (google.api.http) = {
      post: "/v2/{parent=projects/*/locations/*/generators/*}/evaluations"
      body: "generator_evaluation"
    };
    option (google.api.method_signature) = "parent,generator_evaluation";
    option (google.longrunning.operation_info) = {
      response_type: "GeneratorEvaluation"
      metadata_type: "google.cloud.dialogflow.v2.GeneratorEvaluationOperationMetadata"
    };
  }

  // Gets an evaluation of generator.
  rpc GetGeneratorEvaluation(GetGeneratorEvaluationRequest)
      returns (GeneratorEvaluation) {
    option (google.api.http) = {
      get: "/v2/{name=projects/*/locations/*/generators/*/evaluations/*}"
    };
    option (google.api.method_signature) = "name";
  }

  // Lists evaluations of generator.
  rpc ListGeneratorEvaluations(ListGeneratorEvaluationsRequest)
      returns (ListGeneratorEvaluationsResponse) {
    option (google.api.http) = {
      get: "/v2/{parent=projects/*/locations/*/generators/*}/evaluations"
    };
    option (google.api.method_signature) = "parent";
  }

  // Deletes an evaluation of generator.
  rpc DeleteGeneratorEvaluation(DeleteGeneratorEvaluationRequest)
      returns (google.protobuf.Empty) {
    option (google.api.http) = {
      delete: "/v2/{name=projects/*/locations/*/generators/*/evaluations/*}"
    };
    option (google.api.method_signature) = "name";
  }
}

// Request of CreateGeneratorEvaluation.
message CreateGeneratorEvaluationRequest {
  // Required. The generator resource name. Format:
  // `projects/<Project ID>/locations/<Location ID>/generators/<Generator ID>`
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      child_type: "dialogflow.googleapis.com/GeneratorEvaluation"
    }
  ];

  // Required. The generator evaluation to be created.
  GeneratorEvaluation generator_evaluation = 2
      [(google.api.field_behavior) = REQUIRED];
}

// Request of GetGeneratorEvaluation.
message GetGeneratorEvaluationRequest {
  // Required. The generator evaluation resource name. Format:
  // `projects/<Project ID>/locations/<Location ID>/generators/<Generator
  // ID>/evaluations/<Evaluation ID>`
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "dialogflow.googleapis.com/GeneratorEvaluation"
    }
  ];
}

// Request of ListGeneratorEvaluations.
message ListGeneratorEvaluationsRequest {
  // Required. The generator resource name. Format:
  // `projects/<Project ID>/locations/<Location ID>/generators/<Generator ID>`
  // Wildcard value `-` is supported on generator_id to list evaluations across
  // all generators under same project.
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      child_type: "dialogflow.googleapis.com/GeneratorEvaluation"
    }
  ];

  // Optional. Maximum number of evaluations to return in a
  // single page. By default 100 and at most 1000.
  int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The next_page_token value returned from a previous list request.
  string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Response of ListGeneratorEvaluations.
message ListGeneratorEvaluationsResponse {
  // The list of evaluations to return.
  repeated GeneratorEvaluation generator_evaluations = 1;

  // Token to retrieve the next page of results, or empty if there are no more
  // results in the list.
  string next_page_token = 2;
}

// Request of DeleteGeneratorEvaluation.
message DeleteGeneratorEvaluationRequest {
  // Required. The generator evaluation resource name. Format:
  // `projects/<Project ID>/locations/<Location ID>/generators/<Generator ID>/
  // evaluations/<Evaluation ID>`
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "dialogflow.googleapis.com/GeneratorEvaluation"
    }
  ];
}

// Represents evaluation result of a generator.
message GeneratorEvaluation {
  option (google.api.resource) = {
    type: "dialogflow.googleapis.com/GeneratorEvaluation"
    pattern: "projects/{project}/locations/{location}/generators/{generator}/evaluations/{evaluation}"
    plural: "generatorEvaluations"
    singular: "generatorEvaluation"
  };

  // Output only. Identifier. The resource name of the evaluation. Format:
  // `projects/<Project ID>/locations/<Location ID>/generators/<Generator ID>/
  // evaluations/<Evaluation ID>`
  string name = 1 [
    (google.api.field_behavior) = IDENTIFIER,
    (google.api.field_behavior) = OUTPUT_ONLY
  ];

  // Optional. The display name of the generator evaluation. At most 64 bytes
  // long.
  string display_name = 2 [(google.api.field_behavior) = OPTIONAL];

  // Required. The configuration of the evaluation task.
  GeneratorEvaluationConfig generator_evaluation_config = 3
      [(google.api.field_behavior) = REQUIRED];

  // Output only. Creation time of this generator evaluation.
  google.protobuf.Timestamp create_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Completion time of this generator evaluation.
  google.protobuf.Timestamp complete_time = 8
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Required. The initial generator that was used when creating this
  // evaluation. This is a copy of the generator read from storage when creating
  // the evaluation.
  Generator initial_generator = 5 [(google.api.field_behavior) = REQUIRED];

  // Metrics details.
  oneof metrics {
    // Output only. Only available when the summarization generator is provided.
    SummarizationEvaluationMetrics summarization_metrics = 6
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. The result status of the evaluation pipeline. Provides the
  // status information including if the evaluation is still in progress,
  // completed or failed with certain error and user actionable message.
  EvaluationStatus evaluation_status = 10
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. A read only boolean field reflecting Zone Separation
  // status of the model. The field is an aggregated
  // value of ZS status of its underlying dependencies. See more details in
  // go/zicy-resource-placement#resource-status
  optional bool satisfies_pzs = 11 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. A read only boolean field reflecting Zone Isolation status
  // of the model. The field is an aggregated value of
  // ZI status of its underlying dependencies. See more details in
  // go/zicy-resource-placement#resource-status
  optional bool satisfies_pzi = 12 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Evaluation metrics for summarization generator.
message SummarizationEvaluationMetrics {
  // Decomposition details for accuracy.
  message AccuracyDecomposition {
    // Output only. The breakdown point of the summary.
    string point = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The accuracy reasoning of the breakdown point.
    string accuracy_reasoning = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Whether the breakdown point is accurate or not.
    optional bool is_accurate = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Decomposition details for adherence.
  message AdherenceDecomposition {
    option deprecated = true;

    // Output only. The breakdown point of the given instructions.
    string point = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The adherence reasoning of the breakdown point.
    string adherence_reasoning = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Whether the breakdown point is adherent or not.
    bool is_adherent = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Rubric result of the adherence evaluation. A rubric is ued to determine
  // if the summary adheres to all aspects of the given instructions.
  message AdherenceRubric {
    // Output only. The question generated from instruction that used to
    // evaluate summary.
    string question = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The reasoning of the rubric question is addressed or not.
    string reasoning = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. A boolean that indicates whether the rubric question is
    // addressed or not.
    optional bool is_addressed = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Rubric details of the completeness evaluation result.
  message CompletenessRubric {
    // Output only. The question generated from instruction that used to
    // evaluate summary.
    string question = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. A boolean that indicates whether the rubric question is
    // addressed or not.
    optional bool is_addressed = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Decomposition details
  message Decomposition {
    option deprecated = true;

    // One of decomposition details.
    oneof decomposition {
      // only available for accuracy metric.
      AccuracyDecomposition accuracy_decomposition = 1;

      // only available for adherence metric.
      AdherenceDecomposition adherence_decomposition = 2;
    }
  }

  // Evaluation result that contains one of accuracy, adherence or completeness
  // evaluation result.
  message EvaluationResult {
    // One of evaluation result details.
    oneof result {
      // Only available for accuracy metric.
      AccuracyDecomposition accuracy_decomposition = 1;

      // Only available for adherence metric.
      AdherenceRubric adherence_rubric = 2;

      // Only available for completeness metric.
      CompletenessRubric completeness_rubric = 3;
    }
  }

  // Evaluation result per conversation(&summary), metric and section.
  message SummarizationEvaluationResult {
    // Output only. conversation session id
    string session_id = 1
        [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. metric name, e.g. accuracy, completeness, adherence, etc.
    string metric = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. section/task name, e.g. action, situation, etc
    string section = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. score calculated from decompositions
    float score = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Summary of this section
    string section_summary = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. List of decompostion details
    repeated Decomposition decompositions = 7
        [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. List of evaluation results.
    repeated EvaluationResult evaluation_results = 8
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Overall performance per metric. This is the aggregated score for each
  // metric across all conversations that are selected for summarization
  // evaluation.
  message OverallScoresByMetric {
    // Output only. Metric name. e.g. accuracy, adherence, completeness.
    string metric = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // A pair of section name and input token count of the input summary section.
  message SectionToken {
    // Output only. The name of the summary instruction.
    string section = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Token count.
    optional int64 token_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Aggregated evaluation result on conversation level. This conatins
  // evaluation results of all the metrics and sections.
  message ConversationDetail {
    // Aggregated result on metric level. This conatins the evaluation results
    // of all the sections.
    message MetricDetail {
      // Section level result.
      message SectionDetail {
        // Output only. The name of the summary instruction.
        string section = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

        // Output only. Aggregated(average) score on this section across all
        // evaluation results. Either decompositions or rubrics.
        optional float score = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

        // Output only. Summary for this section
        string section_summary = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

        // Output only. List of evaluation result. The list only contains one
        // kind of the evaluation result.
        repeated EvaluationResult evaluation_results = 5
            [(google.api.field_behavior) = OUTPUT_ONLY];
      }

      // Output only. Metrics name. e.g. accuracy, adherence, completeness.
      string metric = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. Aggregated(average) score on this metric across all
      // sections.
      optional float score = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. List of section details.
      repeated SectionDetail section_details = 4
          [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. Conversation transcript that used for summarization
    // evaluation as a reference.
    repeated MessageEntry message_entries = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Summary sections that used for summarization evaluation as a
    // reference.
    repeated SummarySuggestion.SummarySection summary_sections = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. List of metric details.
    repeated MetricDetail metric_details = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Conversation level token count per section. This is an
    // aggregated(sum) result of input token of summary acorss all metrics for a
    // single conversation.
    repeated SectionToken section_tokens = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. A list of evaluation results per conversation(&summary),
  // metric and section.
  repeated SummarizationEvaluationResult summarization_evaluation_results = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. User bucket uri for merged evaluation score and aggregation
  // score csv.
  string summarization_evaluation_merged_results_uri = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. A list of aggregated(average) scores per metric section.
  repeated OverallScoresByMetric overall_metrics = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Overall token per section. This is an aggregated(sum) result
  // of input token of summary acorss all conversations that are selected for
  // summarization evaluation.
  repeated SectionToken overall_section_tokens = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. List of conversation details.
  repeated ConversationDetail conversation_details = 8
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Generator evaluation input config.
message GeneratorEvaluationConfig {
  // Enumeration of input data source type.
  enum InputDataSourceType {
    // Unspecified InputDataSourceType. Should not be used.
    INPUT_DATA_SOURCE_TYPE_UNSPECIFIED = 0;

    // Fetch data from Agent Assist storage. If this source type is chosen,
    // input_data_config.start_time and input_data_config.end_timestamp must be
    // provided.
    AGENT_ASSIST_CONVERSATIONS = 1;

    // Fetch data from Insights storage. If this source type is chosen,
    // input_data_config.start_time and input_data_config.end_timestamp must be
    // provided.
    INSIGHTS_CONVERSATIONS = 2;
  }

  // The distinctive configs for Agent Assist conversations as the conversation
  // source.
  message AgentAssistInputDataConfig {
    // Required. The start of the time range for conversations to be evaluated.
    // Only conversations created at or after this timestamp will be sampled.
    google.protobuf.Timestamp start_time = 1
        [(google.api.field_behavior) = REQUIRED];

    // Required. The end of the time range for conversations to be evaluated.
    // Only conversations ended at or before this timestamp will be sampled.
    google.protobuf.Timestamp end_time = 2
        [(google.api.field_behavior) = REQUIRED];
  }

  // The distinctive configs for dataset as the conversation source.
  message DatasetInputDataConfig {
    // Required. The identifier of the dataset to be evaluated.
    // Format:
    // `projects/<ProjectId>/locations/<LocationID>/datasets/<DatasetID>`.
    string dataset = 1 [(google.api.field_behavior) = REQUIRED];
  }

  // Input data config details
  message InputDataConfig {
    // Summary generation options.
    enum SummaryGenerationOption {
      // Default option will not be used
      SUMMARY_GENERATION_OPTION_UNSPECIFIED = 0;

      // Always Generate summary for all conversations.
      ALWAYS_GENERATE = 1;

      // Gnerate only missing summaries.
      GENERATE_IF_MISSING = 2;

      // Do not generate new summaries. Only use existing summaries found.
      DO_NOT_GENERATE = 3;
    }

    // Required. The source type of input data.
    InputDataSourceType input_data_source_type = 1
        [deprecated = true, (google.api.field_behavior) = REQUIRED];

    // Optional. The start timestamp to fetch conversation data.
    google.protobuf.Timestamp start_time = 2
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. The end timestamp to fetch conversation data.
    google.protobuf.Timestamp end_time = 3
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. Desired number of conversation-summary pairs to be evaluated.
    int32 sample_size = 4 [(google.api.field_behavior) = OPTIONAL];

    // Optional. Whether the summary generation is allowed when the pre-existing
    // qualified summaries are insufficient to cover the sample size.
    bool is_summary_generation_allowed = 5
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. Option to control whether summaries are generated during
    // evaluation.
    SummaryGenerationOption summary_generation_option = 8
        [(google.api.field_behavior) = OPTIONAL];

    // The source specific config for the input data.
    oneof source_specific_config {
      // The distinctive configs for Agent Assist conversations as the
      // conversation source.
      AgentAssistInputDataConfig agent_assist_input_data_config = 6;

      // The distinctive configs for dataset as the conversation source.
      DatasetInputDataConfig dataset_input_data_config = 7;
    }
  }

  // Evaluation configs for summarization generator.
  message SummarizationConfig {
    // Optional. Enable accuracy evaluation.
    bool enable_accuracy_evaluation = 1
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. Version for summarization accuracy. This will determine the
    // prompt and model used at backend.
    string accuracy_evaluation_version = 2
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. Enable completeness evaluation.
    bool enable_completeness_evaluation = 3
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. Version for summarization completeness. This will determine the
    // prompt and model used at backend.
    string completeness_evaluation_version = 4
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. Version for summarization evaluation.
    optional string evaluator_version = 5
        [(google.api.field_behavior) = OPTIONAL];
  }

  // Required. The config/source of input data.
  InputDataConfig input_data_config = 1
      [(google.api.field_behavior) = REQUIRED];

  // Required. The output Cloud Storage bucket path to store eval files, e.g.
  // per_summary_accuracy_score report. This path is provided by customer and
  // files stored in it are visible to customer, no internal data should be
  // stored in this path.
  string output_gcs_bucket_path = 2 [(google.api.field_behavior) = REQUIRED];

  // Feature used for evaluation.
  oneof evaluation_feature_config {
    // Evaluation configs for summarization generator.
    SummarizationConfig summarization_config = 3;
  }
}

// A common evalaution pipeline status.
message EvaluationStatus {
  // Output only. If the value is `false`, it means the evaluation is still in
  // progress. If `true`, the operation is completed, and either `error` or
  // `response` is available.
  optional bool done = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The error result of the evaluation in case of failure in
  // evaluation pipeline.
  google.rpc.Status pipeline_status = 2
      [(google.api.field_behavior) = OUTPUT_ONLY];
}