// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1beta1;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/aiplatform/v1beta1/explanation.proto";
import "google/cloud/aiplatform/v1beta1/model_evaluation_slice.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1";
option go_package = "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "ModelEvaluationProto";
option java_package = "com.google.cloud.aiplatform.v1beta1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
option ruby_package = "Google::Cloud::AIPlatform::V1beta1";

// A collection of metrics calculated by comparing Model's predictions on all of
// the test data against annotations from the test data.
message ModelEvaluation {
  option (google.api.resource) = {
    type: "aiplatform.googleapis.com/ModelEvaluation"
    pattern: "projects/{project}/locations/{location}/models/{model}/evaluations/{evaluation}"
  };

  message ModelEvaluationExplanationSpec {
    // Explanation type.
    //
    // For AutoML Image Classification models, possible values are:
    //
    //   * `image-integrated-gradients`
    //   * `image-xrai`
    string explanation_type = 1;

    // Explanation spec details.
    ExplanationSpec explanation_spec = 2;
  }

  // Configuration for bias detection.
  message BiasConfig {
    // Specification for how the data should be sliced for bias. It contains a
    // list of slices, with limitation of two slices. The first slice of data
    // will be the slice_a. The second slice in the list (slice_b) will be
    // compared against the first slice. If only a single slice is provided,
    // then slice_a will be compared against "not slice_a".
    // Below are examples with feature "education" with value "low", "medium",
    // "high" in the dataset:
    //
    // Example 1:
    //
    //     bias_slices = [{'education': 'low'}]
    //
    // A single slice provided. In this case, slice_a is the collection of data
    // with 'education' equals 'low', and slice_b is the collection of data with
    // 'education' equals 'medium' or 'high'.
    //
    // Example 2:
    //
    //     bias_slices = [{'education': 'low'},
    //                    {'education': 'high'}]
    //
    // Two slices provided. In this case, slice_a is the collection of data
    // with 'education' equals 'low', and slice_b is the collection of data with
    // 'education' equals 'high'.
    ModelEvaluationSlice.Slice.SliceSpec bias_slices = 1;

    // Positive labels selection on the target field.
    repeated string labels = 2;
  }

  // Output only. The resource name of the ModelEvaluation.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // The display name of the ModelEvaluation.
  string display_name = 10;

  // Points to a YAML file stored on Google Cloud Storage describing the
  // [metrics][google.cloud.aiplatform.v1beta1.ModelEvaluation.metrics] of this
  // ModelEvaluation. The schema is defined as an OpenAPI 3.0.2 [Schema
  // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject).
  string metrics_schema_uri = 2;

  // Evaluation metrics of the Model. The schema of the metrics is stored in
  // [metrics_schema_uri][google.cloud.aiplatform.v1beta1.ModelEvaluation.metrics_schema_uri]
  google.protobuf.Value metrics = 3;

  // Output only. Timestamp when this ModelEvaluation was created.
  google.protobuf.Timestamp create_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // All possible
  // [dimensions][google.cloud.aiplatform.v1beta1.ModelEvaluationSlice.Slice.dimension]
  // of ModelEvaluationSlices. The dimensions can be used as the filter of the
  // [ModelService.ListModelEvaluationSlices][google.cloud.aiplatform.v1beta1.ModelService.ListModelEvaluationSlices]
  // request, in the form of `slice.dimension = <dimension>`.
  repeated string slice_dimensions = 5;

  // Aggregated explanation metrics for the Model's prediction output over the
  // data this ModelEvaluation uses. This field is populated only if the Model
  // is evaluated with explanations, and only for AutoML tabular Models.
  //
  ModelExplanation model_explanation = 8;

  // Describes the values of
  // [ExplanationSpec][google.cloud.aiplatform.v1beta1.ExplanationSpec] that are
  // used for explaining the predicted values on the evaluated data.
  repeated ModelEvaluationExplanationSpec explanation_specs = 9;

  // The metadata of the ModelEvaluation.
  // For the ModelEvaluation uploaded from Managed Pipeline, metadata contains a
  // structured value with keys of "pipeline_job_id", "evaluation_dataset_type",
  // "evaluation_dataset_path".
  google.protobuf.Value metadata = 11;

  // Specify the configuration for bias detection.
  BiasConfig bias_configs = 12;
}