# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: google/cloud/datalabeling_v1beta1/proto/evaluation.proto

import sys

_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1"))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database

# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2
from google.cloud.datalabeling_v1beta1.proto import (
    annotation_pb2 as google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__pb2,
)
from google.cloud.datalabeling_v1beta1.proto import (
    annotation_spec_set_pb2 as google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__spec__set__pb2,
)
from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2


DESCRIPTOR = _descriptor.FileDescriptor(
    name="google/cloud/datalabeling_v1beta1/proto/evaluation.proto",
    package="google.cloud.datalabeling.v1beta1",
    syntax="proto3",
    serialized_options=_b(
        "\n%com.google.cloud.datalabeling.v1beta1P\001ZMgoogle.golang.org/genproto/googleapis/cloud/datalabeling/v1beta1;datalabeling"
    ),
    serialized_pb=_b(
        '\n8google/cloud/datalabeling_v1beta1/proto/evaluation.proto\x12!google.cloud.datalabeling.v1beta1\x1a\x1cgoogle/api/annotations.proto\x1a\x38google/cloud/datalabeling_v1beta1/proto/annotation.proto\x1a\x41google/cloud/datalabeling_v1beta1/proto/annotation_spec_set.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\x89\x03\n\nEvaluation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x43\n\x06\x63onfig\x18\x02 \x01(\x0b\x32\x33.google.cloud.datalabeling.v1beta1.EvaluationConfig\x12;\n\x17\x65valuation_job_run_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12P\n\x12\x65valuation_metrics\x18\x05 \x01(\x0b\x32\x34.google.cloud.datalabeling.v1beta1.EvaluationMetrics\x12J\n\x0f\x61nnotation_type\x18\x06 \x01(\x0e\x32\x31.google.cloud.datalabeling.v1beta1.AnnotationType\x12\x1c\n\x14\x65valuated_item_count\x18\x07 \x01(\x03"\x91\x01\n\x10\x45valuationConfig\x12j\n\x1f\x62ounding_box_evaluation_options\x18\x01 \x01(\x0b\x32?.google.cloud.datalabeling.v1beta1.BoundingBoxEvaluationOptionsH\x00\x42\x11\n\x0fvertical_option"5\n\x1c\x42oundingBoxEvaluationOptions\x12\x15\n\riou_threshold\x18\x01 \x01(\x02"\xd9\x01\n\x11\x45valuationMetrics\x12Z\n\x16\x63lassification_metrics\x18\x01 \x01(\x0b\x32\x38.google.cloud.datalabeling.v1beta1.ClassificationMetricsH\x00\x12]\n\x18object_detection_metrics\x18\x02 \x01(\x0b\x32\x39.google.cloud.datalabeling.v1beta1.ObjectDetectionMetricsH\x00\x42\t\n\x07metrics"\xa3\x01\n\x15\x43lassificationMetrics\x12<\n\x08pr_curve\x18\x01 \x01(\x0b\x32*.google.cloud.datalabeling.v1beta1.PrCurve\x12L\n\x10\x63onfusion_matrix\x18\x02 \x01(\x0b\x32\x32.google.cloud.datalabeling.v1beta1.ConfusionMatrix"V\n\x16ObjectDetectionMetrics\x12<\n\x08pr_curve\x18\x01 \x01(\x0b\x32*.google.cloud.datalabeling.v1beta1.PrCurve"\xe6\x03\n\x07PrCurve\x12J\n\x0f\x61nnotation_spec\x18\x01 \x01(\x0b\x32\x31.google.cloud.datalabeling.v1beta1.AnnotationSpec\x12\x18\n\x10\x61rea_under_curve\x18\x02 \x01(\x02\x12\x65\n\x1a\x63onfidence_metrics_entries\x18\x03 \x03(\x0b\x32\x41.google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry\x12\x1e\n\x16mean_average_precision\x18\x04 \x01(\x02\x1a\xed\x01\n\x16\x43onfidenceMetricsEntry\x12\x1c\n\x14\x63onfidence_threshold\x18\x01 \x01(\x02\x12\x0e\n\x06recall\x18\x02 \x01(\x02\x12\x11\n\tprecision\x18\x03 \x01(\x02\x12\x10\n\x08\x66\x31_score\x18\x04 \x01(\x02\x12\x12\n\nrecall_at1\x18\x05 \x01(\x02\x12\x15\n\rprecision_at1\x18\x06 \x01(\x02\x12\x14\n\x0c\x66\x31_score_at1\x18\x07 \x01(\x02\x12\x12\n\nrecall_at5\x18\x08 \x01(\x02\x12\x15\n\rprecision_at5\x18\t \x01(\x02\x12\x14\n\x0c\x66\x31_score_at5\x18\n \x01(\x02"\xfc\x02\n\x0f\x43onfusionMatrix\x12\x43\n\x03row\x18\x01 \x03(\x0b\x32\x36.google.cloud.datalabeling.v1beta1.ConfusionMatrix.Row\x1av\n\x14\x43onfusionMatrixEntry\x12J\n\x0f\x61nnotation_spec\x18\x01 \x01(\x0b\x32\x31.google.cloud.datalabeling.v1beta1.AnnotationSpec\x12\x12\n\nitem_count\x18\x02 \x01(\x05\x1a\xab\x01\n\x03Row\x12J\n\x0f\x61nnotation_spec\x18\x01 \x01(\x0b\x32\x31.google.cloud.datalabeling.v1beta1.AnnotationSpec\x12X\n\x07\x65ntries\x18\x02 \x03(\x0b\x32G.google.cloud.datalabeling.v1beta1.ConfusionMatrix.ConfusionMatrixEntryBx\n%com.google.cloud.datalabeling.v1beta1P\x01ZMgoogle.golang.org/genproto/googleapis/cloud/datalabeling/v1beta1;datalabelingb\x06proto3'
    ),
    dependencies=[
        google_dot_api_dot_annotations__pb2.DESCRIPTOR,
        google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__pb2.DESCRIPTOR,
        google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__spec__set__pb2.DESCRIPTOR,
        google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,
    ],
)


_EVALUATION = _descriptor.Descriptor(
    name="Evaluation",
    full_name="google.cloud.datalabeling.v1beta1.Evaluation",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="name",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.name",
            index=0,
            number=1,
            type=9,
            cpp_type=9,
            label=1,
            has_default_value=False,
            default_value=_b("").decode("utf-8"),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="config",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.config",
            index=1,
            number=2,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="evaluation_job_run_time",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.evaluation_job_run_time",
            index=2,
            number=3,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="create_time",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.create_time",
            index=3,
            number=4,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="evaluation_metrics",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.evaluation_metrics",
            index=4,
            number=5,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="annotation_type",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.annotation_type",
            index=5,
            number=6,
            type=14,
            cpp_type=8,
            label=1,
            has_default_value=False,
            default_value=0,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="evaluated_item_count",
            full_name="google.cloud.datalabeling.v1beta1.Evaluation.evaluated_item_count",
            index=6,
            number=7,
            type=3,
            cpp_type=2,
            label=1,
            has_default_value=False,
            default_value=0,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=284,
    serialized_end=677,
)


_EVALUATIONCONFIG = _descriptor.Descriptor(
    name="EvaluationConfig",
    full_name="google.cloud.datalabeling.v1beta1.EvaluationConfig",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="bounding_box_evaluation_options",
            full_name="google.cloud.datalabeling.v1beta1.EvaluationConfig.bounding_box_evaluation_options",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        )
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[
        _descriptor.OneofDescriptor(
            name="vertical_option",
            full_name="google.cloud.datalabeling.v1beta1.EvaluationConfig.vertical_option",
            index=0,
            containing_type=None,
            fields=[],
        )
    ],
    serialized_start=680,
    serialized_end=825,
)


_BOUNDINGBOXEVALUATIONOPTIONS = _descriptor.Descriptor(
    name="BoundingBoxEvaluationOptions",
    full_name="google.cloud.datalabeling.v1beta1.BoundingBoxEvaluationOptions",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="iou_threshold",
            full_name="google.cloud.datalabeling.v1beta1.BoundingBoxEvaluationOptions.iou_threshold",
            index=0,
            number=1,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        )
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=827,
    serialized_end=880,
)


_EVALUATIONMETRICS = _descriptor.Descriptor(
    name="EvaluationMetrics",
    full_name="google.cloud.datalabeling.v1beta1.EvaluationMetrics",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="classification_metrics",
            full_name="google.cloud.datalabeling.v1beta1.EvaluationMetrics.classification_metrics",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="object_detection_metrics",
            full_name="google.cloud.datalabeling.v1beta1.EvaluationMetrics.object_detection_metrics",
            index=1,
            number=2,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[
        _descriptor.OneofDescriptor(
            name="metrics",
            full_name="google.cloud.datalabeling.v1beta1.EvaluationMetrics.metrics",
            index=0,
            containing_type=None,
            fields=[],
        )
    ],
    serialized_start=883,
    serialized_end=1100,
)


_CLASSIFICATIONMETRICS = _descriptor.Descriptor(
    name="ClassificationMetrics",
    full_name="google.cloud.datalabeling.v1beta1.ClassificationMetrics",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="pr_curve",
            full_name="google.cloud.datalabeling.v1beta1.ClassificationMetrics.pr_curve",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="confusion_matrix",
            full_name="google.cloud.datalabeling.v1beta1.ClassificationMetrics.confusion_matrix",
            index=1,
            number=2,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=1103,
    serialized_end=1266,
)


_OBJECTDETECTIONMETRICS = _descriptor.Descriptor(
    name="ObjectDetectionMetrics",
    full_name="google.cloud.datalabeling.v1beta1.ObjectDetectionMetrics",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="pr_curve",
            full_name="google.cloud.datalabeling.v1beta1.ObjectDetectionMetrics.pr_curve",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        )
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=1268,
    serialized_end=1354,
)


_PRCURVE_CONFIDENCEMETRICSENTRY = _descriptor.Descriptor(
    name="ConfidenceMetricsEntry",
    full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="confidence_threshold",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.confidence_threshold",
            index=0,
            number=1,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="recall",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.recall",
            index=1,
            number=2,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="precision",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.precision",
            index=2,
            number=3,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="f1_score",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.f1_score",
            index=3,
            number=4,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="recall_at1",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.recall_at1",
            index=4,
            number=5,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="precision_at1",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.precision_at1",
            index=5,
            number=6,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="f1_score_at1",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.f1_score_at1",
            index=6,
            number=7,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="recall_at5",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.recall_at5",
            index=7,
            number=8,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="precision_at5",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.precision_at5",
            index=8,
            number=9,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="f1_score_at5",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry.f1_score_at5",
            index=9,
            number=10,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=1606,
    serialized_end=1843,
)

_PRCURVE = _descriptor.Descriptor(
    name="PrCurve",
    full_name="google.cloud.datalabeling.v1beta1.PrCurve",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="annotation_spec",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.annotation_spec",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="area_under_curve",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.area_under_curve",
            index=1,
            number=2,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="confidence_metrics_entries",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.confidence_metrics_entries",
            index=2,
            number=3,
            type=11,
            cpp_type=10,
            label=3,
            has_default_value=False,
            default_value=[],
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="mean_average_precision",
            full_name="google.cloud.datalabeling.v1beta1.PrCurve.mean_average_precision",
            index=3,
            number=4,
            type=2,
            cpp_type=6,
            label=1,
            has_default_value=False,
            default_value=float(0),
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[_PRCURVE_CONFIDENCEMETRICSENTRY],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=1357,
    serialized_end=1843,
)


_CONFUSIONMATRIX_CONFUSIONMATRIXENTRY = _descriptor.Descriptor(
    name="ConfusionMatrixEntry",
    full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.ConfusionMatrixEntry",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="annotation_spec",
            full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.ConfusionMatrixEntry.annotation_spec",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="item_count",
            full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.ConfusionMatrixEntry.item_count",
            index=1,
            number=2,
            type=5,
            cpp_type=1,
            label=1,
            has_default_value=False,
            default_value=0,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=1934,
    serialized_end=2052,
)

_CONFUSIONMATRIX_ROW = _descriptor.Descriptor(
    name="Row",
    full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.Row",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="annotation_spec",
            full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.Row.annotation_spec",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=1,
            has_default_value=False,
            default_value=None,
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
        _descriptor.FieldDescriptor(
            name="entries",
            full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.Row.entries",
            index=1,
            number=2,
            type=11,
            cpp_type=10,
            label=3,
            has_default_value=False,
            default_value=[],
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        ),
    ],
    extensions=[],
    nested_types=[],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=2055,
    serialized_end=2226,
)

_CONFUSIONMATRIX = _descriptor.Descriptor(
    name="ConfusionMatrix",
    full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix",
    filename=None,
    file=DESCRIPTOR,
    containing_type=None,
    fields=[
        _descriptor.FieldDescriptor(
            name="row",
            full_name="google.cloud.datalabeling.v1beta1.ConfusionMatrix.row",
            index=0,
            number=1,
            type=11,
            cpp_type=10,
            label=3,
            has_default_value=False,
            default_value=[],
            message_type=None,
            enum_type=None,
            containing_type=None,
            is_extension=False,
            extension_scope=None,
            serialized_options=None,
            file=DESCRIPTOR,
        )
    ],
    extensions=[],
    nested_types=[_CONFUSIONMATRIX_CONFUSIONMATRIXENTRY, _CONFUSIONMATRIX_ROW],
    enum_types=[],
    serialized_options=None,
    is_extendable=False,
    syntax="proto3",
    extension_ranges=[],
    oneofs=[],
    serialized_start=1846,
    serialized_end=2226,
)

_EVALUATION.fields_by_name["config"].message_type = _EVALUATIONCONFIG
_EVALUATION.fields_by_name[
    "evaluation_job_run_time"
].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
_EVALUATION.fields_by_name[
    "create_time"
].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
_EVALUATION.fields_by_name["evaluation_metrics"].message_type = _EVALUATIONMETRICS
_EVALUATION.fields_by_name[
    "annotation_type"
].enum_type = (
    google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__pb2._ANNOTATIONTYPE
)
_EVALUATIONCONFIG.fields_by_name[
    "bounding_box_evaluation_options"
].message_type = _BOUNDINGBOXEVALUATIONOPTIONS
_EVALUATIONCONFIG.oneofs_by_name["vertical_option"].fields.append(
    _EVALUATIONCONFIG.fields_by_name["bounding_box_evaluation_options"]
)
_EVALUATIONCONFIG.fields_by_name[
    "bounding_box_evaluation_options"
].containing_oneof = _EVALUATIONCONFIG.oneofs_by_name["vertical_option"]
_EVALUATIONMETRICS.fields_by_name[
    "classification_metrics"
].message_type = _CLASSIFICATIONMETRICS
_EVALUATIONMETRICS.fields_by_name[
    "object_detection_metrics"
].message_type = _OBJECTDETECTIONMETRICS
_EVALUATIONMETRICS.oneofs_by_name["metrics"].fields.append(
    _EVALUATIONMETRICS.fields_by_name["classification_metrics"]
)
_EVALUATIONMETRICS.fields_by_name[
    "classification_metrics"
].containing_oneof = _EVALUATIONMETRICS.oneofs_by_name["metrics"]
_EVALUATIONMETRICS.oneofs_by_name["metrics"].fields.append(
    _EVALUATIONMETRICS.fields_by_name["object_detection_metrics"]
)
_EVALUATIONMETRICS.fields_by_name[
    "object_detection_metrics"
].containing_oneof = _EVALUATIONMETRICS.oneofs_by_name["metrics"]
_CLASSIFICATIONMETRICS.fields_by_name["pr_curve"].message_type = _PRCURVE
_CLASSIFICATIONMETRICS.fields_by_name[
    "confusion_matrix"
].message_type = _CONFUSIONMATRIX
_OBJECTDETECTIONMETRICS.fields_by_name["pr_curve"].message_type = _PRCURVE
_PRCURVE_CONFIDENCEMETRICSENTRY.containing_type = _PRCURVE
_PRCURVE.fields_by_name[
    "annotation_spec"
].message_type = (
    google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__spec__set__pb2._ANNOTATIONSPEC
)
_PRCURVE.fields_by_name[
    "confidence_metrics_entries"
].message_type = _PRCURVE_CONFIDENCEMETRICSENTRY
_CONFUSIONMATRIX_CONFUSIONMATRIXENTRY.fields_by_name[
    "annotation_spec"
].message_type = (
    google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__spec__set__pb2._ANNOTATIONSPEC
)
_CONFUSIONMATRIX_CONFUSIONMATRIXENTRY.containing_type = _CONFUSIONMATRIX
_CONFUSIONMATRIX_ROW.fields_by_name[
    "annotation_spec"
].message_type = (
    google_dot_cloud_dot_datalabeling__v1beta1_dot_proto_dot_annotation__spec__set__pb2._ANNOTATIONSPEC
)
_CONFUSIONMATRIX_ROW.fields_by_name[
    "entries"
].message_type = _CONFUSIONMATRIX_CONFUSIONMATRIXENTRY
_CONFUSIONMATRIX_ROW.containing_type = _CONFUSIONMATRIX
_CONFUSIONMATRIX.fields_by_name["row"].message_type = _CONFUSIONMATRIX_ROW
DESCRIPTOR.message_types_by_name["Evaluation"] = _EVALUATION
DESCRIPTOR.message_types_by_name["EvaluationConfig"] = _EVALUATIONCONFIG
DESCRIPTOR.message_types_by_name[
    "BoundingBoxEvaluationOptions"
] = _BOUNDINGBOXEVALUATIONOPTIONS
DESCRIPTOR.message_types_by_name["EvaluationMetrics"] = _EVALUATIONMETRICS
DESCRIPTOR.message_types_by_name["ClassificationMetrics"] = _CLASSIFICATIONMETRICS
DESCRIPTOR.message_types_by_name["ObjectDetectionMetrics"] = _OBJECTDETECTIONMETRICS
DESCRIPTOR.message_types_by_name["PrCurve"] = _PRCURVE
DESCRIPTOR.message_types_by_name["ConfusionMatrix"] = _CONFUSIONMATRIX
_sym_db.RegisterFileDescriptor(DESCRIPTOR)

Evaluation = _reflection.GeneratedProtocolMessageType(
    "Evaluation",
    (_message.Message,),
    dict(
        DESCRIPTOR=_EVALUATION,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""Describes an evaluation between 2 annotated datasets. Created by an
  evaluation plan.
  
  
  Attributes:
      name:
          Resource name of an evaluation. Format: 'projects/{project\_id
          }/datasets/{dataset\_id}/evaluations/{evaluation\_id}'
      config:
          Options used in evaluation plan for creating the evaluation.
      evaluation_job_run_time:
          Output only. Timestamp when the evaluation plan triggered this
          evaluation flow.
      create_time:
          Output only. Timestamp when this model evaluation was created.
      evaluation_metrics:
          Output only. Metrics of the evaluation.
      annotation_type:
          Type of the annotation to compute metrics for in the
          groundtruth and annotation labeled dataset. Required for
          creation.
      evaluated_item_count:
          Output only. Count of items in groundtruth dataset included in
          this evaluation. Will be unset if annotation type is not
          applicable.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.Evaluation)
    ),
)
_sym_db.RegisterMessage(Evaluation)

EvaluationConfig = _reflection.GeneratedProtocolMessageType(
    "EvaluationConfig",
    (_message.Message,),
    dict(
        DESCRIPTOR=_EVALUATIONCONFIG,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""
    Attributes:
      vertical_option:
          Vertical specific options for general metrics.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.EvaluationConfig)
    ),
)
_sym_db.RegisterMessage(EvaluationConfig)

BoundingBoxEvaluationOptions = _reflection.GeneratedProtocolMessageType(
    "BoundingBoxEvaluationOptions",
    (_message.Message,),
    dict(
        DESCRIPTOR=_BOUNDINGBOXEVALUATIONOPTIONS,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""Options regarding evaluation between bounding boxes.
  
  
  Attributes:
      iou_threshold:
          Minimize IoU required to consider 2 bounding boxes are
          matched.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.BoundingBoxEvaluationOptions)
    ),
)
_sym_db.RegisterMessage(BoundingBoxEvaluationOptions)

EvaluationMetrics = _reflection.GeneratedProtocolMessageType(
    "EvaluationMetrics",
    (_message.Message,),
    dict(
        DESCRIPTOR=_EVALUATIONMETRICS,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""
    Attributes:
      metrics:
          Common metrics covering most genernal cases.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.EvaluationMetrics)
    ),
)
_sym_db.RegisterMessage(EvaluationMetrics)

ClassificationMetrics = _reflection.GeneratedProtocolMessageType(
    "ClassificationMetrics",
    (_message.Message,),
    dict(
        DESCRIPTOR=_CLASSIFICATIONMETRICS,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""
    Attributes:
      pr_curve:
          Precision-recall curve.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.ClassificationMetrics)
    ),
)
_sym_db.RegisterMessage(ClassificationMetrics)

ObjectDetectionMetrics = _reflection.GeneratedProtocolMessageType(
    "ObjectDetectionMetrics",
    (_message.Message,),
    dict(
        DESCRIPTOR=_OBJECTDETECTIONMETRICS,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""
    Attributes:
      pr_curve:
          Precision-recall curve.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.ObjectDetectionMetrics)
    ),
)
_sym_db.RegisterMessage(ObjectDetectionMetrics)

PrCurve = _reflection.GeneratedProtocolMessageType(
    "PrCurve",
    (_message.Message,),
    dict(
        ConfidenceMetricsEntry=_reflection.GeneratedProtocolMessageType(
            "ConfidenceMetricsEntry",
            (_message.Message,),
            dict(
                DESCRIPTOR=_PRCURVE_CONFIDENCEMETRICSENTRY,
                __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
                __doc__="""
    Attributes:
        confidence_threshold:
            Threshold used for this entry, for example, IoU threshold for
            bounding box problem, or detection threshold for
            classification.
        recall:
            Recall value.
        precision:
            Precision value.
        f1_score:
            Harmonic mean of recall and precision.
        recall_at1:
            Recall value for entries with label that has highest score.
        precision_at1:
            Precision value for entries with label that has highest score.
        f1_score_at1:
            The harmonic mean of [recall\_at1][google.cloud.datalabeling.v
            1beta1.PrCurve.ConfidenceMetricsEntry.recall\_at1] and [precis
            ion\_at1][google.cloud.datalabeling.v1beta1.PrCurve.Confidence
            MetricsEntry.precision\_at1].
        recall_at5:
            Recall value for entries with label that has highest 5 scores.
        precision_at5:
            Precision value for entries with label that has highest 5
            scores.
        f1_score_at5:
            The harmonic mean of [recall\_at5][google.cloud.datalabeling.v
            1beta1.PrCurve.ConfidenceMetricsEntry.recall\_at5] and [precis
            ion\_at5][google.cloud.datalabeling.v1beta1.PrCurve.Confidence
            MetricsEntry.precision\_at5].
    """,
                # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.PrCurve.ConfidenceMetricsEntry)
            ),
        ),
        DESCRIPTOR=_PRCURVE,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""
    Attributes:
      annotation_spec:
          PR curve against which annotation spec. Could be empty.
      area_under_curve:
          Area under precision recall curve.
      confidence_metrics_entries:
          entries to draw PR graph.
      mean_average_precision:
          mean average prcision of this curve.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.PrCurve)
    ),
)
_sym_db.RegisterMessage(PrCurve)
_sym_db.RegisterMessage(PrCurve.ConfidenceMetricsEntry)

ConfusionMatrix = _reflection.GeneratedProtocolMessageType(
    "ConfusionMatrix",
    (_message.Message,),
    dict(
        ConfusionMatrixEntry=_reflection.GeneratedProtocolMessageType(
            "ConfusionMatrixEntry",
            (_message.Message,),
            dict(
                DESCRIPTOR=_CONFUSIONMATRIX_CONFUSIONMATRIXENTRY,
                __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
                __doc__="""
    Attributes:
        annotation_spec:
            The predicted annotation spec.
        item_count:
            Number of items being predicted as this label.
    """,
                # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.ConfusionMatrix.ConfusionMatrixEntry)
            ),
        ),
        Row=_reflection.GeneratedProtocolMessageType(
            "Row",
            (_message.Message,),
            dict(
                DESCRIPTOR=_CONFUSIONMATRIX_ROW,
                __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
                __doc__="""A row in the confusion matrix.
    
    
    Attributes:
        annotation_spec:
            the original annotation spec of this row.
        entries:
            Info describing predicted label distribution.
    """,
                # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.ConfusionMatrix.Row)
            ),
        ),
        DESCRIPTOR=_CONFUSIONMATRIX,
        __module__="google.cloud.datalabeling_v1beta1.proto.evaluation_pb2",
        __doc__="""Confusion matrix of the model running the classification. Not applicable
  when label filtering is specified in evaluation option.
  """,
        # @@protoc_insertion_point(class_scope:google.cloud.datalabeling.v1beta1.ConfusionMatrix)
    ),
)
_sym_db.RegisterMessage(ConfusionMatrix)
_sym_db.RegisterMessage(ConfusionMatrix.ConfusionMatrixEntry)
_sym_db.RegisterMessage(ConfusionMatrix.Row)


DESCRIPTOR._options = None
# @@protoc_insertion_point(module_scope)
