From 8d917a3872e9212f8ff64f379e77b3458b9508de Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Tue, 16 Jun 2026 15:08:27 -0700 Subject: [PATCH] chore: GenAI Client(evals) - add EvaluationExperiment type PiperOrigin-RevId: 933321330 --- agentplatform/_genai/types/__init__.py | 8 +++ agentplatform/_genai/types/common.py | 78 ++++++++++++++++++++++++++ vertexai/_genai/types/__init__.py | 8 +++ vertexai/_genai/types/common.py | 78 ++++++++++++++++++++++++++ 4 files changed, 172 insertions(+) diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index 999aed44fb..62910baf2d 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -505,6 +505,10 @@ from .common import EvaluationDataset from .common import EvaluationDatasetDict from .common import EvaluationDatasetOrDict +from .common import EvaluationExperiment +from .common import EvaluationExperimentDict +from .common import EvaluationExperimentMergeStrategy +from .common import EvaluationExperimentOrDict from .common import EvaluationInstance from .common import EvaluationInstanceDict from .common import EvaluationInstanceOrDict @@ -3284,6 +3288,9 @@ "ObservabilityEvalCase", "ObservabilityEvalCaseDict", "ObservabilityEvalCaseOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", "RubricGroup", "RubricGroupDict", "RubricGroupOrDict", @@ -3380,6 +3387,7 @@ "GenerateMemoriesResponseGeneratedMemoryAction", "SkillRevisionState", "PromptOptimizerMethod", + "EvaluationExperimentMergeStrategy", "OptimizationMethod", "PromptData", "PromptDataDict", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 6db6c8249b..1446ea1f73 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -535,6 +535,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum): """The data driven prompt optimizer designer for prompts from Android core API.""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class OptimizationMethod(_common.CaseInSensitiveEnum): """The method for data driven prompt optimization.""" @@ -22747,6 +22758,73 @@ class ObservabilityEvalCaseDict(TypedDict, total=False): ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict] +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class RubricGroup(_common.BaseModel): """A group of rubrics. diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index bd2836756e..d088967399 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -443,6 +443,10 @@ from .common import EvaluationDataset from .common import EvaluationDatasetDict from .common import EvaluationDatasetOrDict +from .common import EvaluationExperiment +from .common import EvaluationExperimentDict +from .common import EvaluationExperimentMergeStrategy +from .common import EvaluationExperimentOrDict from .common import EvaluationInstance from .common import EvaluationInstanceDict from .common import EvaluationInstanceOrDict @@ -2634,6 +2638,10 @@ "EvaluateMethodConfig", "EvaluateMethodConfigDict", "EvaluateMethodConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", + "EvaluationExperimentMergeStrategy", "EvaluateDatasetConfig", "EvaluateDatasetConfigDict", "EvaluateDatasetConfigOrDict", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index f9ac6e1353..009abda22f 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -498,6 +498,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum): """The data driven prompt optimizer designer for prompts from Android core API.""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class OptimizationMethod(_common.CaseInSensitiveEnum): """The method for data driven prompt optimization.""" @@ -19121,6 +19132,73 @@ class ObservabilityEvalCaseDict(TypedDict, total=False): ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict] +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class RubricGroup(_common.BaseModel): """A group of rubrics.