Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions agentplatform/_genai/types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,10 @@
from .common import EvaluationDataset
from .common import EvaluationDatasetDict
from .common import EvaluationDatasetOrDict
from .common import EvaluationExperiment
from .common import EvaluationExperimentDict
from .common import EvaluationExperimentMergeStrategy
from .common import EvaluationExperimentOrDict
from .common import EvaluationInstance
from .common import EvaluationInstanceDict
from .common import EvaluationInstanceOrDict
Expand Down Expand Up @@ -3284,6 +3288,9 @@
"ObservabilityEvalCase",
"ObservabilityEvalCaseDict",
"ObservabilityEvalCaseOrDict",
"EvaluationExperiment",
"EvaluationExperimentDict",
"EvaluationExperimentOrDict",
"RubricGroup",
"RubricGroupDict",
"RubricGroupOrDict",
Expand Down Expand Up @@ -3380,6 +3387,7 @@
"GenerateMemoriesResponseGeneratedMemoryAction",
"SkillRevisionState",
"PromptOptimizerMethod",
"EvaluationExperimentMergeStrategy",
"OptimizationMethod",
"PromptData",
"PromptDataDict",
Expand Down
78 changes: 78 additions & 0 deletions agentplatform/_genai/types/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum):
"""The data driven prompt optimizer designer for prompts from Android core API."""


class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum):
"""Merge strategy for the evaluation experiment."""

MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED"
"""Unspecified merge strategy."""
SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY"
"""Default. Runs are treated as an independent, sequential history."""
SHARED_RESULT_SET = "SHARED_RESULT_SET"
"""Runs are parallel iterations contributing to a shared result set."""


class OptimizationMethod(_common.CaseInSensitiveEnum):
"""The method for data driven prompt optimization."""

Expand Down Expand Up @@ -22747,6 +22758,73 @@ class ObservabilityEvalCaseDict(TypedDict, total=False):
ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict]


class EvaluationExperiment(_common.BaseModel):
"""Represents an experiment for iterating on and visualizing evaluation runs."""

name: Optional[str] = Field(
default=None,
description="""The resource name of the EvaluationExperiment. Format:
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""",
)
display_name: Optional[str] = Field(
default=None, description="""The display name of the evaluation experiment."""
)
evaluation_runs: Optional[list[str]] = Field(
default=None,
description="""The EvaluationRuns that are part of this experiment.""",
)
labels: Optional[dict[str, str]] = Field(
default=None, description="""Labels for the evaluation experiment."""
)
merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field(
default=None, description="""Merge strategy for the evaluation experiment."""
)
metadata: Optional[dict[str, Any]] = Field(
default=None,
description="""Metadata about the evaluation experiment, can be used by the caller
to store additional tracking information about the experiment.""",
)
create_time: Optional[datetime.datetime] = Field(
default=None, description="""Timestamp when this experiment was created."""
)
update_time: Optional[datetime.datetime] = Field(
default=None, description="""Timestamp when this experiment was last updated."""
)


class EvaluationExperimentDict(TypedDict, total=False):
"""Represents an experiment for iterating on and visualizing evaluation runs."""

name: Optional[str]
"""The resource name of the EvaluationExperiment. Format:
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`."""

display_name: Optional[str]
"""The display name of the evaluation experiment."""

evaluation_runs: Optional[list[str]]
"""The EvaluationRuns that are part of this experiment."""

labels: Optional[dict[str, str]]
"""Labels for the evaluation experiment."""

merge_strategy: Optional[EvaluationExperimentMergeStrategy]
"""Merge strategy for the evaluation experiment."""

metadata: Optional[dict[str, Any]]
"""Metadata about the evaluation experiment, can be used by the caller
to store additional tracking information about the experiment."""

create_time: Optional[datetime.datetime]
"""Timestamp when this experiment was created."""

update_time: Optional[datetime.datetime]
"""Timestamp when this experiment was last updated."""


EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict]


class RubricGroup(_common.BaseModel):
"""A group of rubrics.

Expand Down
8 changes: 8 additions & 0 deletions vertexai/_genai/types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,10 @@
from .common import EvaluationDataset
from .common import EvaluationDatasetDict
from .common import EvaluationDatasetOrDict
from .common import EvaluationExperiment
from .common import EvaluationExperimentDict
from .common import EvaluationExperimentMergeStrategy
from .common import EvaluationExperimentOrDict
from .common import EvaluationInstance
from .common import EvaluationInstanceDict
from .common import EvaluationInstanceOrDict
Expand Down Expand Up @@ -2634,6 +2638,10 @@
"EvaluateMethodConfig",
"EvaluateMethodConfigDict",
"EvaluateMethodConfigOrDict",
"EvaluationExperiment",
"EvaluationExperimentDict",
"EvaluationExperimentOrDict",
"EvaluationExperimentMergeStrategy",
"EvaluateDatasetConfig",
"EvaluateDatasetConfigDict",
"EvaluateDatasetConfigOrDict",
Expand Down
78 changes: 78 additions & 0 deletions vertexai/_genai/types/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum):
"""The data driven prompt optimizer designer for prompts from Android core API."""


class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum):
"""Merge strategy for the evaluation experiment."""

MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED"
"""Unspecified merge strategy."""
SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY"
"""Default. Runs are treated as an independent, sequential history."""
SHARED_RESULT_SET = "SHARED_RESULT_SET"
"""Runs are parallel iterations contributing to a shared result set."""


class OptimizationMethod(_common.CaseInSensitiveEnum):
"""The method for data driven prompt optimization."""

Expand Down Expand Up @@ -19121,6 +19132,73 @@ class ObservabilityEvalCaseDict(TypedDict, total=False):
ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict]


class EvaluationExperiment(_common.BaseModel):
"""Represents an experiment for iterating on and visualizing evaluation runs."""

name: Optional[str] = Field(
default=None,
description="""The resource name of the EvaluationExperiment. Format:
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""",
)
display_name: Optional[str] = Field(
default=None, description="""The display name of the evaluation experiment."""
)
evaluation_runs: Optional[list[str]] = Field(
default=None,
description="""The EvaluationRuns that are part of this experiment.""",
)
labels: Optional[dict[str, str]] = Field(
default=None, description="""Labels for the evaluation experiment."""
)
merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field(
default=None, description="""Merge strategy for the evaluation experiment."""
)
metadata: Optional[dict[str, Any]] = Field(
default=None,
description="""Metadata about the evaluation experiment, can be used by the caller
to store additional tracking information about the experiment.""",
)
create_time: Optional[datetime.datetime] = Field(
default=None, description="""Timestamp when this experiment was created."""
)
update_time: Optional[datetime.datetime] = Field(
default=None, description="""Timestamp when this experiment was last updated."""
)


class EvaluationExperimentDict(TypedDict, total=False):
"""Represents an experiment for iterating on and visualizing evaluation runs."""

name: Optional[str]
"""The resource name of the EvaluationExperiment. Format:
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`."""

display_name: Optional[str]
"""The display name of the evaluation experiment."""

evaluation_runs: Optional[list[str]]
"""The EvaluationRuns that are part of this experiment."""

labels: Optional[dict[str, str]]
"""Labels for the evaluation experiment."""

merge_strategy: Optional[EvaluationExperimentMergeStrategy]
"""Merge strategy for the evaluation experiment."""

metadata: Optional[dict[str, Any]]
"""Metadata about the evaluation experiment, can be used by the caller
to store additional tracking information about the experiment."""

create_time: Optional[datetime.datetime]
"""Timestamp when this experiment was created."""

update_time: Optional[datetime.datetime]
"""Timestamp when this experiment was last updated."""


EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict]


class RubricGroup(_common.BaseModel):
"""A group of rubrics.

Expand Down
Loading