From e0ec3a7ffa5f6e54b353e60abb7308773e4597ad Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Tue, 16 Jun 2026 15:47:00 -0700 Subject: [PATCH] chore: GenAI Client(evals) - add get/list EvaluationExperiment PiperOrigin-RevId: 933340869 --- agentplatform/_genai/evals.py | 411 +++++++++++++++++++ agentplatform/_genai/types/__init__.py | 30 ++ agentplatform/_genai/types/common.py | 227 ++++++++++ tests/unit/agentplatform/genai/test_evals.py | 92 +++++ vertexai/_genai/evals.py | 407 ++++++++++++++++++ vertexai/_genai/types/__init__.py | 30 ++ vertexai/_genai/types/common.py | 227 ++++++++++ 7 files changed, 1424 insertions(+) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index fd0ae68139..0d9975341a 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -721,6 +721,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -781,6 +795,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1740,6 +1792,78 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _get_evaluation_metric( self, *, @@ -2025,6 +2149,76 @@ def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: @@ -3151,6 +3345,77 @@ def delete_evaluation_metric( config=config, ) + @_common.experimental_warning( + "The Vertex SDK GenAI evals.get_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._get_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.list_evaluation_experiments method is" + " experimental, and may change in future versions." + ) + def list_evaluation_experiments( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None, + ) -> types.ListEvaluationExperimentsResponse: + """Lists EvaluationExperiments. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_experiment"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation experiments response. + """ + if config is None: + config = types.ListEvaluationExperimentsConfig() + if isinstance(config, dict): + config = types.ListEvaluationExperimentsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_experiments( + config=config, + ) + class AsyncEvals(_api_module.BaseModule): @@ -3906,6 +4171,80 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4199,6 +4538,78 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index 999aed44fb..f4c06f4210 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -88,6 +88,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -118,6 +119,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListRagCorporaRequestParameters @@ -505,6 +507,10 @@ from .common import EvaluationDataset from .common import EvaluationDatasetDict from .common import EvaluationDatasetOrDict +from .common import EvaluationExperiment +from .common import EvaluationExperimentDict +from .common import EvaluationExperimentMergeStrategy +from .common import EvaluationExperimentOrDict from .common import EvaluationInstance from .common import EvaluationInstanceDict from .common import EvaluationInstanceOrDict @@ -682,6 +688,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -830,6 +839,12 @@ from .common import ListDatasetVersionsResponse from .common import ListDatasetVersionsResponseDict from .common import ListDatasetVersionsResponseOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -2171,6 +2186,12 @@ "GenerateInstanceRubricsResponse", "GenerateInstanceRubricsResponseDict", "GenerateInstanceRubricsResponseOrDict", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", "GetEvaluationMetricConfig", "GetEvaluationMetricConfigDict", "GetEvaluationMetricConfigOrDict", @@ -2183,6 +2204,12 @@ "GetEvaluationItemConfig", "GetEvaluationItemConfigDict", "GetEvaluationItemConfigOrDict", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "ListEvaluationMetricsConfig", "ListEvaluationMetricsConfigDict", "ListEvaluationMetricsConfigOrDict", @@ -3375,6 +3402,7 @@ "EvaluationItemType", "SamplingMethod", "EvaluationRunState", + "EvaluationExperimentMergeStrategy", "OptimizeTarget", "MemoryMetadataMergeStrategy", "GenerateMemoriesResponseGeneratedMemoryAction", @@ -3418,10 +3446,12 @@ "_GenerateUserScenariosParameters", "_GenerateLossClustersParameters", "_GenerateInstanceRubricsRequest", + "_GetEvaluationExperimentParameters", "_GetEvaluationMetricParameters", "_GetEvaluationRunParameters", "_GetEvaluationSetParameters", "_GetEvaluationItemParameters", + "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", "_OptimizeRequestParameters", "_CustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 6db6c8249b..6df36023e2 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -472,6 +472,17 @@ class EvaluationRunState(_common.CaseInSensitiveEnum): """Evaluation run is performing rubric generation.""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class OptimizeTarget(_common.CaseInSensitiveEnum): """Specifies the method for calling the optimize_prompt.""" @@ -5620,6 +5631,117 @@ class GenerateInstanceRubricsResponseDict(TypedDict, total=False): ] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class GetEvaluationMetricConfig(_common.BaseModel): """Config for getting an evaluation metric.""" @@ -5786,6 +5908,111 @@ class _GetEvaluationItemParametersDict(TypedDict, total=False): ] +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponseDict] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class ListEvaluationMetricsConfig(_common.BaseModel): """Config for listing evaluation metrics.""" diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 3a5ff27c74..45d08cd792 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -9849,3 +9849,95 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self) request_body.get("evaluationConfig", {}).get("allowCrossRegionModel") is True ) + + +class TestGetEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": self.experiment_name, + "displayName": "my_experiment", + "evaluationRuns": [ + "projects/123/locations/us-central1/evaluationRuns/789" + ], + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_get_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.get_evaluation_experiment(name=self.experiment_name) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.name == self.experiment_name + assert experiment.display_name == "my_experiment" + assert experiment.evaluation_runs == [ + "projects/123/locations/us-central1/evaluationRuns/789" + ] + + def test_get_evaluation_experiment_uses_short_name_in_url(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.get_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path == "evaluationExperiments/456" + + def test_get_evaluation_experiment_empty_name_raises(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + with pytest.raises(ValueError, match="name cannot be empty"): + evals_module.get_evaluation_experiment(name="") + + +class TestListEvaluationExperiments: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "evaluationExperiments": [ + { + "name": "projects/123/locations/us-central1/evaluationExperiments/1", + "displayName": "exp_1", + }, + { + "name": "projects/123/locations/us-central1/evaluationExperiments/2", + "displayName": "exp_2", + }, + ] + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_list_evaluation_experiments_returns_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + response = evals_module.list_evaluation_experiments() + + assert len(response.evaluation_experiments) == 2 + assert response.evaluation_experiments[0].display_name == "exp_1" + assert response.evaluation_experiments[1].display_name == "exp_2" + + def test_list_evaluation_experiments_passes_filter_and_order_by(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.list_evaluation_experiments( + filter='display_name="exp_1"', order_by="create_time desc" + ) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path.startswith("evaluationExperiments?") + assert "orderBy=create_time+desc" in path diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 6f5a9a3e58..42fb6e7227 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -721,6 +721,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -781,6 +795,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1740,6 +1792,78 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _get_evaluation_metric( self, *, @@ -2025,6 +2149,76 @@ def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: @@ -3151,6 +3345,73 @@ def delete_evaluation_metric( config=config, ) + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._get_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.list_evaluation_experiments method is" + " experimental, and may change in future versions." + ) + def list_evaluation_experiments( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None, + ) -> types.ListEvaluationExperimentsResponse: + """Lists EvaluationExperiments. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_experiment"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation experiments response. + """ + if config is None: + config = types.ListEvaluationExperimentsConfig() + if isinstance(config, dict): + config = types.ListEvaluationExperimentsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_experiments( + config=config, + ) + class AsyncEvals(_api_module.BaseModule): @@ -3906,6 +4167,80 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4199,6 +4534,78 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index bd2836756e..040a6323d3 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -83,6 +83,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -107,6 +108,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListSandboxEnvironmentSnapshotsRequestParameters @@ -443,6 +445,10 @@ from .common import EvaluationDataset from .common import EvaluationDatasetDict from .common import EvaluationDatasetOrDict +from .common import EvaluationExperiment +from .common import EvaluationExperimentDict +from .common import EvaluationExperimentMergeStrategy +from .common import EvaluationExperimentOrDict from .common import EvaluationInstance from .common import EvaluationInstanceDict from .common import EvaluationInstanceOrDict @@ -608,6 +614,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -711,6 +720,12 @@ from .common import ListDatasetVersionsResponse from .common import ListDatasetVersionsResponseDict from .common import ListDatasetVersionsResponseOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -2634,6 +2649,21 @@ "EvaluateMethodConfig", "EvaluateMethodConfigDict", "EvaluateMethodConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", + "EvaluationExperimentMergeStrategy", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", + "_GetEvaluationExperimentParameters", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "_ListEvaluationExperimentsParameters", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "EvaluateDatasetConfig", "EvaluateDatasetConfigDict", "EvaluateDatasetConfigOrDict", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index f9ac6e1353..8392788cc8 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -498,6 +498,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum): """The data driven prompt optimizer designer for prompts from Android core API.""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class OptimizationMethod(_common.CaseInSensitiveEnum): """The method for data driven prompt optimization.""" @@ -19121,6 +19132,222 @@ class ObservabilityEvalCaseDict(TypedDict, total=False): ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict] +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponseDict] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class RubricGroup(_common.BaseModel): """A group of rubrics.