From 364775307aa9c0b4d30844da4c7f23c05221dfcd Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Thu, 1 Oct 2026 06:41:34 -0700 Subject: [PATCH 1/7] fix: replay session_events into a scratch session and delete it afterwards `AdkApp.async_stream_query(session_events=...)` raised `AttributeError` in the `vertexai` template and, in every template, left behind the managed session it created for the replayed events. The session is now read back before the replay and deleted when the query ends. Fixes #7118, fixes #7119 PiperOrigin-RevId: 991643489 --- agentplatform/frameworks/adk.py | 53 +++-- .../frameworks/test_frameworks_adk.py | 63 +++++- .../test_agent_engine_templates_adk.py | 204 ++++++++++++++++++ vertexai/agent_engines/templates/adk.py | 58 +++-- 4 files changed, 340 insertions(+), 38 deletions(-) diff --git a/agentplatform/frameworks/adk.py b/agentplatform/frameworks/adk.py index 45d755453a..f6a0fe9976 100644 --- a/agentplatform/frameworks/adk.py +++ b/agentplatform/frameworks/adk.py @@ -1209,6 +1209,8 @@ async def async_stream_query( session_events (Optional[List[Dict[str, Any]]]): Optional. The session events to use for the query. This will be used to initialize the session if `session_id` is not provided. + That session exists only for this query and is deleted once the + stream ends. run_config (Optional[Dict[str, Any]]): Optional. The run config to use for the query. If you want to pass in a `run_config` pydantic object, you can pass in a dict @@ -1244,10 +1246,15 @@ async def async_stream_query( raise ValueError( "Only one of session_id and session_events should be specified." ) + scratch_session_id = None if not session_id: session = await self.async_create_session(user_id=user_id) session_id = session["id"] if session_events is not None: + scratch_session_id = session_id + + try: + if scratch_session_id: # We allow for session_events to be an empty list. from google.adk.events.event import Event @@ -1255,7 +1262,7 @@ async def async_stream_query( session_obj = await session_service.get_session( app_name=self._app_name(), user_id=user_id, - session_id=session_id, + session_id=scratch_session_id, ) for event in session_events: if not isinstance(event, Event): @@ -1265,28 +1272,38 @@ async def async_stream_query( event=event, ) - run_config = _validate_run_config(run_config) - if run_config: - events_async = self._tmpl_attrs.get("runner").run_async( - user_id=user_id, - session_id=session_id, - new_message=content, - run_config=run_config, - **kwargs, - ) - else: - events_async = self._tmpl_attrs.get("runner").run_async( - user_id=user_id, - session_id=session_id, - new_message=content, - **kwargs, - ) + run_config = _validate_run_config(run_config) + if run_config: + events_async = self._tmpl_attrs.get("runner").run_async( + user_id=user_id, + session_id=session_id, + new_message=content, + run_config=run_config, + **kwargs, + ) + else: + events_async = self._tmpl_attrs.get("runner").run_async( + user_id=user_id, + session_id=session_id, + new_message=content, + **kwargs, + ) - try: async for event in events_async: # Yield the event data as a dictionary yield _runtimes_utils.dump_event_for_json(event) finally: + # The caller never sees the id of a session created for + # session_events, so nothing else can delete it. + if scratch_session_id: + try: + await self._tmpl_attrs.get("session_service").delete_session( + app_name=self._app_name(), + user_id=user_id, + session_id=scratch_session_id, + ) + except Exception as e: # pylint: disable=broad-exception-caught + _warn(f"Failed to delete scratch session {scratch_session_id}: {e}") # Avoid telemetry data loss having to do with CPU throttling on instance turndown _ = await _force_flush_otel( tracing_enabled=self._tracing_enabled(), diff --git a/tests/unit/agentplatform/frameworks/test_frameworks_adk.py b/tests/unit/agentplatform/frameworks/test_frameworks_adk.py index 11932c7c1b..d646a5123c 100644 --- a/tests/unit/agentplatform/frameworks/test_frameworks_adk.py +++ b/tests/unit/agentplatform/frameworks/test_frameworks_adk.py @@ -612,7 +612,7 @@ async def test_async_stream_query_with_empty_session_events( events.append(event) assert app._tmpl_attrs.get("session_service") is not None sessions = app.list_sessions(user_id=_TEST_USER_ID) - assert len(sessions.sessions) == 1 + assert not sessions.sessions @pytest.mark.asyncio async def test_async_stream_query_with_session_events( @@ -633,7 +633,66 @@ async def test_async_stream_query_with_session_events( events.append(event) assert app._tmpl_attrs.get("session_service") is not None sessions = app.list_sessions(user_id=_TEST_USER_ID) - assert len(sessions.sessions) == 1 + assert not sessions.sessions + + @pytest.mark.asyncio + async def test_async_stream_query_deletes_session_when_replay_fails( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = adk_template.AdkApp(agent=Agent(name=_TEST_AGENT_NAME, model=_TEST_MODEL)) + app.set_up() + app._tmpl_attrs["runner"] = _MockRunner() + with pytest.raises(ValueError): + async for _ in app.async_stream_query( + user_id=_TEST_USER_ID, + session_events=[123], + message="test message", + ): + pass + assert not app.list_sessions(user_id=_TEST_USER_ID).sessions + + @pytest.mark.asyncio + async def test_async_stream_query_keeps_session_without_session_events( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = adk_template.AdkApp(agent=Agent(name=_TEST_AGENT_NAME, model=_TEST_MODEL)) + app.set_up() + app._tmpl_attrs["runner"] = _MockRunner() + async for _ in app.async_stream_query( + user_id=_TEST_USER_ID, + message="test message", + ): + pass + assert len(app.list_sessions(user_id=_TEST_USER_ID).sessions) == 1 + + @pytest.mark.asyncio + async def test_async_stream_query_warns_when_session_delete_fails( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = adk_template.AdkApp(agent=Agent(name=_TEST_AGENT_NAME, model=_TEST_MODEL)) + app.set_up() + app._tmpl_attrs["runner"] = _MockRunner() + with mock.patch.object( + app._tmpl_attrs["session_service"], + "delete_session", + side_effect=RuntimeError("delete failed"), + ), mock.patch.object(adk_template, "_warn") as warn_mock: + events = [ + event + async for event in app.async_stream_query( + user_id=_TEST_USER_ID, + session_events=[], + message="test message", + ) + ] + assert len(events) == 1 + warn_mock.assert_called_once() @pytest.mark.asyncio @mock.patch.dict( diff --git a/tests/unit/vertex_adk/test_agent_engine_templates_adk.py b/tests/unit/vertex_adk/test_agent_engine_templates_adk.py index a0d36dc1ac..c5b3bbdd3f 100644 --- a/tests/unit/vertex_adk/test_agent_engine_templates_adk.py +++ b/tests/unit/vertex_adk/test_agent_engine_templates_adk.py @@ -70,6 +70,83 @@ def __init__(self, name: str, model: str): _TEST_USER_ID = "test_user_id" _TEST_AGENT_NAME = "test_agent" _TEST_AGENT = Agent(name=_TEST_AGENT_NAME, model=_TEST_MODEL) +_TEST_SESSION_EVENTS = [ + { + "author": "user", + "content": { + "parts": [ + { + "text": "What is the exchange rate from US dollars to " + "Swedish krona on 2025-09-25?" + } + ], + "role": "user", + }, + "id": "8967297909049524224", + "invocationId": "e-308f65d7-a99f-41e3-b80d-40feb5f1b065", + "timestamp": 1765832134.629513, + }, + { + "author": "currency_exchange_agent", + "content": { + "parts": [ + { + "functionCall": { + "args": { + "currency_date": "2025-09-25", + "currency_from": "USD", + "currency_to": "SEK", + }, + "id": "adk-136738ad-9e57-4cfb-8e23-b0f3e50a37d7", + "name": "get_exchange_rate", + } + } + ], + "role": "model", + }, + "id": "3155402589927899136", + "invocationId": "e-308f65d7-a99f-41e3-b80d-40feb5f1b065", + "timestamp": 1765832134.723713, + }, + { + "author": "currency_exchange_agent", + "content": { + "parts": [ + { + "functionResponse": { + "id": "adk-136738ad-9e57-4cfb-8e23-b0f3e50a37d7", + "name": "get_exchange_rate", + "response": { + "amount": 1, + "base": "USD", + "date": "2025-09-25", + "rates": {"SEK": 9.4118}, + }, + } + } + ], + "role": "user", + }, + "id": "1678221912150376448", + "invocationId": "e-308f65d7-a99f-41e3-b80d-40feb5f1b065", + "timestamp": 1765832135.764961, + }, + { + "author": "currency_exchange_agent", + "content": { + "parts": [ + { + "text": "The exchange rate from US dollars to Swedish " + "krona on 2025-09-25 is 1 USD to 9.4118 SEK." + } + ], + "role": "model", + }, + "id": "2470855446567583744", + "invocationId": "e-308f65d7-a99f-41e3-b80d-40feb5f1b065", + "timestamp": 1765832135.853299, + }, +] _TEST_SESSION = { "id": "ca18c25a-644b-4e13-9b24-78c150ec3eb9", "app_name": "default_app_name", @@ -317,6 +394,23 @@ async def run_async(self, *args, **kwargs): ) +class _SessionReadingRunner: + """Records which events the session holds when the run starts.""" + + def __init__(self, session_service, app_name): + self._session_service = session_service + self._app_name = app_name + self.seen_event_ids = None + + async def run_async(self, *, user_id, session_id, **kwargs): + session = await self._session_service.get_session( + app_name=self._app_name, user_id=user_id, session_id=session_id + ) + self.seen_event_ids = [event.id for event in session.events] + async for event in _MockRunner().run_async(): + yield event + + @pytest.mark.usefixtures("google_auth_mock") class TestAdkApp: def test_adk_version(self): @@ -436,6 +530,116 @@ async def test_async_stream_query( events.append(event) assert len(events) == 1 + @pytest.mark.asyncio + async def test_async_stream_query_replays_session_events_then_deletes_session( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = agent_engines.AdkApp(agent=_TEST_AGENT) + app.set_up() + runner = _SessionReadingRunner( + app._tmpl_attrs["session_service"], app._app_name() + ) + app._tmpl_attrs["runner"] = runner + events = [ + event + async for event in app.async_stream_query( + user_id=_TEST_USER_ID, + session_events=_TEST_SESSION_EVENTS, + message="on the day after that?", + ) + ] + assert len(events) == 1 + assert runner.seen_event_ids == [e["id"] for e in _TEST_SESSION_EVENTS] + assert not app.list_sessions(user_id=_TEST_USER_ID).sessions + + @pytest.mark.asyncio + async def test_async_stream_query_deletes_session_when_replay_fails( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = agent_engines.AdkApp(agent=_TEST_AGENT) + app.set_up() + app._tmpl_attrs["runner"] = _MockRunner() + with pytest.raises(ValueError): + async for _ in app.async_stream_query( + user_id=_TEST_USER_ID, + session_events=[123], + message="test message", + ): + pass + assert not app.list_sessions(user_id=_TEST_USER_ID).sessions + + @pytest.mark.asyncio + async def test_async_stream_query_keeps_session_without_session_events( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = agent_engines.AdkApp(agent=_TEST_AGENT) + app.set_up() + app._tmpl_attrs["runner"] = _MockRunner() + async for _ in app.async_stream_query( + user_id=_TEST_USER_ID, + message="test message", + ): + pass + assert len(app.list_sessions(user_id=_TEST_USER_ID).sessions) == 1 + + @pytest.mark.asyncio + async def test_async_stream_query_warns_when_session_delete_fails( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = agent_engines.AdkApp(agent=_TEST_AGENT) + app.set_up() + app._tmpl_attrs["runner"] = _MockRunner() + with mock.patch.object( + app._tmpl_attrs["session_service"], + "delete_session", + side_effect=RuntimeError("delete failed"), + ), mock.patch.object(adk_template, "_warn") as warn_mock: + events = [ + event + async for event in app.async_stream_query( + user_id=_TEST_USER_ID, + session_events=[], + message="test message", + ) + ] + assert len(events) == 1 + warn_mock.assert_called_once() + + @pytest.mark.asyncio + async def test_async_stream_query_passes_run_config( + self, + default_instrumentor_builder_mock: mock.Mock, + get_project_id_mock: mock.Mock, + ): + app = agent_engines.AdkApp(agent=_TEST_AGENT) + app.set_up() + runner_kwargs = {} + + async def run_async(**kwargs): + runner_kwargs.update(kwargs) + async for event in _MockRunner().run_async(): + yield event + + app._tmpl_attrs["runner"] = mock.Mock(run_async=run_async) + events = [ + event + async for event in app.async_stream_query( + user_id=_TEST_USER_ID, + message="test message", + run_config={"max_llm_calls": 5}, + ) + ] + assert len(events) == 1 + assert runner_kwargs["run_config"].max_llm_calls == 5 + def test_set_up_runner_auto_create_session_enabled( self, default_instrumentor_builder_mock: mock.Mock, diff --git a/vertexai/agent_engines/templates/adk.py b/vertexai/agent_engines/templates/adk.py index a2e700cdc2..10521ac85e 100644 --- a/vertexai/agent_engines/templates/adk.py +++ b/vertexai/agent_engines/templates/adk.py @@ -1180,6 +1180,8 @@ async def async_stream_query( session_events (Optional[List[Dict[str, Any]]]): Optional. The session events to use for the query. This will be used to initialize the session if `session_id` is not provided. + That session exists only for this query and is deleted once the + stream ends. run_config (Optional[Dict[str, Any]]): Optional. The run config to use for the query. If you want to pass in a `run_config` pydantic object, you can pass in a dict @@ -1215,44 +1217,64 @@ async def async_stream_query( raise ValueError( "Only one of session_id and session_events should be specified." ) + scratch_session_id = None if not session_id: session = await self.async_create_session(user_id=user_id) session_id = session["id"] if session_events is not None: + scratch_session_id = session_id + + try: + if scratch_session_id: # We allow for session_events to be an empty list. from google.adk.events.event import Event session_service = self._tmpl_attrs.get("session_service") + session_obj = await session_service.get_session( + app_name=self._app_name(), + user_id=user_id, + session_id=scratch_session_id, + ) for event in session_events: if not isinstance(event, Event): event = Event.model_validate(event) await session_service.append_event( - session=session, + session=session_obj, event=event, ) - run_config = _validate_run_config(run_config) - if run_config: - events_async = self._tmpl_attrs.get("runner").run_async( - user_id=user_id, - session_id=session_id, - new_message=content, - run_config=run_config, - **kwargs, - ) - else: - events_async = self._tmpl_attrs.get("runner").run_async( - user_id=user_id, - session_id=session_id, - new_message=content, - **kwargs, - ) + run_config = _validate_run_config(run_config) + if run_config: + events_async = self._tmpl_attrs.get("runner").run_async( + user_id=user_id, + session_id=session_id, + new_message=content, + run_config=run_config, + **kwargs, + ) + else: + events_async = self._tmpl_attrs.get("runner").run_async( + user_id=user_id, + session_id=session_id, + new_message=content, + **kwargs, + ) - try: async for event in events_async: # Yield the event data as a dictionary yield _utils.dump_event_for_json(event) finally: + # The caller never sees the id of a session created for + # session_events, so nothing else can delete it. + if scratch_session_id: + try: + await self._tmpl_attrs.get("session_service").delete_session( + app_name=self._app_name(), + user_id=user_id, + session_id=scratch_session_id, + ) + except Exception as e: # pylint: disable=broad-exception-caught + _warn(f"Failed to delete scratch session {scratch_session_id}: {e}") # Avoid telemetry data loss having to do with CPU throttling on instance turndown _ = await _force_flush_otel( tracing_enabled=self._tracing_enabled(), From 5fa14552cd06179e8ea08f0ea0d29ea6cafb1ca0 Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Thu, 1 Oct 2026 11:23:33 -0700 Subject: [PATCH 2/7] feat: Add per-step judges and cross-region routing to evals PiperOrigin-RevId: 991795392 --- agentplatform/_genai/_evals_common.py | 37 +-- .../_genai/_evals_metric_handlers.py | 26 +- agentplatform/_genai/_evals_metric_loaders.py | 13 +- agentplatform/_genai/_transformers.py | 6 +- agentplatform/_genai/evals.py | 14 + agentplatform/_genai/types/common.py | 43 ++- .../test_evaluate_predefined_metrics.py | 83 ++++++ tests/unit/agentplatform/genai/test_evals.py | 248 ++++++++++++++++++ 8 files changed, 440 insertions(+), 30 deletions(-) diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index db85f3819d..b448395ecb 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -2886,6 +2886,21 @@ def _resolve_dataset_inputs( return processed_eval_dataset, num_response_candidates +def _prebuilt_evaluation_run_metric( + resolved_metric: types.Metric, +) -> types.EvaluationRunMetric: + """Builds the evaluation run metric for a resolved RubricMetric.""" + if resolved_metric.name in _evals_constant.SUPPORTED_PREDEFINED_METRICS: + metric_config = t.t_metrics([resolved_metric])[0] + else: + metric_config = { + "predefined_metric_spec": {"metric_spec_name": resolved_metric.name} + } + return types.EvaluationRunMetric( + metric=resolved_metric.name, metric_config=metric_config + ) + + def _resolve_evaluation_run_metrics( metrics: Union[list[types.EvaluationRunMetric], list[types.Metric]], api_client: Any ) -> list[types.EvaluationRunMetric]: @@ -2903,14 +2918,7 @@ def _resolve_evaluation_run_metrics( resolved_metric = metric_instance.resolve(api_client=api_client) if resolved_metric.name: resolved_metrics_list.append( - types.EvaluationRunMetric( - metric=resolved_metric.name, - metric_config=types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name=resolved_metric.name, - ) - ), - ) + _prebuilt_evaluation_run_metric(resolved_metric) ) except Exception as e: logger.error( @@ -2944,14 +2952,7 @@ def _resolve_evaluation_run_metrics( ) if resolved_metric.name: resolved_metrics_list.append( - types.EvaluationRunMetric( - metric=resolved_metric.name, - metric_config=types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name=resolved_metric.name, - ) - ), - ) + _prebuilt_evaluation_run_metric(resolved_metric) ) else: raise TypeError( @@ -3020,6 +3021,7 @@ def _execute_evaluation( # type: ignore[no-untyped-def] dest: Optional[str] = None, location: Optional[str] = None, evaluation_service_qps: Optional[float] = None, + allow_cross_region_model: Optional[bool] = None, **kwargs, ) -> types.EvaluationResult: """Evaluates a dataset using the provided metrics. @@ -3035,6 +3037,8 @@ def _execute_evaluation( # type: ignore[no-untyped-def] evaluation_service_qps: The rate limit (queries per second) for calls to the evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota. + allow_cross_region_model: Opt-in flag to authorize cross-region + routing for judge models. **kwargs: Extra arguments to pass to evaluation, such as `agent_info`. Returns: @@ -3117,6 +3121,7 @@ def _execute_evaluation( # type: ignore[no-untyped-def] evaluation_result = _evals_metric_handlers.compute_metrics_and_aggregate( evaluation_run_config, evaluation_service_qps=evaluation_service_qps, + allow_cross_region_model=allow_cross_region_model, ) t2 = time.perf_counter() logger.info("Evaluation took: %f seconds", t2 - t1) diff --git a/agentplatform/_genai/_evals_metric_handlers.py b/agentplatform/_genai/_evals_metric_handlers.py index 8a7bf4c3b4..f8fafb4b53 100644 --- a/agentplatform/_genai/_evals_metric_handlers.py +++ b/agentplatform/_genai/_evals_metric_handlers.py @@ -288,6 +288,7 @@ class MetricHandler(abc.ABC, Generic[T]): def __init__(self, module: "evals.Evals", metric: T): self.module = module self.metric: T = metric + self.allow_cross_region_model: Optional[bool] = None @property @abc.abstractmethod @@ -761,6 +762,7 @@ def get_metric_result( lambda: self.module._evaluate_instances( metrics=[self.metric], instance=instance, + allow_cross_region_model=self.allow_cross_region_model, ), self.metric_name, ) @@ -981,7 +983,7 @@ def __init__(self, module: "evals.Evals", metric: types.Metric): raise ValueError( f"Metric '{self.metric.name}' is not a supported predefined metric." ) - if ( + if self.metric.name.startswith("multi_turn") and ( self.metric.judge_model or self.metric.judge_model_generation_config or self.metric.judge_model_sampling_count @@ -989,7 +991,8 @@ def __init__(self, module: "evals.Evals", metric: types.Metric): logger.warning( "Autorater config settings (judge_model, " "judge_model_generation_config, judge_model_sampling_count) " - "are ignored for predefined metric '%s'.", + "are ignored for multi-turn metric '%s'. Use " + "judge_model_step_configs to set its judges.", self.metric.name, ) @@ -1071,6 +1074,7 @@ def get_metric_result( metrics=[self.metric], instance=payload.get("instance"), autorater_config=payload.get("autorater_config"), + allow_cross_region_model=self.allow_cross_region_model, ), metric_name, ) @@ -1318,6 +1322,7 @@ def get_metric_result( metric_sources=[metric_source], instance=payload.get("instance"), autorater_config=payload.get("autorater_config"), + allow_cross_region_model=self.allow_cross_region_model, ), metric_name, ) @@ -1404,12 +1409,16 @@ def aggregate( def get_handler_for_metric( - module: "evals.Evals", metric: types.Metric + module: "evals.Evals", + metric: types.Metric, + allow_cross_region_model: Optional[bool] = None, ) -> Union[MetricHandlerType, Any]: """Returns a metric handler for the given metric.""" for condition, handler_class in _METRIC_HANDLER_MAPPING: if condition(metric): # type: ignore[no-untyped-call] - return handler_class(module=module, metric=metric) + handler = handler_class(module=module, metric=metric) + handler.allow_cross_region_model = allow_cross_region_model + return handler raise ValueError(f"Unsupported metric: {metric.name}") @@ -1548,6 +1557,7 @@ def _rate_limited_get_metric_result( def compute_metrics_and_aggregate( evaluation_run_config: EvaluationRunConfig, evaluation_service_qps: Optional[float] = None, + allow_cross_region_model: Optional[bool] = None, ) -> types.EvaluationResult: """Computes metrics and aggregates them for a given evaluation run config. @@ -1556,6 +1566,8 @@ def compute_metrics_and_aggregate( evaluation_service_qps: Optional QPS limit for the evaluation service. Defaults to _DEFAULT_EVAL_SERVICE_QPS (10). Users with higher quotas can increase this value. + allow_cross_region_model: Opt-in flag to authorize cross-region + routing for judge models. """ metric_handlers = [] all_futures = [] @@ -1574,7 +1586,11 @@ def compute_metrics_and_aggregate( for eval_metric in evaluation_run_config.metrics: metric_handlers.append( - get_handler_for_metric(evaluation_run_config.evals_module, eval_metric) + get_handler_for_metric( + evaluation_run_config.evals_module, + eval_metric, + allow_cross_region_model=allow_cross_region_model, + ) ) eval_case_count = len(evaluation_run_config.dataset.eval_cases) diff --git a/agentplatform/_genai/_evals_metric_loaders.py b/agentplatform/_genai/_evals_metric_loaders.py index 82df89278f..96b24c711c 100644 --- a/agentplatform/_genai/_evals_metric_loaders.py +++ b/agentplatform/_genai/_evals_metric_loaders.py @@ -215,8 +215,11 @@ def resolve(self, api_client: Any) -> "types.Metric": if self._resolved_metric: return self._resolved_metric + # The shared cache is keyed by name and version only, so a metric with + # overrides (such as judge_model_step_configs) must bypass it. + use_cache = not self.metric_kwargs cache_key = f"{self.name}@{self.version or 'default'}" - if cache_key in LazyLoadedPrebuiltMetric._cache: + if use_cache and cache_key in LazyLoadedPrebuiltMetric._cache: self._resolved_metric = LazyLoadedPrebuiltMetric._cache[cache_key] logger.debug("Metric '%s' found in cache.", cache_key) return self._resolved_metric @@ -225,7 +228,8 @@ def resolve(self, api_client: Any) -> "types.Metric": api_metric = self._resolve_api_predefined() if api_metric: self._resolved_metric = api_metric - LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric + if use_cache: + LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric return self._resolved_metric # Fallback to GCS loading for custom LLM-based Prebuilt Metrics @@ -234,8 +238,9 @@ def resolve(self, api_client: Any) -> "types.Metric": ) try: gcs_metric = self._fetch_and_parse(api_client) - final_cache_key = f"{self.name}@{self.version}" - LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric + if use_cache: + final_cache_key = f"{self.name}@{self.version}" + LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric self._resolved_metric = gcs_metric return self._resolved_metric except Exception as e: diff --git a/agentplatform/_genai/_transformers.py b/agentplatform/_genai/_transformers.py index f4ec56bdb4..44aafef45b 100644 --- a/agentplatform/_genai/_transformers.py +++ b/agentplatform/_genai/_transformers.py @@ -64,10 +64,14 @@ def t_metrics( elif ( metric_name and metric_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS ): - metric_payload_item["predefined_metric_spec"] = { + predefined_spec: dict[str, Any] = { "metric_spec_name": metric_name, "metric_spec_parameters": metric.metric_spec_parameters, } + step_configs = getv(metric, ["judge_model_step_configs"]) + if step_configs: + predefined_spec["step_autorater_configs"] = step_configs + metric_payload_item["predefined_metric_spec"] = predefined_spec # Custom Code Execution Metric elif ( hasattr(metric, "remote_custom_function") and metric.remote_custom_function diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index ab6de8d7ca..56a6cad82e 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -392,6 +392,13 @@ def _EvaluateInstancesRequestParameters_to_vertex( ], ) + if getv(from_object, ["allow_cross_region_model"]) is not None: + setv( + to_object, + ["allowCrossRegionModel"], + getv(from_object, ["allow_cross_region_model"]), + ) + if getv(from_object, ["config"]) is not None: setv(to_object, ["config"], getv(from_object, ["config"])) @@ -1935,6 +1942,7 @@ def _evaluate_instances( metrics: Optional[list[types.MetricOrDict]] = None, instance: Optional[types.EvaluationInstanceOrDict] = None, metric_sources: Optional[list[types.MetricSourceOrDict]] = None, + allow_cross_region_model: Optional[bool] = None, config: Optional[types.EvaluateInstancesConfigOrDict] = None, ) -> types.EvaluateInstancesResponse: """ @@ -1956,6 +1964,7 @@ def _evaluate_instances( metrics=metrics, instance=instance, metric_sources=metric_sources, + allow_cross_region_model=allow_cross_region_model, config=config, ) @@ -3171,6 +3180,8 @@ def evaluate( - evaluation_service_qps: The rate limit (queries per second) for calls to the evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota. + - allow_cross_region_model: Opt-in flag to authorize cross-region + routing for judge models. **kwargs: Extra arguments to pass to evaluation, such as `agent_info`. Returns: @@ -3214,6 +3225,7 @@ def evaluate( dest=config.dest, location=location, evaluation_service_qps=getattr(config, "evaluation_service_qps", None), + allow_cross_region_model=getattr(config, "allow_cross_region_model", None), **kwargs, ) @@ -4944,6 +4956,7 @@ async def _evaluate_instances( metrics: Optional[list[types.MetricOrDict]] = None, instance: Optional[types.EvaluationInstanceOrDict] = None, metric_sources: Optional[list[types.MetricSourceOrDict]] = None, + allow_cross_region_model: Optional[bool] = None, config: Optional[types.EvaluateInstancesConfigOrDict] = None, ) -> types.EvaluateInstancesResponse: """ @@ -4965,6 +4978,7 @@ async def _evaluate_instances( metrics=metrics, instance=instance, metric_sources=metric_sources, + allow_cross_region_model=allow_cross_region_model, config=config, ) diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 4c3da8b398..15c2fce06d 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -1385,6 +1385,12 @@ class Metric(_common.BaseModel): default=None, description="""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys.""", ) + judge_model_step_configs: Optional[dict[str, genai_types.AutoraterConfig]] = Field( + default=None, + description="""Per-step judge overrides for a predefined metric, keyed by step: + "intent_extraction", "rubric_generation" or "rubric_validation". Steps + without an entry keep their default judge.""", + ) # Allow extra fields to support metric-specific config fields. model_config = ConfigDict(extra="allow") @@ -1620,6 +1626,11 @@ class MetricDict(TypedDict, total=False): result_parsing_function: Optional[str] """Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys.""" + judge_model_step_configs: Optional[dict[str, genai_types.AutoraterConfig]] + """Per-step judge overrides for a predefined metric, keyed by step: + "intent_extraction", "rubric_generation" or "rubric_validation". Steps + without an entry keep their default judge.""" + MetricOrDict = Union[Metric, MetricDict] @@ -2070,7 +2081,7 @@ class EvaluationRunConfig(_common.BaseModel): ) autorater_config: Optional[genai_types.AutoraterConfig] = Field( default=None, - description="""The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""", + description="""The autorater config for the evaluation run. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""", ) prompt_template: Optional[EvaluationRunPromptTemplate] = Field( default=None, description="""The prompt template used for inference.""" @@ -2100,7 +2111,7 @@ class EvaluationRunConfigDict(TypedDict, total=False): """The output config for the evaluation run.""" autorater_config: Optional[genai_types.AutoraterConfig] - """The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""" + """The autorater config for the evaluation run. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""" prompt_template: Optional[EvaluationRunPromptTemplateDict] """The prompt template used for inference.""" @@ -4702,7 +4713,7 @@ class _EvaluateInstancesRequestParameters(_common.BaseModel): ) autorater_config: Optional[genai_types.AutoraterConfig] = Field( default=None, - description="""Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""", + description="""Autorater config used for evaluation. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""", ) metrics: Optional[list[Metric]] = Field( default=None, @@ -4716,6 +4727,12 @@ class _EvaluateInstancesRequestParameters(_common.BaseModel): metric_sources: Optional[list[MetricSource]] = Field( default=None, description="""The metrics used for evaluation.""" ) + allow_cross_region_model: Optional[bool] = Field( + default=None, + description="""Allows judge models to be served from other regions. Required + when a judge model's fully-qualified resource name uses a different + region than the request.""", + ) config: Optional[EvaluateInstancesConfig] = Field(default=None, description="""""") @@ -4753,7 +4770,7 @@ class _EvaluateInstancesRequestParametersDict(TypedDict, total=False): """""" autorater_config: Optional[genai_types.AutoraterConfig] - """Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""" + """Autorater config used for evaluation. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""" metrics: Optional[list[MetricDict]] """The metrics used for evaluation. @@ -4766,6 +4783,11 @@ class _EvaluateInstancesRequestParametersDict(TypedDict, total=False): metric_sources: Optional[list[MetricSourceDict]] """The metrics used for evaluation.""" + allow_cross_region_model: Optional[bool] + """Allows judge models to be served from other regions. Required + when a judge model's fully-qualified resource name uses a different + region than the request.""" + config: Optional[EvaluateInstancesConfigDict] """""" @@ -30100,6 +30122,13 @@ class EvaluateMethodConfig(_common.BaseModel): evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota.""", ) + allow_cross_region_model: Optional[bool] = Field( + default=None, + description="""Allows judge models to be served from other regions. When set, + the service may route judge requests to another region if the model is + unavailable in the request's region. Required when a judge model's + fully-qualified resource name uses a different region.""", + ) class EvaluateMethodConfigDict(TypedDict, total=False): @@ -30121,6 +30150,12 @@ class EvaluateMethodConfigDict(TypedDict, total=False): evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota.""" + allow_cross_region_model: Optional[bool] + """Allows judge models to be served from other regions. When set, + the service may route judge requests to another region if the model is + unavailable in the request's region. Required when a judge model's + fully-qualified resource name uses a different region.""" + EvaluateMethodConfigOrDict = Union[EvaluateMethodConfig, EvaluateMethodConfigDict] diff --git a/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py b/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py index 0dbfd98d6d..f6266e6a02 100644 --- a/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py +++ b/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py @@ -16,8 +16,43 @@ from tests.unit.agentplatform.genai.replays import pytest_helper from agentplatform import types +from google.genai import types as genai_types import pandas as pd +# Per-step judge tests are recorded against staging, whose API config already +# has the final step_autorater_configs shape. +_STAGING_BASE_URL = "https://us-central1-staging-aiplatform.sandbox.googleapis.com/" + + +def _haiku_dataset() -> types.EvaluationDataset: + return types.EvaluationDataset( + eval_dataset_df=pd.DataFrame( + { + "prompt": ["Write a haiku about the ocean. Use exactly three lines."], + "response": [ + "Waves fold into foam\nsalt wind carries gull voices\nthe" + " tide keeps its time" + ], + } + ), + candidate_name="gemini-2.5-flash", + ) + + +def _assert_metric_succeeded( + evaluation_result: types.EvaluationResult, metric_name: str +) -> None: + assert isinstance(evaluation_result, types.EvaluationResult) + summary = evaluation_result.summary_metrics[0] + assert summary.metric_name == metric_name + assert summary.num_cases_error == 0 + candidate_result = evaluation_result.eval_case_results[ + 0 + ].response_candidate_results[0] + metric_result = candidate_result.metric_results[metric_name] + assert metric_result.error_message is None + assert metric_result.rubric_verdicts + def test_evaluation_result(client): """Tests that evaluate() produces a correctly structured EvaluationResult.""" @@ -139,6 +174,54 @@ def test_predefined_metric_with_judge_model_ignores_autorater_config(client): assert evaluation_result.summary_metrics[0].metric_name == "safety_v1" +def test_predefined_metric_with_judge_model_step_configs(client): + """Tests that evaluate() sends per-step judges for a predefined metric.""" + client._api_client._http_options.base_url = _STAGING_BASE_URL + model_prefix = ( + f"projects/{client._api_client.project}/locations/" + f"{client._api_client.location}/publishers/google/models" + ) + metric = types.RubricMetric.INSTRUCTION_FOLLOWING( + judge_model_step_configs={ + "rubric_generation": genai_types.AutoraterConfig( + autorater_model=f"{model_prefix}/gemini-2.5-pro" + ), + "rubric_validation": genai_types.AutoraterConfig( + autorater_model=f"{model_prefix}/gemini-2.5-flash" + ), + } + ) + + evaluation_result = client.evals.evaluate( + dataset=_haiku_dataset(), metrics=[metric] + ) + + _assert_metric_succeeded(evaluation_result, "instruction_following_v1") + + +def test_partner_step_judge_with_allow_cross_region_model(client): + """Tests a partner step judge served from another region.""" + client._api_client._http_options.base_url = _STAGING_BASE_URL + metric = types.RubricMetric.INSTRUCTION_FOLLOWING( + judge_model_step_configs={ + "rubric_validation": genai_types.AutoraterConfig( + autorater_model=( + f"projects/{client._api_client.project}/locations/us-east5" + "/publishers/anthropic/models/claude-sonnet-4-5" + ) + ), + } + ) + + evaluation_result = client.evals.evaluate( + dataset=_haiku_dataset(), + metrics=[metric], + config={"allow_cross_region_model": True}, + ) + + _assert_metric_succeeded(evaluation_result, "instruction_following_v1") + + def test_multi_turn_predefined_metric(client): """Tests that evaluate works with multi-turn predefined metrics.""" prompts_data = { diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index f82da80765..232d5ca1c7 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -48,6 +48,7 @@ from google.genai import errors as genai_errors from google.genai import types as genai_types import pandas as pd +import pydantic import pytest _TEST_PROJECT = "test-project" @@ -10721,6 +10722,253 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self) ) +class TestJudgeModelStepConfigs: + """Tests for per-step judges and cross-region routing in evaluate().""" + + _STEP_CONFIGS = { + "rubric_validation": genai_types.AutoraterConfig( + autorater_model="claude-sonnet-4-5" + ) + } + _EVAL_CASE = agentplatform_genai_types.EvalCase( + prompt=genai_types.Content(parts=[genai_types.Part(text="Hello")]), + responses=[ + agentplatform_genai_types.ResponseCandidate( + response=genai_types.Content(parts=[genai_types.Part(text="Hi")]) + ) + ], + ) + _HANDLER_METRICS = [ + agentplatform_genai_types.Metric(name="final_response_quality_v1"), + agentplatform_genai_types.LLMMetric( + name="tone", prompt_template="Rate the tone of {response}." + ), + agentplatform_genai_types.Metric( + name="registered_metric", + metric_resource_name=( + "projects/123/locations/us-central1/evaluationMetrics/456" + ), + ), + ] + _HANDLER_IDS = ["predefined", "llm", "registered"] + + def setup_method(self): + _evals_metric_loaders.LazyLoadedPrebuiltMetric._cache.clear() + + def teardown_method(self): + _evals_metric_loaders.LazyLoadedPrebuiltMetric._cache.clear() + + @staticmethod + def _mock_api_client(): + api_client = mock.MagicMock() + api_client.vertexai = True + api_client.request.return_value.body = json.dumps( + {"metricResults": [{"score": 1.0}]} + ) + return api_client + + def test_t_metrics_sends_step_autorater_configs(self): + metric = agentplatform_genai_types.Metric( + name="final_response_quality_v1", + judge_model_step_configs=self._STEP_CONFIGS, + ) + + payload = _transformers.t_metrics([metric])[0] + + assert ( + payload["predefined_metric_spec"]["step_autorater_configs"] + == self._STEP_CONFIGS + ) + + def test_t_metrics_omits_step_autorater_configs_by_default(self): + metric = agentplatform_genai_types.Metric(name="final_response_quality_v1") + + payload = _transformers.t_metrics([metric])[0] + + assert "step_autorater_configs" not in payload["predefined_metric_spec"] + + def test_predefined_handler_sends_step_configs_and_cross_region_flag(self): + api_client = self._mock_api_client() + metric = agentplatform_genai_types.Metric( + name="final_response_quality_v1", + judge_model_step_configs=self._STEP_CONFIGS, + ) + handler = _evals_metric_handlers.get_handler_for_metric( + evals.Evals(api_client_=api_client), + metric, + allow_cross_region_model=True, + ) + + result = handler.get_metric_result(self._EVAL_CASE, response_index=0) + + assert result.score == 1.0 + request_body = api_client.request.call_args[0][2] + assert request_body["allowCrossRegionModel"] is True + assert request_body["metrics"][0]["predefined_metric_spec"][ + "step_autorater_configs" + ] == {"rubric_validation": {"autorater_model": "claude-sonnet-4-5"}} + + @pytest.mark.parametrize("metric", _HANDLER_METRICS, ids=_HANDLER_IDS) + def test_handlers_send_cross_region_flag(self, metric): + api_client = self._mock_api_client() + handler = _evals_metric_handlers.get_handler_for_metric( + evals.Evals(api_client_=api_client), + metric, + allow_cross_region_model=True, + ) + + result = handler.get_metric_result(self._EVAL_CASE, response_index=0) + + assert result.score == 1.0 + assert api_client.request.call_args[0][2]["allowCrossRegionModel"] is True + + @pytest.mark.parametrize("metric", _HANDLER_METRICS, ids=_HANDLER_IDS) + def test_handlers_omit_cross_region_flag_by_default(self, metric): + api_client = self._mock_api_client() + handler = _evals_metric_handlers.get_handler_for_metric( + evals.Evals(api_client_=api_client), metric + ) + + result = handler.get_metric_result(self._EVAL_CASE, response_index=0) + + assert result.score == 1.0 + assert "allowCrossRegionModel" not in api_client.request.call_args[0][2] + + @pytest.mark.usefixtures("mock_eval_dependencies") + def test_execute_evaluation_passes_cross_region_flag(self, mock_api_client_fixture): + with mock.patch.object( + _evals_metric_handlers, + "compute_metrics_and_aggregate", + side_effect=RuntimeError("stop"), + ) as mock_compute: + with pytest.raises(RuntimeError): + _evals_common._execute_evaluation( + api_client=mock_api_client_fixture, + dataset=agentplatform_genai_types.EvaluationDataset( + eval_dataset_df=pd.DataFrame( + [{"prompt": "p1", "response": "r1"}] + ) + ), + metrics=[agentplatform_genai_types.Metric(name="exact_match")], + allow_cross_region_model=True, + ) + + assert mock_compute.call_args.kwargs["allow_cross_region_model"] is True + + def test_compute_metrics_passes_cross_region_flag_to_handlers(self): + run_config = _evals_metric_handlers.EvaluationRunConfig( + evals_module=evals.Evals(api_client_=mock.MagicMock()), + dataset=agentplatform_genai_types.EvaluationDataset(eval_cases=[]), + metrics=[ + agentplatform_genai_types.Metric(name="final_response_quality_v1") + ], + num_response_candidates=1, + ) + + with mock.patch.object( + _evals_metric_handlers, + "get_handler_for_metric", + side_effect=RuntimeError("stop"), + ) as mock_get_handler: + with pytest.raises(RuntimeError): + _evals_metric_handlers.compute_metrics_and_aggregate( + run_config, allow_cross_region_model=True + ) + + assert mock_get_handler.call_args.kwargs["allow_cross_region_model"] is True + + @mock.patch.object(_evals_common, "_execute_evaluation") + def test_evaluate_passes_allow_cross_region_model(self, mock_execute_evaluation): + evals.Evals(api_client_=mock.MagicMock()).evaluate( + dataset=agentplatform_genai_types.EvaluationDataset( + eval_dataset_df=pd.DataFrame([{"prompt": "p1", "response": "r1"}]) + ), + metrics=[ + agentplatform_genai_types.Metric(name="final_response_quality_v1") + ], + config={"allow_cross_region_model": True}, + ) + + _, kwargs = mock_execute_evaluation.call_args + assert kwargs["allow_cross_region_model"] is True + + @mock.patch.object(_evals_metric_handlers.logger, "warning") + def test_judge_model_warning_only_for_multi_turn_metrics(self, mock_warning): + module = evals.Evals(api_client_=mock.MagicMock()) + + _evals_metric_handlers.PredefinedMetricHandler( + module=module, + metric=agentplatform_genai_types.Metric( + name="final_response_quality_v1", judge_model="gemini-2.5-pro" + ), + ) + mock_warning.assert_not_called() + + _evals_metric_handlers.PredefinedMetricHandler( + module=module, + metric=agentplatform_genai_types.Metric( + name="multi_turn_task_success_v1", judge_model="gemini-2.5-pro" + ), + ) + mock_warning.assert_called_once() + + def test_resolve_evaluation_run_metrics_sends_metric_spec_parameters(self): + metric = _evals_metric_loaders.RubricMetric.FINAL_RESPONSE_QUALITY( + metric_spec_parameters={"guidelines": "Be concise."} + ) + + resolved = _evals_common._resolve_evaluation_run_metrics( + [metric], api_client=mock.MagicMock() + ) + + predefined_spec = resolved[0].metric_config.predefined_metric_spec + assert predefined_spec.metric_spec_name == "final_response_quality_v1" + assert predefined_spec.metric_spec_parameters == {"guidelines": "Be concise."} + + @pytest.mark.parametrize( + "metric", + [ + _evals_metric_loaders.RubricMetric.FINAL_RESPONSE_QUALITY( + judge_model_step_configs=_STEP_CONFIGS + ), + agentplatform_genai_types.Metric( + name="final_response_quality_v1", + judge_model_step_configs=_STEP_CONFIGS, + ), + ], + ids=["rubric_metric", "metric"], + ) + def test_resolve_evaluation_run_metrics_step_configs(self, metric): + # google-genai gets the field once it is public in the discovery doc. + if "step_autorater_configs" in genai_types.PredefinedMetricSpec.model_fields: + resolved = _evals_common._resolve_evaluation_run_metrics( + [metric], api_client=mock.MagicMock() + ) + predefined_spec = resolved[0].metric_config.predefined_metric_spec + assert predefined_spec.step_autorater_configs == self._STEP_CONFIGS + else: + with pytest.raises( + pydantic.ValidationError, match="step_autorater_configs" + ): + _evals_common._resolve_evaluation_run_metrics( + [metric], api_client=mock.MagicMock() + ) + + def test_rubric_metric_overrides_bypass_shared_cache(self): + api_client = mock.MagicMock() + rubric_metric = _evals_metric_loaders.RubricMetric + + default_metric = rubric_metric.FINAL_RESPONSE_QUALITY.resolve(api_client) + overridden_metric = rubric_metric.FINAL_RESPONSE_QUALITY( + judge_model_step_configs=self._STEP_CONFIGS + ).resolve(api_client) + default_again = rubric_metric.FINAL_RESPONSE_QUALITY.resolve(api_client) + + assert default_metric.judge_model_step_configs is None + assert overridden_metric.judge_model_step_configs == self._STEP_CONFIGS + assert default_again.judge_model_step_configs is None + + _TEST_INTERACTION = ( "projects/test-project/locations/us-central1/interactions/test-interaction" ) From 4c045e16c2459d0708bca77c2a82a3c1daff0409 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Fri, 2 Oct 2026 11:32:24 -0700 Subject: [PATCH 3/7] feat: GenAI Client(evals) - add EvaluationDataset.load_from_adk_eval_set PiperOrigin-RevId: 992441610 --- .../_genai/_evals_data_converters.py | 307 ++++++++++- agentplatform/_genai/types/common.py | 54 ++ tests/unit/agentplatform/genai/test_evals.py | 488 ++++++++++++++++++ vertexai/_genai/_evals_data_converters.py | 307 ++++++++++- vertexai/_genai/types/common.py | 54 ++ 5 files changed, 1208 insertions(+), 2 deletions(-) diff --git a/agentplatform/_genai/_evals_data_converters.py b/agentplatform/_genai/_evals_data_converters.py index cae94afed3..2934ba92ea 100644 --- a/agentplatform/_genai/_evals_data_converters.py +++ b/agentplatform/_genai/_evals_data_converters.py @@ -15,15 +15,18 @@ """Dataset converters for evals.""" import copy +import datetime import json import logging -from typing import Any, Optional, Union +from typing import Any, Optional, TypeVar, Union from google.genai import _common from google.genai import types as genai_types +from pydantic import alias_generators from pydantic import ValidationError from typing_extensions import override +from . import _evals_constant from . import _evals_utils from . import _observability_data_converter from . import types @@ -605,6 +608,308 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: return types.EvaluationDataset(eval_cases=eval_cases) +_ADK_EVAL_CASE_FIELDS = frozenset( + { + "eval_id", + "conversation", + "conversation_scenario", + "session_input", + "creation_timestamp", + "rubrics", + "final_session_state", + } +) +_ADK_EVAL_CASE_KEYS = _ADK_EVAL_CASE_FIELDS | { + alias_generators.to_camel(field) for field in _ADK_EVAL_CASE_FIELDS +} +_ADK_SESSION_INPUT_FIELDS = ("app_name", "user_id", "state") +_ADK_DEFAULT_RUBRIC_GROUP = "adk_rubrics" +_REFERENCE_TRAJECTORY = "reference_trajectory" + +_GenaiModel = TypeVar("_GenaiModel", bound=_common.BaseModel) + + +def _get_adk_field(data: dict[str, Any], name: str) -> Any: + """Returns an ADK eval set field, which ADK accepts in snake or camel case.""" + if not isinstance(data, dict): + raise TypeError(f"Expected a JSON object for '{name}', got {type(data)}.") + if name in data: + return data[name] + return data.get(alias_generators.to_camel(name)) + + +def _validate_adk_genai(model: type[_GenaiModel], data: Any) -> _GenaiModel: + """Validates ADK eval set JSON as a genai type. + + Validating in JSON mode decodes base64 bytes fields, such as inline data and + thought signatures, the same way ADK does when it reads an eval set file. + + Args: + model: The genai type to validate as. + data: The parsed JSON value from the eval set. + + Returns: + The validated genai object. + """ + return model.model_validate_json(json.dumps(data)) + + +def _adk_content(data: dict[str, Any], default_role: str) -> genai_types.Content: + """Validates an ADK Content, filling in the role when the file omits it.""" + content = _validate_adk_genai(genai_types.Content, data) + if not content.role: + content.role = default_role + return content + + +def _adk_user_content(invocation: dict[str, Any]) -> genai_types.Content: + """Returns the user content of an ADK invocation.""" + user_content = _get_adk_field(invocation, "user_content") + if user_content is None: + raise ValueError("Every invocation must have user_content.") + return _adk_content(user_content, "user") + + +def _adk_intermediate_events( + intermediate_data: Optional[dict[str, Any]], +) -> list[tuple[Optional[str], genai_types.Content]]: + """Returns (author, content) pairs from ADK IntermediateData or InvocationEvents.""" + if not intermediate_data: + return [] + invocation_events = _get_adk_field(intermediate_data, "invocation_events") + if invocation_events is not None: + return [ + ( + _get_adk_field(event, "author"), + _validate_adk_genai( + genai_types.Content, _get_adk_field(event, "content") + ), + ) + for event in invocation_events + if _get_adk_field(event, "content") + ] + + events: list[tuple[Optional[str], genai_types.Content]] = [] + tool_uses = _get_adk_field(intermediate_data, "tool_uses") + if tool_uses: + events.append( + ( + None, + genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=_validate_adk_genai( + genai_types.FunctionCall, tool_use + ) + ) + for tool_use in tool_uses + ], + ), + ) + ) + tool_responses = _get_adk_field(intermediate_data, "tool_responses") + if tool_responses: + events.append( + ( + None, + genai_types.Content( + role="user", + parts=[ + genai_types.Part( + function_response=_validate_adk_genai( + genai_types.FunctionResponse, tool_response + ) + ) + for tool_response in tool_responses + ], + ), + ) + ) + for author, parts in ( + _get_adk_field(intermediate_data, "intermediate_responses") or [] + ): + events.append( + ( + author, + genai_types.Content( + role="model", + parts=[ + _validate_adk_genai(genai_types.Part, part) for part in parts + ], + ), + ) + ) + return events + + +class AdkEvalSetConverter(_evals_utils.EvalDataConverter): + """Converter for ADK eval sets (the `.evalset.json` format). + + Reads the eval set JSON directly, so google-adk does not need to be + installed. + """ + + def _invocation_to_messages( + self, invocation: dict[str, Any], turn_index: int + ) -> list[types.evals.Message]: + """Converts an earlier ADK invocation into conversation history messages.""" + turn_id = _get_adk_field(invocation, "invocation_id") or str(turn_index) + timestamp = _get_adk_field(invocation, "creation_timestamp") + messages = [ + types.evals.Message( + turn_id=turn_id, + author="user", + content=_adk_user_content(invocation), + creation_timestamp=( + datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + if timestamp + else None + ), + ) + ] + messages.extend( + types.evals.Message(turn_id=turn_id, author=author, content=content) + for author, content in _adk_intermediate_events( + _get_adk_field(invocation, "intermediate_data") + ) + ) + final_response = _get_adk_field(invocation, "final_response") + if final_response: + messages.append( + types.evals.Message( + turn_id=turn_id, content=_adk_content(final_response, "model") + ) + ) + return messages + + def _convert_rubrics( + self, rubrics: list[dict[str, Any]] + ) -> dict[str, types.RubricGroup]: + """Groups ADK rubrics by their type, which ADK uses to select rubrics.""" + groups: dict[str, list[types.evals.Rubric]] = {} + for rubric in rubrics: + rubric_type = _get_adk_field(rubric, "type") + rubric_content = _get_adk_field(rubric, "rubric_content") or {} + groups.setdefault(rubric_type or _ADK_DEFAULT_RUBRIC_GROUP, []).append( + types.evals.Rubric( + rubric_id=_get_adk_field(rubric, "rubric_id"), + type=rubric_type, + content=types.evals.RubricContent( + property=types.evals.RubricContentProperty( + description=_get_adk_field(rubric_content, "text_property") + ) + ), + ) + ) + return { + name: types.RubricGroup(rubrics=group) for name, group in groups.items() + } + + def _convert_eval_case(self, case: dict[str, Any]) -> types.EvalCase: + """Converts one ADK EvalCase into an EvalCase.""" + if not isinstance(case, dict): + raise TypeError(f"Expected an ADK EvalCase object, got {type(case)}.") + conversation = _get_adk_field(case, "conversation") + scenario = _get_adk_field(case, "conversation_scenario") + if (conversation is None) == (scenario is None): + raise ValueError( + "Exactly one of conversation and conversation_scenario must be set." + ) + + fields: dict[str, Any] = {"eval_case_id": _get_adk_field(case, "eval_id")} + rubrics = list(_get_adk_field(case, "rubrics") or []) + if scenario is not None: + fields["user_scenario"] = types.evals.UserScenario( + starting_prompt=_get_adk_field(scenario, "starting_prompt"), + conversation_plan=_get_adk_field(scenario, "conversation_plan"), + ) + else: + if not conversation: + raise ValueError("conversation must have at least one invocation.") + *earlier_invocations, last_invocation = conversation + fields["prompt"] = _adk_user_content(last_invocation) + history = [] + for turn_index, invocation in enumerate(earlier_invocations): + history.extend(self._invocation_to_messages(invocation, turn_index)) + if history: + fields["conversation_history"] = history + final_response = _get_adk_field(last_invocation, "final_response") + if final_response: + fields["reference"] = types.ResponseCandidate( + response=_adk_content(final_response, "model") + ) + reference_trajectory: list[genai_types.Content] = [] + for _, content in _adk_intermediate_events( + _get_adk_field(last_invocation, "intermediate_data") + ): + reference_trajectory.extend( + genai_types.Content( + role="model", + parts=[genai_types.Part(function_call=part.function_call)], + ) + for part in content.parts or [] + if part.function_call + ) + if reference_trajectory: + fields[_REFERENCE_TRAJECTORY] = reference_trajectory + rubrics.extend(_get_adk_field(last_invocation, "rubrics") or []) + + if rubrics: + fields["rubric_groups"] = self._convert_rubrics(rubrics) + session_input = _get_adk_field(case, "session_input") + if session_input: + fields[_evals_constant.SESSION_INPUT] = { + name: value + for name in _ADK_SESSION_INPUT_FIELDS + if (value := _get_adk_field(session_input, name)) is not None + } + + reserved_keys = set(fields) + for field_name in types.EvalCase.model_fields: + reserved_keys.update((field_name, alias_generators.to_camel(field_name))) + for key, value in case.items(): + if key in _ADK_EVAL_CASE_KEYS: + continue + if key in reserved_keys: + logger.warning( + "Skipping ADK eval case field '%s' because it conflicts with" + " an EvalCase field.", + key, + ) + continue + fields[key] = value + return types.EvalCase(**fields) + + @override + def convert(self, raw_data: dict[str, Any]) -> types.EvaluationDataset: + """Converts a parsed ADK EvalSet into an EvaluationDataset.""" + if not isinstance(raw_data, dict): + raise ValueError( + "Expected an ADK EvalSet JSON object, got" + f" {type(raw_data)}. For the legacy list format, load the" + " file with google.adk.evaluation.local_eval_sets_manager" + ".load_eval_set_from_file() and pass" + " eval_set.model_dump(mode='json') instead." + ) + adk_eval_cases = _get_adk_field(raw_data, "eval_cases") + if not isinstance(adk_eval_cases, list): + raise ValueError("Expected an ADK EvalSet with an 'eval_cases' list.") + + eval_cases = [] + for i, case in enumerate(adk_eval_cases): + try: + eval_cases.append(self._convert_eval_case(case)) + except (TypeError, ValueError) as e: + case_id = ( + _get_adk_field(case, "eval_id") if isinstance(case, dict) else None + ) + raise ValueError( + f"Failed to convert ADK eval case '{case_id or i}': {e}" + ) from e + return types.EvaluationDataset(eval_cases=eval_cases) + + def auto_detect_dataset_schema( raw_dataset: list[dict[str, Any]], ) -> Union[EvalDatasetSchema, str]: diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 15c2fce06d..d59acc7cfa 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -3182,6 +3182,60 @@ def load_from_observability_eval_cases( return EvaluationDataset(eval_dataset_df=eval_dataset_df) + @classmethod + def load_from_adk_eval_set( + cls, eval_set: Union[str, "os.PathLike[str]", dict[str, Any]] + ) -> "EvaluationDataset": + """Loads an ADK eval set (`.evalset.json`) into an EvaluationDataset. + + Each ADK eval case becomes an EvalCase with `eval_case_id` set to its + `eval_id`. For a static conversation, the last invocation's user content + becomes `prompt` and its final response becomes `reference`. Earlier + invocations, including their tool calls, become `conversation_history`. + A conversation scenario becomes `user_scenario`. Rubrics on the case and + on the last invocation go into `rubric_groups`, keyed by rubric type + ("adk_rubrics" when unset). + + Some ADK data is kept as EvalCase extra fields: `reference_trajectory` + holds the tool calls of the last invocation, `session_inputs` holds the + session input, and custom ADK eval case fields keep their names. Other ADK + fields, such as `final_session_state` and the scenario's `user_persona`, + are not carried over. Steps that convert eval cases to a DataFrame, such + as `run_inference`, drop the extra fields, `rubric_groups` and + `eval_case_id`. google-adk does not need to be installed. + + Example: + import agentplatform + from agentplatform import types + + client = agentplatform.Client( + project="my-project", location="us-central1" + ) + dataset = types.EvaluationDataset.load_from_adk_eval_set( + "my_agent/my_eval_set.evalset.json" + ) + dataset_with_responses = client.evals.run_inference( + src=dataset, agent=my_agent + ) + result = client.evals.evaluate( + dataset=dataset_with_responses, + metrics=[types.RubricMetric.FINAL_RESPONSE_MATCH], + ) + + Args: + eval_set: The path to an ADK `.evalset.json` file, or the eval set as + a parsed JSON dict. + + Returns: + An EvaluationDataset with one EvalCase per ADK eval case. + """ + from .. import _evals_data_converters # pylint: disable=g-import-not-at-top + + if not isinstance(eval_set, dict): + with open(eval_set, "r", encoding="utf-8") as f: + eval_set = json.load(f) + return _evals_data_converters.AdkEvalSetConverter().convert(eval_set) + def show(self) -> None: """Shows the evaluation dataset.""" from .. import _evals_visualization diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 232d5ca1c7..28807dfc67 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -9296,9 +9296,407 @@ def test_predefined_metric_retry_fail_on_resource_exhausted( assert summary_metric.num_cases_error == 1 +_ADK_EVAL_SET = { + "eval_set_id": "home_automation", + "name": "Home automation", + "eval_cases": [ + { + "eval_id": "turn_off_device", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": { + "parts": [{"text": "Turn off device_2."}], + "role": "user", + }, + "final_response": {"parts": [{"text": "device_2 is now off."}]}, + "intermediate_data": { + "tool_uses": [ + { + "name": "set_device_info", + "args": {"device_id": "device_2", "status": "OFF"}, + } + ], + "intermediate_responses": [], + }, + } + ], + "session_input": { + "app_name": "home_automation_agent", + "user_id": "user", + "state": {"home": {"rooms": 3}}, + }, + "rubrics": [ + { + "rubric_id": "confirms_action", + "rubric_content": { + "text_property": "The response confirms the device is off." + }, + "type": "FINAL_RESPONSE_QUALITY", + } + ], + "tags": ["smoke"], + }, + { + "eval_id": "list_then_turn_off", + "conversation": [ + { + "invocation_id": "inv-2a", + "user_content": { + "parts": [{"text": "Which devices are on?"}], + "role": "user", + }, + "final_response": { + "parts": [{"text": "device_1 is on."}], + "role": "model", + }, + "intermediate_data": { + "invocation_events": [ + { + "author": "home_agent", + "content": { + "parts": [ + { + "function_call": { + "name": "list_devices", + "args": {"status": "ON"}, + } + } + ], + "role": "model", + }, + }, + { + "author": "home_agent", + "content": { + "parts": [ + { + "function_response": { + "name": "list_devices", + "response": {"devices": ["device_1"]}, + } + } + ], + "role": "user", + }, + }, + ] + }, + "creation_timestamp": 1700000000.0, + }, + { + "invocation_id": "inv-2b", + "user_content": { + "parts": [{"text": "Turn it off."}], + "role": "user", + }, + "final_response": { + "parts": [{"text": "device_1 is now off."}], + "role": "model", + }, + "intermediate_data": { + "invocation_events": [ + { + "author": "home_agent", + "content": { + "parts": [ + { + "function_call": { + "name": "set_device_info", + "args": { + "device_id": "device_1", + "status": "OFF", + }, + } + } + ], + "role": "model", + }, + } + ] + }, + "rubrics": [ + { + "rubric_id": "polite", + "rubric_content": { + "text_property": "The response is polite." + }, + } + ], + }, + ], + }, + { + "evalId": "comfortable_bedroom", + "conversationScenario": { + "startingPrompt": "I want my bedroom to be comfortable.", + "conversationPlan": "Ask for 21 degrees once the agent asks.", + "userPersona": "NOVICE", + }, + "sessionInput": {"appName": "home_automation_agent", "userId": "user"}, + }, + ], +} + + class TestEvaluationDataset: """Contains set of tests for the EvaluationDataset class methods.""" + def test_load_from_adk_eval_set_file(self, tmp_path): + path = tmp_path / "home_automation.evalset.json" + path.write_text(json.dumps(_ADK_EVAL_SET)) + + dataset = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + path + ) + + assert [case.eval_case_id for case in dataset.eval_cases] == [ + "turn_off_device", + "list_then_turn_off", + "comfortable_bedroom", + ] + + def test_load_from_adk_eval_set_single_turn_with_tool_uses(self): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ).eval_cases[0] + + assert case.prompt == genai_types.Content( + parts=[genai_types.Part(text="Turn off device_2.")], role="user" + ) + assert case.reference.response == genai_types.Content( + parts=[genai_types.Part(text="device_2 is now off.")], role="model" + ) + assert case.conversation_history is None + assert case.reference_trajectory == [ + genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=genai_types.FunctionCall( + name="set_device_info", + args={"device_id": "device_2", "status": "OFF"}, + ) + ) + ], + ) + ] + assert case.session_inputs == { + "app_name": "home_automation_agent", + "user_id": "user", + "state": {"home": {"rooms": 3}}, + } + assert case.rubric_groups == { + "FINAL_RESPONSE_QUALITY": agentplatform_genai_types.RubricGroup( + rubrics=[ + agentplatform_genai_types.evals.Rubric( + rubric_id="confirms_action", + type="FINAL_RESPONSE_QUALITY", + content=agentplatform_genai_types.evals.RubricContent( + property=agentplatform_genai_types.evals.RubricContentProperty( + description="The response confirms the device is off." + ) + ), + ) + ] + ) + } + assert case.tags == ["smoke"] + + def test_load_from_adk_eval_set_multi_turn(self): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ).eval_cases[1] + + assert case.prompt.parts[0].text == "Turn it off." + assert [ + (message.turn_id, message.author, message.content.role) + for message in case.conversation_history + ] == [ + ("inv-2a", "user", "user"), + ("inv-2a", "home_agent", "model"), + ("inv-2a", "home_agent", "user"), + ("inv-2a", None, "model"), + ] + history = case.conversation_history + assert history[0].content.parts[0].text == "Which devices are on?" + assert history[0].creation_timestamp.timestamp() == 1700000000.0 + assert history[1].content.parts[0].function_call.name == "list_devices" + assert history[2].content.parts[0].function_response.response == { + "devices": ["device_1"] + } + assert history[3].content.parts[0].text == "device_1 is on." + assert case.reference.response.parts[0].text == "device_1 is now off." + assert [ + content.parts[0].function_call.name for content in case.reference_trajectory + ] == ["set_device_info"] + assert case.rubric_groups["adk_rubrics"].rubrics[0].rubric_id == "polite" + + def test_load_from_adk_eval_set_conversation_scenario_camel_case(self): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ).eval_cases[2] + + assert case.eval_case_id == "comfortable_bedroom" + assert case.prompt is None + assert case.user_scenario == agentplatform_genai_types.evals.UserScenario( + starting_prompt="I want my bedroom to be comfortable.", + conversation_plan="Ask for 21 degrees once the agent asks.", + ) + assert case.session_inputs == { + "app_name": "home_automation_agent", + "user_id": "user", + } + + def test_load_from_adk_eval_set_decodes_base64_inline_data(self): + eval_set = { + "eval_set_id": "images", + "eval_cases": [ + { + "eval_id": "image", + "conversation": [ + { + "user_content": { + "parts": [ + { + "inline_data": { + "mime_type": "image/png", + "data": base64.b64encode( + b"\x89PNG" + ).decode(), + } + } + ] + } + } + ], + } + ], + } + + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + eval_set + ).eval_cases[0] + + assert case.prompt.role == "user" + assert case.prompt.parts[0].inline_data.data == b"\x89PNG" + assert case.reference is None + + def test_load_from_adk_eval_set_camel_case_intermediate_data(self, caplog): + eval_set = { + "eval_set_id": "camel", + "eval_cases": [ + { + "evalId": "camel_case", + "conversation": [ + { + "invocationId": "first", + "userContent": {"parts": [{"text": "Look it up."}]}, + "finalResponse": {"parts": [{"text": "It is 42."}]}, + "intermediateData": { + "toolUses": [{"name": "lookup", "args": {"q": "x"}}], + "toolResponses": [ + {"name": "lookup", "response": {"answer": 42}} + ], + "intermediateResponses": [ + ["helper_agent", [{"text": "Looking."}]] + ], + }, + }, + {"userContent": {"parts": [{"text": "Thanks."}]}}, + ], + "evalCaseId": "override", + "userScenario": {"startingPrompt": "Hi"}, + "category": "lookup", + } + ], + } + + with caplog.at_level("WARNING", logger=_evals_data_converters.logger.name): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + eval_set + ).eval_cases[0] + + assert case.eval_case_id == "camel_case" + assert case.user_scenario is None + assert case.category == "lookup" + assert "'evalCaseId'" in caplog.text + assert "'userScenario'" in caplog.text + assert case.prompt.parts[0].text == "Thanks." + assert [ + (message.turn_id, message.author, message.content.role) + for message in case.conversation_history + ] == [ + ("first", "user", "user"), + ("first", None, "model"), + ("first", None, "user"), + ("first", "helper_agent", "model"), + ("first", None, "model"), + ] + history = case.conversation_history + assert history[1].content.parts[0].function_call.args == {"q": "x"} + assert history[2].content.parts[0].function_response.response == {"answer": 42} + assert history[3].content.parts[0].text == "Looking." + assert history[4].content.parts[0].text == "It is 42." + assert case.reference is None + assert "reference_trajectory" not in case.model_extra + + @pytest.mark.parametrize( + "eval_set, error", + [ + ({"eval_set_id": "no_cases"}, "'eval_cases' list"), + ( + {"eval_cases": [{"eval_id": "neither"}]}, + "'neither'.*Exactly one of conversation and conversation_scenario", + ), + ( + {"eval_cases": [{"eval_id": "bad_turn", "conversation": ["Hi"]}]}, + "'bad_turn'.*Expected a JSON object", + ), + ( + { + "eval_cases": [ + { + "eval_id": "both", + "conversation": [], + "conversation_scenario": {"starting_prompt": "Hi"}, + } + ] + }, + "'both'.*Exactly one of conversation and conversation_scenario", + ), + ( + {"eval_cases": [{"eval_id": "empty", "conversation": []}]}, + "at least one invocation", + ), + ( + { + "eval_cases": [ + { + "eval_id": "no_user_content", + "conversation": [ + {"final_response": {"parts": [{"text": "Hi"}]}} + ], + } + ] + }, + "user_content", + ), + ], + ) + def test_load_from_adk_eval_set_invalid(self, eval_set, error): + with pytest.raises(ValueError, match=error): + agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set(eval_set) + + def test_load_from_adk_eval_set_legacy_list_format(self, tmp_path): + path = tmp_path / "legacy.evalset.json" + path.write_text(json.dumps([{"name": "legacy", "data": []}])) + + with pytest.raises(ValueError, match="legacy list format"): + agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + str(path) + ) + @mock.patch.object(_gcs_utils, "GcsUtils") def test_load_from_observability_eval_cases(self, mock_gcs_utils): """Tests that load_from_observability_eval_cases reads data from GCS.""" @@ -10207,6 +10605,96 @@ def test_resolve_dataset_preserves_conversation_history( ptd_values = uploaded_data["prompt"]["promptTemplateData"]["values"] assert "conversation_history" in ptd_values + @pytest.mark.parametrize( + "eval_id, expected_request", + [ + ( + "turn_off_device", + { + "prompt": {"text": "Turn off device_2."}, + "goldenResponse": {"text": "device_2 is now off."}, + }, + ), + ( + "comfortable_bedroom", + { + "prompt": { + "userScenario": { + "startingPrompt": "I want my bedroom to be comfortable.", + "conversationPlan": "Ask for 21 degrees once the agent asks.", + } + } + }, + ), + ], + ) + @mock.patch.object(_evals_common, "evals") + @mock.patch.object(_evals_common, "_gcs_utils") + def test_resolve_dataset_from_adk_eval_set( + self, mock_gcs_utils, mock_evals_module, eval_id, expected_request + ): + mock_gcs_instance = mock_gcs_utils.GcsUtils.return_value + mock_gcs_instance.upload_json_to_prefix.return_value = ( + "gs://bucket/path/request.json" + ) + mock_evals_instance = mock_evals_module.Evals.return_value + mock_evals_instance.create_evaluation_item.return_value.name = "eval_item_1" + mock_evals_instance.create_evaluation_set.return_value.name = "eval_set_1" + loaded = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[ + case for case in loaded.eval_cases if case.eval_case_id == eval_id + ] + ) + + result = _evals_common._resolve_dataset( + api_client=self.mock_api_client, + dataset=dataset, + dest="gs://bucket/prefix", + ) + + assert result.evaluation_set == "eval_set_1" + mock_evals_instance.create_evaluation_set.assert_called_once_with( + evaluation_items=["eval_item_1"] + ) + uploaded_data = mock_gcs_instance.upload_json_to_prefix.call_args.kwargs["data"] + assert uploaded_data == expected_request + + @mock.patch.object(_evals_common, "evals") + @mock.patch.object(_evals_common, "_gcs_utils") + def test_resolve_dataset_from_adk_eval_set_multi_turn( + self, mock_gcs_utils, mock_evals_module + ): + mock_gcs_instance = mock_gcs_utils.GcsUtils.return_value + mock_gcs_instance.upload_json_to_prefix.return_value = ( + "gs://bucket/path/request.json" + ) + mock_evals_instance = mock_evals_module.Evals.return_value + mock_evals_instance.create_evaluation_item.return_value.name = "eval_item_1" + mock_evals_instance.create_evaluation_set.return_value.name = "eval_set_1" + loaded = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[loaded.eval_cases[1]] + ) + + _evals_common._resolve_dataset( + api_client=self.mock_api_client, + dataset=dataset, + dest="gs://bucket/prefix", + ) + + uploaded_data = mock_gcs_instance.upload_json_to_prefix.call_args.kwargs["data"] + values = uploaded_data["prompt"]["promptTemplateData"]["values"] + assert values["prompt"]["parts"] == [{"text": "Turn it off."}] + history_text = values["conversation_history"]["parts"][0]["text"] + assert history_text.startswith("user: Which devices are on?") + assert history_text.endswith("model: device_1 is on.") + assert uploaded_data["goldenResponse"] == {"text": "device_1 is now off."} + class TestResolveDatasetWithInteractions: """Tests for resolving interactions_data_source in _resolve_dataset.""" diff --git a/vertexai/_genai/_evals_data_converters.py b/vertexai/_genai/_evals_data_converters.py index 16e7779ad9..f293152cbe 100644 --- a/vertexai/_genai/_evals_data_converters.py +++ b/vertexai/_genai/_evals_data_converters.py @@ -15,15 +15,18 @@ """Dataset converters for evals.""" import copy +import datetime import json import logging -from typing import Any, Optional, Union +from typing import Any, Optional, TypeVar, Union from google.genai import _common from google.genai import types as genai_types +from pydantic import alias_generators from pydantic import ValidationError from typing_extensions import override +from . import _evals_constant from . import _evals_utils from . import _observability_data_converter from . import types @@ -605,6 +608,308 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: return types.EvaluationDataset(eval_cases=eval_cases) +_ADK_EVAL_CASE_FIELDS = frozenset( + { + "eval_id", + "conversation", + "conversation_scenario", + "session_input", + "creation_timestamp", + "rubrics", + "final_session_state", + } +) +_ADK_EVAL_CASE_KEYS = _ADK_EVAL_CASE_FIELDS | { + alias_generators.to_camel(field) for field in _ADK_EVAL_CASE_FIELDS +} +_ADK_SESSION_INPUT_FIELDS = ("app_name", "user_id", "state") +_ADK_DEFAULT_RUBRIC_GROUP = "adk_rubrics" +_REFERENCE_TRAJECTORY = "reference_trajectory" + +_GenaiModel = TypeVar("_GenaiModel", bound=_common.BaseModel) + + +def _get_adk_field(data: dict[str, Any], name: str) -> Any: + """Returns an ADK eval set field, which ADK accepts in snake or camel case.""" + if not isinstance(data, dict): + raise TypeError(f"Expected a JSON object for '{name}', got {type(data)}.") + if name in data: + return data[name] + return data.get(alias_generators.to_camel(name)) + + +def _validate_adk_genai(model: type[_GenaiModel], data: Any) -> _GenaiModel: + """Validates ADK eval set JSON as a genai type. + + Validating in JSON mode decodes base64 bytes fields, such as inline data and + thought signatures, the same way ADK does when it reads an eval set file. + + Args: + model: The genai type to validate as. + data: The parsed JSON value from the eval set. + + Returns: + The validated genai object. + """ + return model.model_validate_json(json.dumps(data)) + + +def _adk_content(data: dict[str, Any], default_role: str) -> genai_types.Content: + """Validates an ADK Content, filling in the role when the file omits it.""" + content = _validate_adk_genai(genai_types.Content, data) + if not content.role: + content.role = default_role + return content + + +def _adk_user_content(invocation: dict[str, Any]) -> genai_types.Content: + """Returns the user content of an ADK invocation.""" + user_content = _get_adk_field(invocation, "user_content") + if user_content is None: + raise ValueError("Every invocation must have user_content.") + return _adk_content(user_content, "user") + + +def _adk_intermediate_events( + intermediate_data: Optional[dict[str, Any]], +) -> list[tuple[Optional[str], genai_types.Content]]: + """Returns (author, content) pairs from ADK IntermediateData or InvocationEvents.""" + if not intermediate_data: + return [] + invocation_events = _get_adk_field(intermediate_data, "invocation_events") + if invocation_events is not None: + return [ + ( + _get_adk_field(event, "author"), + _validate_adk_genai( + genai_types.Content, _get_adk_field(event, "content") + ), + ) + for event in invocation_events + if _get_adk_field(event, "content") + ] + + events: list[tuple[Optional[str], genai_types.Content]] = [] + tool_uses = _get_adk_field(intermediate_data, "tool_uses") + if tool_uses: + events.append( + ( + None, + genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=_validate_adk_genai( + genai_types.FunctionCall, tool_use + ) + ) + for tool_use in tool_uses + ], + ), + ) + ) + tool_responses = _get_adk_field(intermediate_data, "tool_responses") + if tool_responses: + events.append( + ( + None, + genai_types.Content( + role="user", + parts=[ + genai_types.Part( + function_response=_validate_adk_genai( + genai_types.FunctionResponse, tool_response + ) + ) + for tool_response in tool_responses + ], + ), + ) + ) + for author, parts in ( + _get_adk_field(intermediate_data, "intermediate_responses") or [] + ): + events.append( + ( + author, + genai_types.Content( + role="model", + parts=[ + _validate_adk_genai(genai_types.Part, part) for part in parts + ], + ), + ) + ) + return events + + +class AdkEvalSetConverter(_evals_utils.EvalDataConverter): + """Converter for ADK eval sets (the `.evalset.json` format). + + Reads the eval set JSON directly, so google-adk does not need to be + installed. + """ + + def _invocation_to_messages( + self, invocation: dict[str, Any], turn_index: int + ) -> list[types.evals.Message]: + """Converts an earlier ADK invocation into conversation history messages.""" + turn_id = _get_adk_field(invocation, "invocation_id") or str(turn_index) + timestamp = _get_adk_field(invocation, "creation_timestamp") + messages = [ + types.evals.Message( + turn_id=turn_id, + author="user", + content=_adk_user_content(invocation), + creation_timestamp=( + datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + if timestamp + else None + ), + ) + ] + messages.extend( + types.evals.Message(turn_id=turn_id, author=author, content=content) + for author, content in _adk_intermediate_events( + _get_adk_field(invocation, "intermediate_data") + ) + ) + final_response = _get_adk_field(invocation, "final_response") + if final_response: + messages.append( + types.evals.Message( + turn_id=turn_id, content=_adk_content(final_response, "model") + ) + ) + return messages + + def _convert_rubrics( + self, rubrics: list[dict[str, Any]] + ) -> dict[str, types.RubricGroup]: + """Groups ADK rubrics by their type, which ADK uses to select rubrics.""" + groups: dict[str, list[types.evals.Rubric]] = {} + for rubric in rubrics: + rubric_type = _get_adk_field(rubric, "type") + rubric_content = _get_adk_field(rubric, "rubric_content") or {} + groups.setdefault(rubric_type or _ADK_DEFAULT_RUBRIC_GROUP, []).append( + types.evals.Rubric( + rubric_id=_get_adk_field(rubric, "rubric_id"), + type=rubric_type, + content=types.evals.RubricContent( + property=types.evals.RubricContentProperty( + description=_get_adk_field(rubric_content, "text_property") + ) + ), + ) + ) + return { + name: types.RubricGroup(rubrics=group) for name, group in groups.items() + } + + def _convert_eval_case(self, case: dict[str, Any]) -> types.EvalCase: + """Converts one ADK EvalCase into an EvalCase.""" + if not isinstance(case, dict): + raise TypeError(f"Expected an ADK EvalCase object, got {type(case)}.") + conversation = _get_adk_field(case, "conversation") + scenario = _get_adk_field(case, "conversation_scenario") + if (conversation is None) == (scenario is None): + raise ValueError( + "Exactly one of conversation and conversation_scenario must be set." + ) + + fields: dict[str, Any] = {"eval_case_id": _get_adk_field(case, "eval_id")} + rubrics = list(_get_adk_field(case, "rubrics") or []) + if scenario is not None: + fields["user_scenario"] = types.evals.UserScenario( + starting_prompt=_get_adk_field(scenario, "starting_prompt"), + conversation_plan=_get_adk_field(scenario, "conversation_plan"), + ) + else: + if not conversation: + raise ValueError("conversation must have at least one invocation.") + *earlier_invocations, last_invocation = conversation + fields["prompt"] = _adk_user_content(last_invocation) + history = [] + for turn_index, invocation in enumerate(earlier_invocations): + history.extend(self._invocation_to_messages(invocation, turn_index)) + if history: + fields["conversation_history"] = history + final_response = _get_adk_field(last_invocation, "final_response") + if final_response: + fields["reference"] = types.ResponseCandidate( + response=_adk_content(final_response, "model") + ) + reference_trajectory: list[genai_types.Content] = [] + for _, content in _adk_intermediate_events( + _get_adk_field(last_invocation, "intermediate_data") + ): + reference_trajectory.extend( + genai_types.Content( + role="model", + parts=[genai_types.Part(function_call=part.function_call)], + ) + for part in content.parts or [] + if part.function_call + ) + if reference_trajectory: + fields[_REFERENCE_TRAJECTORY] = reference_trajectory + rubrics.extend(_get_adk_field(last_invocation, "rubrics") or []) + + if rubrics: + fields["rubric_groups"] = self._convert_rubrics(rubrics) + session_input = _get_adk_field(case, "session_input") + if session_input: + fields[_evals_constant.SESSION_INPUT] = { + name: value + for name in _ADK_SESSION_INPUT_FIELDS + if (value := _get_adk_field(session_input, name)) is not None + } + + reserved_keys = set(fields) + for field_name in types.EvalCase.model_fields: + reserved_keys.update((field_name, alias_generators.to_camel(field_name))) + for key, value in case.items(): + if key in _ADK_EVAL_CASE_KEYS: + continue + if key in reserved_keys: + logger.warning( + "Skipping ADK eval case field '%s' because it conflicts with" + " an EvalCase field.", + key, + ) + continue + fields[key] = value + return types.EvalCase(**fields) + + @override + def convert(self, raw_data: dict[str, Any]) -> types.EvaluationDataset: + """Converts a parsed ADK EvalSet into an EvaluationDataset.""" + if not isinstance(raw_data, dict): + raise ValueError( + "Expected an ADK EvalSet JSON object, got" + f" {type(raw_data)}. For the legacy list format, load the" + " file with google.adk.evaluation.local_eval_sets_manager" + ".load_eval_set_from_file() and pass" + " eval_set.model_dump(mode='json') instead." + ) + adk_eval_cases = _get_adk_field(raw_data, "eval_cases") + if not isinstance(adk_eval_cases, list): + raise ValueError("Expected an ADK EvalSet with an 'eval_cases' list.") + + eval_cases = [] + for i, case in enumerate(adk_eval_cases): + try: + eval_cases.append(self._convert_eval_case(case)) + except (TypeError, ValueError) as e: + case_id = ( + _get_adk_field(case, "eval_id") if isinstance(case, dict) else None + ) + raise ValueError( + f"Failed to convert ADK eval case '{case_id or i}': {e}" + ) from e + return types.EvaluationDataset(eval_cases=eval_cases) + + def auto_detect_dataset_schema( raw_dataset: list[dict[str, Any]], ) -> Union[EvalDatasetSchema, str]: diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 5bb3b8811b..385087ec8e 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -3515,6 +3515,60 @@ def load_from_observability_eval_cases( return EvaluationDataset(eval_dataset_df=eval_dataset_df) + @classmethod + def load_from_adk_eval_set( + cls, eval_set: Union[str, "os.PathLike[str]", dict[str, Any]] + ) -> "EvaluationDataset": + """Loads an ADK eval set (`.evalset.json`) into an EvaluationDataset. + + Each ADK eval case becomes an EvalCase with `eval_case_id` set to its + `eval_id`. For a static conversation, the last invocation's user content + becomes `prompt` and its final response becomes `reference`. Earlier + invocations, including their tool calls, become `conversation_history`. + A conversation scenario becomes `user_scenario`. Rubrics on the case and + on the last invocation go into `rubric_groups`, keyed by rubric type + ("adk_rubrics" when unset). + + Some ADK data is kept as EvalCase extra fields: `reference_trajectory` + holds the tool calls of the last invocation, `session_inputs` holds the + session input, and custom ADK eval case fields keep their names. Other ADK + fields, such as `final_session_state` and the scenario's `user_persona`, + are not carried over. Steps that convert eval cases to a DataFrame, such + as `run_inference`, drop the extra fields, `rubric_groups` and + `eval_case_id`. google-adk does not need to be installed. + + Example: + import vertexai + from vertexai._genai import types + + client = vertexai.Client( + project="my-project", location="us-central1" + ) + dataset = types.EvaluationDataset.load_from_adk_eval_set( + "my_agent/my_eval_set.evalset.json" + ) + dataset_with_responses = client.evals.run_inference( + src=dataset, agent=my_agent + ) + result = client.evals.evaluate( + dataset=dataset_with_responses, + metrics=[types.RubricMetric.FINAL_RESPONSE_MATCH], + ) + + Args: + eval_set: The path to an ADK `.evalset.json` file, or the eval set as + a parsed JSON dict. + + Returns: + An EvaluationDataset with one EvalCase per ADK eval case. + """ + from .. import _evals_data_converters # pylint: disable=g-import-not-at-top + + if not isinstance(eval_set, dict): + with open(eval_set, "r", encoding="utf-8") as f: + eval_set = json.load(f) + return _evals_data_converters.AdkEvalSetConverter().convert(eval_set) + def show(self) -> None: """Shows the evaluation dataset.""" from .. import _evals_visualization From b1378bb54566396506dfe17041f4e8d3fb715573 Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Fri, 2 Oct 2026 15:01:47 -0700 Subject: [PATCH 4/7] feat: surface the LRO name in Agent Engine create and update requests PiperOrigin-RevId: 992557497 --- agentplatform/_genai/runtimes.py | 10 ++++++++-- tests/unit/agentplatform/genai/test_agent_engines.py | 4 ++-- vertexai/_genai/agent_engines.py | 10 ++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/agentplatform/_genai/runtimes.py b/agentplatform/_genai/runtimes.py index 602805f1a6..ab235e3765 100644 --- a/agentplatform/_genai/runtimes.py +++ b/agentplatform/_genai/runtimes.py @@ -1913,6 +1913,7 @@ def create( build_config=config.build_config, ) operation = self._create(config=api_config) + logger.info(f"Create Agent Runtime backing LRO: {operation.name}") reasoning_engine_id = _runtimes_utils._get_reasoning_engine_id( operation_name=operation.name ) @@ -1943,7 +1944,9 @@ def create( f"runtime=client.runtimes.get(name='{runtime.api_resource.name}')" ) elif operation.error: - raise RuntimeError(f"Failed to create Agent Runtime: {operation.error}") + raise RuntimeError( + f"Operation {operation.name} failed to create Agent Runtime: {operation.error}" + ) else: logger.warning("The operation returned an empty response.") if not self._is_lightweight_creation(agent, config): @@ -2719,6 +2722,7 @@ def update( build_config=config.build_config, ) operation = self._update(name=name, config=api_config) + logger.info(f"Update Agent Runtime backing LRO: {operation.name}") reasoning_engine_id = _runtimes_utils._get_reasoning_engine_id( resource_name=name ) @@ -2743,7 +2747,9 @@ def update( f"runtime=client.runtimes.get(name='{runtime.api_resource.name}')" ) elif operation.error: - raise RuntimeError(f"Failed to update Agent Runtime: {operation.error}") + raise RuntimeError( + f"Operation {operation.name} failed to update Agent Runtime: {operation.error}" + ) if runtime.api_resource.spec: self._register_api_methods(runtime=runtime) return runtime # type: ignore[no-any-return] diff --git a/tests/unit/agentplatform/genai/test_agent_engines.py b/tests/unit/agentplatform/genai/test_agent_engines.py index 7e84a80ff2..08f8c37e60 100644 --- a/tests/unit/agentplatform/genai/test_agent_engines.py +++ b/tests/unit/agentplatform/genai/test_agent_engines.py @@ -4167,7 +4167,7 @@ def test_create_runtime_error( env_vars=_TEST_AGENT_ENGINE_ENV_VARS_INPUT, ), ) - assert "Failed to create agent engine" in str(excinfo.value) + assert "failed to create Agent Runtime" in str(excinfo.value) @mock.patch.object(_runtimes_utils, "_await_operation") def test_update_runtime_description(self, mock_await_operation): @@ -4185,7 +4185,7 @@ def test_update_runtime_description(self, mock_await_operation): description=_TEST_AGENT_ENGINE_DESCRIPTION, ), ) - assert "Failed to update agent engine" in str(excinfo.value) + assert "failed to update Agent Runtime" in str(excinfo.value) @pytest.mark.parametrize( "test_case_name, test_operation_schemas, want_log_output", diff --git a/vertexai/_genai/agent_engines.py b/vertexai/_genai/agent_engines.py index 7e40b697ca..e8aa8be81e 100644 --- a/vertexai/_genai/agent_engines.py +++ b/vertexai/_genai/agent_engines.py @@ -2008,6 +2008,7 @@ def create( build_config=config.build_config, ) operation = self._create(config=api_config) + logger.info(f"Create Agent Engine backing LRO: {operation.name}") reasoning_engine_id = _agent_engines_utils._get_reasoning_engine_id( operation_name=operation.name ) @@ -2038,7 +2039,9 @@ def create( f"agent_engine=client.agent_engines.get(name='{agent_engine.api_resource.name}')" ) elif operation.error: - raise RuntimeError(f"Failed to create Agent Engine: {operation.error}") + raise RuntimeError( + f"Operation {operation.name} failed to create Agent Engine: {operation.error}" + ) else: logger.warning("The operation returned an empty response.") if not self._is_lightweight_creation(agent, config): @@ -2820,6 +2823,7 @@ def update( build_config=config.build_config, ) operation = self._update(name=name, config=api_config) + logger.info(f"Update Agent Engine backing LRO: {operation.name}") reasoning_engine_id = _agent_engines_utils._get_reasoning_engine_id( resource_name=name ) @@ -2844,7 +2848,9 @@ def update( f"agent_engine=client.agent_engines.get(name='{agent_engine.api_resource.name}')" ) elif operation.error: - raise RuntimeError(f"Failed to update Agent Engine: {operation.error}") + raise RuntimeError( + f"Operation {operation.name} failed to update Agent Engine: {operation.error}" + ) if agent_engine.api_resource.spec: self._register_api_methods(agent_engine=agent_engine) return agent_engine # type: ignore[no-any-return] From 39693a9d956449f810c18d340eb57526faa9cb00 Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Tue, 6 Oct 2026 11:45:32 -0700 Subject: [PATCH 5/7] feat: update discovery doc PiperOrigin-RevId: 994529596 --- agentplatform/_genai/types/__init__.py | 6 ++++ agentplatform/_genai/types/common.py | 49 ++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index 40378c1d0f..1c122ccd3a 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -1995,6 +1995,9 @@ from .common import SandboxEnvironmentTemplateOperationDict from .common import SandboxEnvironmentTemplateOperationOrDict from .common import SandboxEnvironmentTemplateOrDict +from .common import SandboxEnvironmentTemplatePersistentDiskConfig +from .common import SandboxEnvironmentTemplatePersistentDiskConfigDict +from .common import SandboxEnvironmentTemplatePersistentDiskConfigOrDict from .common import SandboxEnvironmentTemplateResourceRequirements from .common import SandboxEnvironmentTemplateResourceRequirementsDict from .common import SandboxEnvironmentTemplateResourceRequirementsOrDict @@ -3516,6 +3519,9 @@ "CreateSandboxEnvironmentTemplateConfig", "CreateSandboxEnvironmentTemplateConfigDict", "CreateSandboxEnvironmentTemplateConfigOrDict", + "SandboxEnvironmentTemplatePersistentDiskConfig", + "SandboxEnvironmentTemplatePersistentDiskConfigDict", + "SandboxEnvironmentTemplatePersistentDiskConfigOrDict", "SandboxEnvironmentTemplate", "SandboxEnvironmentTemplateDict", "SandboxEnvironmentTemplateOrDict", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index d59acc7cfa..56efc40a53 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -1733,7 +1733,7 @@ def evaluate(instance: dict[str, Any]) -> float: ) code_execution_region: Optional[str] = Field( default=None, - description="""Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Must be a region where the Code Execution Sandbox is available. Supported regions: northamerica-northeast1, southamerica-east1, us-central1, us-east1, us-east4, us-west1, us-west4, europe-central2, europe-north1, europe-southwest1, europe-west1, europe-west2, europe-west3, europe-west4, europe-west6, europe-west8, europe-west9, me-west1, asia-east1, asia-east2, asia-northeast1, asia-northeast3, asia-south1, asia-south2, asia-southeast1, australia-southeast2. If unset, the request's originating region is used; requests from regions where the sandbox is unavailable will fail with UNIMPLEMENTED.""", + description="""Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Must be a region where the Code Execution Sandbox is available. For the current list of [supported regions](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-engine/locations). If unset, the request's originating region is used; requests from regions where the sandbox is unavailable will fail with UNIMPLEMENTED.""", ) @@ -1755,7 +1755,7 @@ def evaluate(instance: dict[str, Any]) -> float: are available to the function as instance[field_name].""" code_execution_region: Optional[str] - """Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Must be a region where the Code Execution Sandbox is available. Supported regions: northamerica-northeast1, southamerica-east1, us-central1, us-east1, us-east4, us-west1, us-west4, europe-central2, europe-north1, europe-southwest1, europe-west1, europe-west2, europe-west3, europe-west4, europe-west6, europe-west8, europe-west9, me-west1, asia-east1, asia-east2, asia-northeast1, asia-northeast3, asia-south1, asia-south2, asia-southeast1, australia-southeast2. If unset, the request's originating region is used; requests from regions where the sandbox is unavailable will fail with UNIMPLEMENTED.""" + """Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Must be a region where the Code Execution Sandbox is available. For the current list of [supported regions](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-engine/locations). If unset, the request's originating region is used; requests from regions where the sandbox is unavailable will fail with UNIMPLEMENTED.""" CustomCodeExecutionSpecOrDict = Union[ @@ -18001,6 +18001,42 @@ class _CreateSandboxEnvironmentTemplateRequestParametersDict(TypedDict, total=Fa ] +class SandboxEnvironmentTemplatePersistentDiskConfig(_common.BaseModel): + """Configuration for attaching a persistent disk (PD) to each SandboxEnvironment created from this template. A persistent disk provides durable, per-sandbox block storage whose contents survive across the sandbox lifecycle events that this service supports (e.g. pause/resume), unlike ephemeral local storage which is lost when the underlying runtime is torn down.""" + + enabled: Optional[bool] = Field( + default=None, + description="""Optional. Whether a persistent disk is attached to sandboxes created from this template. Defaults to `false`. This flag lets a template carry (and preserve) disk configuration while keeping the disk detached, so it can be toggled on later without re-specifying the rest of the config. When `false`, the remaining fields in this message are ignored.""", + ) + mount_path: Optional[str] = Field( + default=None, + description="""Optional. The absolute path inside the sandbox container at which the persistent disk is mounted. Defaults to `/workspace` when unset. Ignored when `enabled` is `false`. Only writes beneath this path land on the disk. Writes elsewhere go to the container's writable layer, which counts against the container's ephemeral storage and is lost when the sandbox's runtime is torn down, so this should be the directory the workload actually writes to. Paths that would shadow the container's system directories (for example `/etc`, `/proc`, or `/usr` itself) are rejected.""", + ) + size_gb: Optional[int] = Field( + default=None, + description="""Optional. The size of the persistent disk in GB. Must be non-negative. When `enabled` is `true`, a positive value is required; when unset or zero while enabled, the service applies a default size. Ignored when `enabled` is `false`.""", + ) + + +class SandboxEnvironmentTemplatePersistentDiskConfigDict(TypedDict, total=False): + """Configuration for attaching a persistent disk (PD) to each SandboxEnvironment created from this template. A persistent disk provides durable, per-sandbox block storage whose contents survive across the sandbox lifecycle events that this service supports (e.g. pause/resume), unlike ephemeral local storage which is lost when the underlying runtime is torn down.""" + + enabled: Optional[bool] + """Optional. Whether a persistent disk is attached to sandboxes created from this template. Defaults to `false`. This flag lets a template carry (and preserve) disk configuration while keeping the disk detached, so it can be toggled on later without re-specifying the rest of the config. When `false`, the remaining fields in this message are ignored.""" + + mount_path: Optional[str] + """Optional. The absolute path inside the sandbox container at which the persistent disk is mounted. Defaults to `/workspace` when unset. Ignored when `enabled` is `false`. Only writes beneath this path land on the disk. Writes elsewhere go to the container's writable layer, which counts against the container's ephemeral storage and is lost when the sandbox's runtime is torn down, so this should be the directory the workload actually writes to. Paths that would shadow the container's system directories (for example `/etc`, `/proc`, or `/usr` itself) are rejected.""" + + size_gb: Optional[int] + """Optional. The size of the persistent disk in GB. Must be non-negative. When `enabled` is `true`, a positive value is required; when unset or zero while enabled, the service applies a default size. Ignored when `enabled` is `false`.""" + + +SandboxEnvironmentTemplatePersistentDiskConfigOrDict = Union[ + SandboxEnvironmentTemplatePersistentDiskConfig, + SandboxEnvironmentTemplatePersistentDiskConfigDict, +] + + class SandboxEnvironmentTemplate(_common.BaseModel): """A sandbox environment template.""" @@ -18050,6 +18086,12 @@ class SandboxEnvironmentTemplate(_common.BaseModel): default=None, description="""Optional. Immutable. Whether to provision the SandboxEnvironmentTemplate via the GKE TD pool.""", ) + persistent_disk_config: Optional[SandboxEnvironmentTemplatePersistentDiskConfig] = ( + Field( + default=None, + description="""Optional. Configuration for attaching a persistent disk (PD) to each SandboxEnvironment created from this template. When unset (or when `enabled` is `false`), sandboxes created from this template are not backed by a persistent disk and rely on ephemeral storage only. See PersistentDiskConfig for details.""", + ) + ) class SandboxEnvironmentTemplateDict(TypedDict, total=False): @@ -18089,6 +18131,9 @@ class SandboxEnvironmentTemplateDict(TypedDict, total=False): use_gke_td: Optional[bool] """Optional. Immutable. Whether to provision the SandboxEnvironmentTemplate via the GKE TD pool.""" + persistent_disk_config: Optional[SandboxEnvironmentTemplatePersistentDiskConfigDict] + """Optional. Configuration for attaching a persistent disk (PD) to each SandboxEnvironment created from this template. When unset (or when `enabled` is `false`), sandboxes created from this template are not backed by a persistent disk and rely on ephemeral storage only. See PersistentDiskConfig for details.""" + SandboxEnvironmentTemplateOrDict = Union[ SandboxEnvironmentTemplate, SandboxEnvironmentTemplateDict From 16489d32d124f27d2bc7ff45f5d46cd6e5e5a3b4 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Tue, 6 Oct 2026 13:25:28 -0700 Subject: [PATCH 6/7] chore: GenAI Client(evals) - Add public alias for extract_dataset_rows. PiperOrigin-RevId: 994589437 --- agentplatform/_genai/_evals_visualization.py | 3 ++ tests/unit/agentplatform/genai/test_evals.py | 30 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/agentplatform/_genai/_evals_visualization.py b/agentplatform/_genai/_evals_visualization.py index b6d335e7c5..c529b6f14e 100644 --- a/agentplatform/_genai/_evals_visualization.py +++ b/agentplatform/_genai/_evals_visualization.py @@ -232,6 +232,9 @@ def _extract_dataset_rows(dataset: types.EvaluationDataset) -> list[dict[str, An return processed_rows +extract_dataset_rows = _extract_dataset_rows + + def get_evaluation_html(eval_result_json: str) -> str: """Returns a self-contained HTML for single evaluation visualization.""" payload_b64 = _encode_to_base64(eval_result_json) diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 28807dfc67..22e368fd56 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -844,6 +844,36 @@ def test_get_inference_html(self): assert payload_b64 in html assert "DOMPurify" in html + def test_extract_dataset_rows(self): + dataset = common_types.EvaluationDataset( + eval_cases=[ + common_types.EvalCase( + prompt=genai_types.Content( + parts=[genai_types.Part(text="What is 2+2?")] + ), + responses=[ + common_types.ResponseCandidate( + response=genai_types.Content( + parts=[genai_types.Part(text="4")] + ) + ) + ], + reference=common_types.ResponseCandidate( + response=genai_types.Content( + parts=[genai_types.Part(text="Four")] + ) + ), + ) + ] + ) + + rows = _evals_visualization.extract_dataset_rows(dataset) + + assert len(rows) == 1 + assert rows[0]["prompt_display_text"] == "What is 2+2?" + assert rows[0]["response_display_text"] == "4" + assert rows[0]["reference"] == "Four" + def test_display_loss_clusters_response_no_ipython(self): """Tests graceful fallback when not in IPython.""" from agentplatform._genai import _evals_visualization From 0d4713a62ffe6a68f11417e3a90807845a12e869 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:05:23 -0700 Subject: [PATCH 7/7] chore(main): release 2.4.0 (#7177) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Jeff Scudder --- .release-please-manifest.json | 2 +- CHANGELOG.md | 15 +++++++++++++++ agentplatform/version.py | 2 +- google/cloud/aiplatform/gapic_version.py | 2 +- .../v1/schema/predict/instance/gapic_version.py | 2 +- .../schema/predict/instance_v1/gapic_version.py | 2 +- .../v1/schema/predict/params/gapic_version.py | 2 +- .../v1/schema/predict/params_v1/gapic_version.py | 2 +- .../v1/schema/predict/prediction/gapic_version.py | 2 +- .../schema/predict/prediction_v1/gapic_version.py | 2 +- .../trainingjob/definition/gapic_version.py | 2 +- .../trainingjob/definition_v1/gapic_version.py | 2 +- .../schema/predict/instance/gapic_version.py | 2 +- .../predict/instance_v1beta1/gapic_version.py | 2 +- .../schema/predict/params/gapic_version.py | 2 +- .../predict/params_v1beta1/gapic_version.py | 2 +- .../schema/predict/prediction/gapic_version.py | 2 +- .../predict/prediction_v1beta1/gapic_version.py | 2 +- .../trainingjob/definition/gapic_version.py | 2 +- .../definition_v1beta1/gapic_version.py | 2 +- google/cloud/aiplatform/version.py | 2 +- google/cloud/aiplatform_v1/gapic_version.py | 2 +- google/cloud/aiplatform_v1beta1/gapic_version.py | 2 +- pypi/_vertex_ai_placeholder/version.py | 2 +- ...ippet_metadata_google.cloud.aiplatform.v1.json | 2 +- ..._metadata_google.cloud.aiplatform.v1beta1.json | 2 +- 26 files changed, 40 insertions(+), 25 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ccf4e22358..edb1d1b648 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.3.0" + ".": "2.4.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 7043a994dc..0fc32e4c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [2.4.0](https://github.com/googleapis/python-aiplatform/compare/v2.3.0...v2.4.0) (2026-10-06) + + +### Features + +* Add per-step judges and cross-region routing to evals ([5fa1455](https://github.com/googleapis/python-aiplatform/commit/5fa14552cd06179e8ea08f0ea0d29ea6cafb1ca0)) +* GenAI Client(evals) - add EvaluationDataset.load_from_adk_eval_set ([4c045e1](https://github.com/googleapis/python-aiplatform/commit/4c045e16c2459d0708bca77c2a82a3c1daff0409)) +* Surface the LRO name in Agent Engine create and update requests ([b1378bb](https://github.com/googleapis/python-aiplatform/commit/b1378bb54566396506dfe17041f4e8d3fb715573)) +* Update discovery doc ([39693a9](https://github.com/googleapis/python-aiplatform/commit/39693a9d956449f810c18d340eb57526faa9cb00)) + + +### Bug Fixes + +* Replay session_events into a scratch session and delete it afterwards ([3647753](https://github.com/googleapis/python-aiplatform/commit/364775307aa9c0b4d30844da4c7f23c05221dfcd)) + ## [2.3.0](https://github.com/googleapis/python-aiplatform/compare/v2.2.0...v2.3.0) (2026-09-25) diff --git a/agentplatform/version.py b/agentplatform/version.py index e46e6f6b55..b2ab5e12dd 100644 --- a/agentplatform/version.py +++ b/agentplatform/version.py @@ -14,4 +14,4 @@ # """Version file for agentplatform.""" -__version__ = "2.3.0" +__version__ = "2.4.0" diff --git a/google/cloud/aiplatform/gapic_version.py b/google/cloud/aiplatform/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/gapic_version.py +++ b/google/cloud/aiplatform/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/predict/instance/gapic_version.py b/google/cloud/aiplatform/v1/schema/predict/instance/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/predict/instance/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/predict/instance/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/predict/instance_v1/gapic_version.py b/google/cloud/aiplatform/v1/schema/predict/instance_v1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/predict/instance_v1/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/predict/instance_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/predict/params/gapic_version.py b/google/cloud/aiplatform/v1/schema/predict/params/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/predict/params/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/predict/params/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/predict/params_v1/gapic_version.py b/google/cloud/aiplatform/v1/schema/predict/params_v1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/predict/params_v1/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/predict/params_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/predict/prediction/gapic_version.py b/google/cloud/aiplatform/v1/schema/predict/prediction/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/predict/prediction/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/predict/prediction/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/predict/prediction_v1/gapic_version.py b/google/cloud/aiplatform/v1/schema/predict/prediction_v1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/predict/prediction_v1/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/predict/prediction_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/trainingjob/definition/gapic_version.py b/google/cloud/aiplatform/v1/schema/trainingjob/definition/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/trainingjob/definition/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/trainingjob/definition/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1/schema/trainingjob/definition_v1/gapic_version.py b/google/cloud/aiplatform/v1/schema/trainingjob/definition_v1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1/schema/trainingjob/definition_v1/gapic_version.py +++ b/google/cloud/aiplatform/v1/schema/trainingjob/definition_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/predict/instance/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/predict/instance/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/predict/instance/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/predict/instance/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/predict/instance_v1beta1/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/predict/instance_v1beta1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/predict/instance_v1beta1/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/predict/instance_v1beta1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/predict/params/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/predict/params/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/predict/params/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/predict/params/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/predict/params_v1beta1/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/predict/params_v1beta1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/predict/params_v1beta1/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/predict/params_v1beta1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/predict/prediction/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/predict/prediction/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/predict/prediction/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/predict/prediction/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/predict/prediction_v1beta1/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/predict/prediction_v1beta1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/predict/prediction_v1beta1/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/predict/prediction_v1beta1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition_v1beta1/gapic_version.py b/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition_v1beta1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition_v1beta1/gapic_version.py +++ b/google/cloud/aiplatform/v1beta1/schema/trainingjob/definition_v1beta1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform/version.py b/google/cloud/aiplatform/version.py index 1473927a5b..fc83fa3688 100644 --- a/google/cloud/aiplatform/version.py +++ b/google/cloud/aiplatform/version.py @@ -15,4 +15,4 @@ # limitations under the License. # -__version__ = "2.3.0" +__version__ = "2.4.0" diff --git a/google/cloud/aiplatform_v1/gapic_version.py b/google/cloud/aiplatform_v1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform_v1/gapic_version.py +++ b/google/cloud/aiplatform_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/google/cloud/aiplatform_v1beta1/gapic_version.py b/google/cloud/aiplatform_v1beta1/gapic_version.py index 28334e0617..9d82f1368a 100644 --- a/google/cloud/aiplatform_v1beta1/gapic_version.py +++ b/google/cloud/aiplatform_v1beta1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" # {x-release-please-version} +__version__ = "2.4.0" # {x-release-please-version} diff --git a/pypi/_vertex_ai_placeholder/version.py b/pypi/_vertex_ai_placeholder/version.py index 2aef837d6a..a837dc2d22 100644 --- a/pypi/_vertex_ai_placeholder/version.py +++ b/pypi/_vertex_ai_placeholder/version.py @@ -15,4 +15,4 @@ # limitations under the License. # -__version__ = "2.3.0" +__version__ = "2.4.0" diff --git a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json index 93e970b1d7..4bb980e0b7 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-aiplatform", - "version": "2.3.0" + "version": "2.4.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json index 7aea8677b4..4a73ed3107 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-aiplatform", - "version": "2.3.0" + "version": "2.4.0" }, "snippets": [ {