From 0f99e73f0001a5cbe124a8da4bde6c4b08ed89b5 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Wed, 17 Dec 2025 13:45:55 +0100 Subject: [PATCH 1/3] remove spec patches as it now part of the official spec --- .../localstack/aws/spec-patches.json | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/localstack-core/localstack/aws/spec-patches.json b/localstack-core/localstack/aws/spec-patches.json index 7f46a4b7e7b15..28144dfe783d7 100644 --- a/localstack-core/localstack/aws/spec-patches.json +++ b/localstack-core/localstack/aws/spec-patches.json @@ -1372,43 +1372,5 @@ "path": "/operations/CreateApiMapping/http/responseCode", "value": 200 } - ], - "cloudwatch/2010-08-01/service-2": [ - { - "op": "add", - "path": "/metadata/awsQueryCompatible", - "value": {} - }, - { - "op": "add", - "path": "/metadata/jsonVersion", - "value": "1.0" - }, - { - "op": "add", - "path": "/metadata/targetPrefix", - "value": "GraniteServiceVersion20100801" - }, - { - "op": "replace", - "path": "/metadata/protocol", - "value": "smithy-rpc-v2-cbor" - }, - { - "op": "replace", - "path": "/metadata/protocols", - "value": [ - "smithy-rpc-v2-cbor", - "json", - "query" - ] - }, - { - "op": "add", - "path": "/shapes/ConflictException/error", - "value": { - "httpStatusCode": 409 - } - } ] } From 1d8b55e85fb94074ec7cffabbac755faa80bd6ed Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Wed, 17 Dec 2025 16:03:32 +0100 Subject: [PATCH 2/3] fix timestamp parsing and serialization for RPC v2 CBOR --- .../localstack/aws/protocol/parser.py | 7 ++++- .../services/cloudwatch/provider_v2.py | 2 +- .../services/cloudwatch/test_cloudwatch.py | 31 +++++++++++++++++++ .../test_cloudwatch.validation.json | 28 ++++++++--------- tests/aws/services/cloudwatch/utils.py | 8 +++-- tests/unit/aws/protocol/test_parser.py | 18 ++++++++++- 6 files changed, 74 insertions(+), 20 deletions(-) diff --git a/localstack-core/localstack/aws/protocol/parser.py b/localstack-core/localstack/aws/protocol/parser.py index 636bc15d7a531..6bbd7d0edcc8d 100644 --- a/localstack-core/localstack/aws/protocol/parser.py +++ b/localstack-core/localstack/aws/protocol/parser.py @@ -1124,8 +1124,13 @@ def _parse_type_tag(self, stream: io.BufferedReader, additional_info: int): raise ProtocolParserError(f"Found CBOR tag not supported by botocore: {tag}") def _parse_type_datetime(self, value: int | float) -> datetime.datetime: + # CBOR overrides any timestamp format defined in the spec: + # https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html#timestamp-type-serialization + # > This protocol uses epoch-seconds, also known as Unix timestamps, with millisecond (1/1000th of a second) + # > resolution. The timestampFormat MUST NOT be respected to customize timestamp serialization. if isinstance(value, (int, float)): - return self._convert_str_to_timestamp(str(value)) + milli_precision_ts = int(value * 1000) / 1000 + return datetime.datetime.fromtimestamp(milli_precision_ts, tz=datetime.UTC) else: raise ProtocolParserError(f"Unable to parse datetime value: {value}") diff --git a/localstack-core/localstack/services/cloudwatch/provider_v2.py b/localstack-core/localstack/services/cloudwatch/provider_v2.py index 42c5d26b2ae38..cf4a697d85f96 100644 --- a/localstack-core/localstack/services/cloudwatch/provider_v2.py +++ b/localstack-core/localstack/services/cloudwatch/provider_v2.py @@ -277,7 +277,7 @@ def get_metric_data( # Paginate timestamp_value_dicts = [ { - "Timestamp": timestamp, + "Timestamp": datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC), "Value": float(value), } for timestamp, value in zip(timestamps, values, strict=False) diff --git a/tests/aws/services/cloudwatch/test_cloudwatch.py b/tests/aws/services/cloudwatch/test_cloudwatch.py index f15dc5ff31b1d..b70a4f1205833 100644 --- a/tests/aws/services/cloudwatch/test_cloudwatch.py +++ b/tests/aws/services/cloudwatch/test_cloudwatch.py @@ -1,6 +1,7 @@ import gzip import json import logging +import re import threading import time from datetime import UTC, datetime, timedelta, timezone @@ -3026,6 +3027,36 @@ def _get_metric_data_sum(): ) snapshot.match("get-metric-data", response) + # we need special assertions for raw timestamp values, based on the protocol: + if protocol == "query": + timestamp = response["GetMetricDataResponse"]["GetMetricDataResult"][ + "MetricDataResults" + ]["member"][0]["Timestamps"]["member"] + assert re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$", timestamp) + + elif protocol == "json": + timestamp = response["MetricDataResults"][0]["Timestamps"][0] + assert isinstance(timestamp, float) + # assert this format: 1765977780.0 + assert re.match(r"^\d{10}\.0", str(timestamp)) + else: + timestamp = response["MetricDataResults"][0]["Timestamps"][0] + assert isinstance(timestamp, datetime) + assert timestamp.microsecond == 0 + assert timestamp.year == now.year + assert now.day - 1 <= timestamp.day <= now.day + 1 + + # we need to decode more for CBOR, to verify we encode it the same way as AWS (datetime format + proper + # underlying format (float) + # See https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html#timestamp-type-serialization + # https://datatracker.ietf.org/doc/html/rfc8949.html#section-3.4 + response_raw = http_client.post_raw( + operation="GetMetricData", + payload=get_metric_input, + ) + # assert that the timestamp is encoded as a Tag (6 major type) with Double of length 8 + assert b"Timestamps\x9f\xc1\xfbA" in response_raw.content + @markers.aws.validated @pytest.mark.skipif(is_old_provider(), reason="Wrong behavior in v1 in SetAlarmState") @pytest.mark.parametrize("protocol", ["json", "smithy-rpc-v2-cbor", "query"]) diff --git a/tests/aws/services/cloudwatch/test_cloudwatch.validation.json b/tests/aws/services/cloudwatch/test_cloudwatch.validation.json index c4c4a484ed761..50c0fa1bc0fb6 100644 --- a/tests/aws/services/cloudwatch/test_cloudwatch.validation.json +++ b/tests/aws/services/cloudwatch/test_cloudwatch.validation.json @@ -1,29 +1,29 @@ { "tests/aws/services/cloudwatch/test_cloudwatch.py::TestCloudWatchMultiProtocol::test_basic_operations_multiple_protocols[json]": { - "last_validated_date": "2025-10-06T14:45:54+00:00", + "last_validated_date": "2025-12-17T13:53:06+00:00", "durations_in_seconds": { - "setup": 0.82, - "call": 2.64, - "teardown": 0.01, - "total": 3.47 + "setup": 0.56, + "call": 3.57, + "teardown": 0.0, + "total": 4.13 } }, "tests/aws/services/cloudwatch/test_cloudwatch.py::TestCloudWatchMultiProtocol::test_basic_operations_multiple_protocols[query]": { - "last_validated_date": "2025-10-06T14:45:59+00:00", + "last_validated_date": "2025-12-17T13:53:13+00:00", "durations_in_seconds": { - "setup": 0.01, - "call": 2.39, - "teardown": 0.02, - "total": 2.42 + "setup": 0.0, + "call": 3.0, + "teardown": 0.0, + "total": 3.0 } }, "tests/aws/services/cloudwatch/test_cloudwatch.py::TestCloudWatchMultiProtocol::test_basic_operations_multiple_protocols[smithy-rpc-v2-cbor]": { - "last_validated_date": "2025-10-06T14:45:56+00:00", + "last_validated_date": "2025-12-17T13:53:10+00:00", "durations_in_seconds": { "setup": 0.0, - "call": 2.4, - "teardown": 0.02, - "total": 2.42 + "call": 3.15, + "teardown": 0.0, + "total": 3.15 } }, "tests/aws/services/cloudwatch/test_cloudwatch.py::TestCloudWatchMultiProtocol::test_exception_serializing_with_no_shape_in_spec[json]": { diff --git a/tests/aws/services/cloudwatch/utils.py b/tests/aws/services/cloudwatch/utils.py index f735a33c151f4..85af14e89c517 100644 --- a/tests/aws/services/cloudwatch/utils.py +++ b/tests/aws/services/cloudwatch/utils.py @@ -5,7 +5,9 @@ import xmltodict from botocore.auth import SigV4Auth from botocore.serialize import create_serializer -from cbor2._decoder import loads as cbor2_loads + +# import the unpatched cbor2 on purpose to avoid being polluted by Kinesis-only patches +from cbor2 import loads as cbor2_loads from requests import Response from localstack import constants @@ -45,9 +47,9 @@ def _build_headers(self, operation: str, query_mode: bool = False) -> dict: ... def _serialize_body(self, body: dict, operation: str) -> str | bytes: # here we use the Botocore serializer directly, since it has some complex behavior, # and we know CloudWatch supports it by default - query_serializer = create_serializer(self.protocol) + protocol_serializer = create_serializer(self.protocol) operation_model = self.service_model.operation_model(operation) - request = query_serializer.serialize_to_request(body, operation_model) + request = protocol_serializer.serialize_to_request(body, operation_model) return request["body"] @property diff --git a/tests/unit/aws/protocol/test_parser.py b/tests/unit/aws/protocol/test_parser.py index 6564e844baaf8..2854bc65b2281 100644 --- a/tests/unit/aws/protocol/test_parser.py +++ b/tests/unit/aws/protocol/test_parser.py @@ -1452,7 +1452,6 @@ def test_restxml_ignores_get_body(): def test_smithy_rpc_v2_cbor(): # we are using a service that LocalStack does not implement yet because it implements `smithy-rpc-v2-cbor` # we can replace this service by CloudWatch once it has support in Botocore - # TODO: test timestamp parsing # example taken from: # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/arc-region-switch/client/create_plan.html @@ -1525,6 +1524,23 @@ def test_smithy_rpc_v2_cbor(): ) +def test_rpc_v2_cbor_timestamp_parsing(): + # This is a real request from the Java SDK v2 + # It does not encode Timestamps like Botocore: it encodes them as Double of Length 8 (botocore uses Integer) + request = HttpRequest( + method="POST", + path="/v1/service/GraniteServiceVersion20100801/operation/PutMetricData", + body=b"\xbfiNamespacelSITE/TRAFFICjMetricData\x81\xbfjMetricNamemPAGES_VISITEDjDimensions\x81\xbfdNamelUNIQUE_PAGESeValuedURLS\xffiTimestamp\xc1\xfbA\xdaP\xaf+\x88\xa3\xd7eValue\xfb?\xf3\xbfg\xf4\xdb\xdf\x8fdUnitdNone\xff\xff", + ) + parser = create_parser(load_service("cloudwatch"), protocol="smithy-rpc-v2-cbor") + parsed_operation_model, parsed_request = parser.parse(request) + timestamp = parsed_request["MetricData"][0]["Timestamp"] + assert isinstance(timestamp, datetime) + assert timestamp.microsecond == 135000 + assert timestamp.minute == 38 + assert timestamp.tzinfo == UTC + + @pytest.mark.parametrize("protocol", ("json", "smithy-rpc-v2-cbor")) def test_protocol_selection(protocol): # we are using a service that LocalStack does not implement yet because it implements `smithy-rpc-v2-cbor` From ffd14ad9abf43541c319bb2cc51a08ec287bde54 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Wed, 17 Dec 2025 16:30:25 +0100 Subject: [PATCH 3/3] fix unit test: @('_')@ mistake --- tests/unit/aws/protocol/test_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/aws/protocol/test_parser.py b/tests/unit/aws/protocol/test_parser.py index 2854bc65b2281..4b686dc636e8c 100644 --- a/tests/unit/aws/protocol/test_parser.py +++ b/tests/unit/aws/protocol/test_parser.py @@ -1537,7 +1537,8 @@ def test_rpc_v2_cbor_timestamp_parsing(): timestamp = parsed_request["MetricData"][0]["Timestamp"] assert isinstance(timestamp, datetime) assert timestamp.microsecond == 135000 - assert timestamp.minute == 38 + assert timestamp.second == 38 + assert timestamp.minute == 22 assert timestamp.tzinfo == UTC