Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

Commit 955ad81

Browse files
peter-smith-phdpeter-smith-phd
committed
SQS: Improve update support for CloudFormation handlers.
1 parent 71b218a commit 955ad81

34 files changed

Lines changed: 1410 additions & 414 deletions

‎localstack-core/localstack/services/cloudformation/provider_utils.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,26 @@ def recursive_convert(obj):
275275
return recursive_convert(input_dict)
276276

277277

278+
def resource_tags_to_remove_or_update(
279+
prev_tags: list[dict], new_tags: list[dict]
280+
) -> tuple[list[str], dict[str, str]]:
281+
"""
282+
When updating resources that have tags, we need to determine which tags to remove and which to add/update,
283+
as these are typically done in separate API calls. The format of prev_tags and new_tags is expected to
284+
be [{ "Key": tagName, "Value": tagValue }, ...]. The return value will be a tuple of (tags_to_remove, tags_to_update),
285+
where:
286+
- tags_to_remove is a list of tag keys that are present in prev_tags but not in new_tags.
287+
- tags_to_update is a dict of tags to add or update, with the format: { tagName: tagValue, ... }.
288+
"""
289+
prev_tag_keys = [tag["Key"] for tag in prev_tags]
290+
new_tag_keys = [tag["Key"] for tag in new_tags]
291+
tags_to_remove = list(set(prev_tag_keys) - set(new_tag_keys))
292+
293+
# convert from list of dicts, to a single dict because that's what tag_queue APIs expect.
294+
tags_to_update = {tag["Key"]: tag["Value"] for tag in new_tags}
295+
return (tags_to_remove, tags_to_update)
296+
297+
278298
# LocalStack specific utilities
279299
def get_schema_path(file_path: Path) -> dict:
280300
file_name_base = file_path.name.removesuffix(".py").removesuffix(".py.enc")

‎localstack-core/localstack/services/sqs/constants.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@
3131
QueueAttributeName.QueueArn,
3232
]
3333

34+
#
35+
# If these attributes are set to their default values, they are effectively
36+
# deleted from the queue attributes and not returned in future calls to get_queue_attributes()
37+
#
38+
DELETE_IF_DEFAULT = {"KmsMasterKeyId": "", "KmsDataKeyReusePeriodSeconds": "300"}
39+
3440
INVALID_STANDARD_QUEUE_ATTRIBUTES = [
3541
QueueAttributeName.FifoQueue,
3642
QueueAttributeName.ContentBasedDeduplication,

‎localstack-core/localstack/services/sqs/provider.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1273,7 +1273,11 @@ def set_queue_attributes(
12731273
for k, v in attributes.items():
12741274
if k in sqs_constants.INTERNAL_QUEUE_ATTRIBUTES:
12751275
raise InvalidAttributeName(f"Unknown Attribute {k}.")
1276-
queue.attributes[k] = v
1276+
if k in sqs_constants.DELETE_IF_DEFAULT and v == sqs_constants.DELETE_IF_DEFAULT[k]:
1277+
if k in queue.attributes:
1278+
del queue.attributes[k]
1279+
else:
1280+
queue.attributes[k] = v
12771281

12781282
# Special cases
12791283
if queue.attributes.get(QueueAttributeName.Policy) == "":

‎localstack-core/localstack/services/sqs/resource_providers/aws_sqs_queue.py‎

Lines changed: 54 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,28 @@ class SQSQueueProvider(ResourceProvider[SQSQueueProperties]):
6464
TYPE = "AWS::SQS::Queue" # Autogenerated. Don't change
6565
SCHEMA = util.get_schema_path(Path(__file__)) # Autogenerated. Don't change
6666

67+
# Values used when a property is removed from a template and needs to be set to its default.
68+
# If AWS changes their defaults in the future, our parity tests should break.
69+
DEFAULT_ATTRIBUTE_VALUES = {
70+
"ReceiveMessageWaitTimeSeconds": "0",
71+
"DelaySeconds": "0",
72+
"KmsMasterKeyId": "",
73+
"RedrivePolicy": "",
74+
"MessageRetentionPeriod": "345600",
75+
"MaximumMessageSize": "262144", # Note: CloudFormation sets this to 256KB on update, but 1MB on create
76+
"VisibilityTimeout": "30",
77+
"KmsDataKeyReusePeriodSeconds": "300",
78+
}
79+
80+
# Private method for creating a unique queue name, if none is specified.
81+
def _autogenerated_queue_name(self, request: ResourceRequest[SQSQueueProperties]) -> str:
82+
queue_name = util.generate_default_name(request.stack_name, request.logical_resource_id)
83+
# Note that it's an SQS FIFO queue only if the FifoQueue property is set to boolean True. If it's None
84+
# (property was omitted) or False, or any type of string (e.g. a typo such as "Fasle"), then it's not a FIFO queue.
85+
if request.desired_state.get("FifoQueue") == True: # noqa: E712
86+
queue_name = f"{queue_name[:-5]}.fifo"
87+
return queue_name
88+
6789
def create(
6890
self,
6991
request: ResourceRequest[SQSQueueProperties],
@@ -74,8 +96,6 @@ def create(
7496
Primary identifier fields:
7597
- /properties/QueueUrl
7698
77-
78-
7999
Create-only properties:
80100
- /properties/FifoQueue
81101
- /properties/QueueName
@@ -92,26 +112,13 @@ def create(
92112
- sqs:TagQueue
93113
94114
"""
95-
# TODO: validations
115+
# TODO: validations - what validations are needed?
96116
model = request.desired_state
97117
sqs = request.aws_client_factory.sqs
98118

99-
if model.get("FifoQueue", False):
100-
model["FifoQueue"] = model["FifoQueue"]
101-
102-
queue_name = model.get("QueueName")
103-
if not queue_name:
104-
# TODO: verify patterns here
105-
if model.get("FifoQueue"):
106-
queue_name = util.generate_default_name(
107-
request.stack_name, request.logical_resource_id
108-
)[:-5]
109-
queue_name = f"{queue_name}.fifo"
110-
else:
111-
queue_name = util.generate_default_name(
112-
request.stack_name, request.logical_resource_id
113-
)
114-
model["QueueName"] = queue_name
119+
# if no QueueName is specified, automatically generate one
120+
if not model.get("QueueName"):
121+
model["QueueName"] = self._autogenerated_queue_name(request)
115122

116123
attributes = self._compile_sqs_queue_attributes(model)
117124
result = request.aws_client_factory.sqs.create_queue(
@@ -184,38 +191,30 @@ def update(
184191
"""
185192
sqs = request.aws_client_factory.sqs
186193
model = request.desired_state
194+
prev_model = request.previous_state
187195

188196
assert request.previous_state is not None
189197

190-
should_replace = (
191-
request.desired_state.get("QueueName", request.previous_state["QueueName"])
192-
!= request.previous_state["QueueName"]
193-
) or (
194-
request.desired_state.get("FifoQueue", request.previous_state.get("FifoQueue"))
195-
!= request.previous_state.get("FifoQueue")
198+
queue_url = prev_model["QueueUrl"]
199+
self._populate_missing_attributes_with_defaults(model)
200+
sqs.set_queue_attributes(
201+
QueueUrl=queue_url, Attributes=self._compile_sqs_queue_attributes(model)
196202
)
197203

198-
if not should_replace:
199-
return ProgressEvent(OperationStatus.SUCCESS, resource_model=request.previous_state)
200-
201-
# TODO: copied from the create handler, extract?
202-
if model.get("FifoQueue"):
203-
queue_name = util.generate_default_name(
204-
request.stack_name, request.logical_resource_id
205-
)[:-5]
206-
queue_name = f"{queue_name}.fifo"
207-
else:
208-
queue_name = util.generate_default_name(request.stack_name, request.logical_resource_id)
209-
210-
# replacement (TODO: find out if we should handle this in the provider or outside of it)
211-
# delete old queue
212-
sqs.delete_queue(QueueUrl=request.previous_state["QueueUrl"])
213-
# create new queue (TODO: re-use create logic to make this more robust, e.g. for
214-
# auto-generated queue names)
215-
model["QueueUrl"] = sqs.create_queue(QueueName=queue_name)["QueueUrl"]
216-
model["Arn"] = sqs.get_queue_attributes(
217-
QueueUrl=model["QueueUrl"], AttributeNames=["QueueArn"]
218-
)["Attributes"]["QueueArn"]
204+
(tags_to_remove, tags_to_add_or_update) = util.resource_tags_to_remove_or_update(
205+
prev_model.get("Tags", []), model.get("Tags", [])
206+
)
207+
sqs.untag_queue(QueueUrl=queue_url, TagKeys=tags_to_remove)
208+
sqs.tag_queue(QueueUrl=queue_url, Tags=tags_to_add_or_update)
209+
210+
model["QueueUrl"] = queue_url
211+
model["Arn"] = request.previous_state["Arn"]
212+
213+
# For QueueName and FifoQueue, always use the value from the previous model. These fields
214+
# are create-only, so they cannot be changed via an update (even though they might be omitted)
215+
model["QueueName"] = prev_model.get("QueueName")
216+
model["FifoQueue"] = prev_model.get("FifoQueue", False)
217+
219218
return ProgressEvent(OperationStatus.SUCCESS, resource_model=model)
220219

221220
def _compile_sqs_queue_attributes(self, properties: SQSQueueProperties) -> dict[str, str]:
@@ -250,6 +249,15 @@ def _compile_sqs_queue_attributes(self, properties: SQSQueueProperties) -> dict[
250249

251250
return result
252251

252+
def _populate_missing_attributes_with_defaults(self, properties: SQSQueueProperties) -> None:
253+
"""
254+
For any attribute that is missing from the desired state, populate it with the default value.
255+
This is the only way to remove an attribute from an existing SQS queue's configuration.
256+
:param properties: the properties passed from cloudformation
257+
"""
258+
for k, v in self.DEFAULT_ATTRIBUTE_VALUES.items():
259+
properties.setdefault(k, v)
260+
253261
def list(
254262
self,
255263
request: ResourceRequest[SQSQueueProperties],

‎tests/aws/services/cloudformation/api/test_stacks.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ def test_update_stack_with_same_template_withoutchange_transformation(
340340
)
341341

342342
@markers.aws.validated
343+
@skip_if_legacy_engine()
343344
def test_update_stack_actual_update(self, deploy_cfn_template, aws_client):
344345
template = load_file(
345346
os.path.join(os.path.dirname(__file__), "../../../templates/sqs_queue_update.yml")

‎tests/aws/services/cloudformation/resources/test_sqs.py‎

Lines changed: 0 additions & 143 deletions
This file was deleted.

0 commit comments

Comments
 (0)