From cb1becd968797cda91b541b65f471627cd496194 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Jun 2023 18:29:07 -0500 Subject: [PATCH 1/5] feat: speed up processing incoming records moves async_mark_unique_cached_records_older_than_1s_to_expire since cache has been cythonized and this function alters the cache --- src/zeroconf/_cache.py | 18 ++++++++++++++++-- src/zeroconf/_handlers.py | 19 +++---------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/zeroconf/_cache.py b/src/zeroconf/_cache.py index 49f92f911..bb064aaa5 100644 --- a/src/zeroconf/_cache.py +++ b/src/zeroconf/_cache.py @@ -21,7 +21,7 @@ """ import itertools -from typing import Dict, Iterable, Iterator, List, Optional, Union, cast +from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union, cast from ._dns import ( DNSAddress, @@ -34,7 +34,7 @@ DNSText, ) from ._utils.time import current_time_millis -from .const import _TYPE_PTR +from .const import _ONE_SECOND, _TYPE_PTR _UNIQUE_RECORD_TYPES = (DNSAddress, DNSHinfo, DNSPointer, DNSText, DNSService) _UniqueRecordsType = Union[DNSAddress, DNSHinfo, DNSPointer, DNSText, DNSService] @@ -226,6 +226,20 @@ def names(self) -> List[str]: """Return a copy of the list of current cache names.""" return list(self.cache) + def async_mark_unique_cached_records_older_than_1s_to_expire( + self, unique_types: Set[Tuple[str, int, int]], answers: Iterable[DNSRecord], now: float + ) -> None: + # rfc6762#section-10.2 para 2 + # Since unique is set, all old records with that name, rrtype, + # and rrclass that were received more than one second ago are declared + # invalid, and marked to expire from the cache in one second. + answers_rrset = set(answers) + for name, type_, class_ in unique_types: + for entry in self.async_all_by_details(name, type_, class_): + if (now - entry.created > _ONE_SECOND) and entry not in answers_rrset: + # Expire in 1s + entry.set_created_ttl(now, 1) + def _dns_record_matches(record: _DNSRecord, key: _str, type_: int, class_: int) -> bool: return key == record.key and type_ == record.type and class_ == record.class_ diff --git a/src/zeroconf/_handlers.py b/src/zeroconf/_handlers.py index 38a1b034b..5effaebc2 100644 --- a/src/zeroconf/_handlers.py +++ b/src/zeroconf/_handlers.py @@ -26,7 +26,6 @@ from typing import ( TYPE_CHECKING, Dict, - Iterable, List, NamedTuple, Optional, @@ -421,7 +420,9 @@ def async_updates_from_response(self, msg: DNSIncoming) -> None: removes.add(record) if unique_types: - self._async_mark_unique_cached_records_older_than_1s_to_expire(unique_types, msg.answers, now) + self.cache.async_mark_unique_cached_records_older_than_1s_to_expire( + unique_types, msg.answers, now + ) if updates: self.async_updates(now, updates) @@ -451,20 +452,6 @@ def async_updates_from_response(self, msg: DNSIncoming) -> None: if updates: self.async_updates_complete(new) - def _async_mark_unique_cached_records_older_than_1s_to_expire( - self, unique_types: Set[Tuple[str, int, int]], answers: Iterable[DNSRecord], now: float - ) -> None: - # rfc6762#section-10.2 para 2 - # Since unique is set, all old records with that name, rrtype, - # and rrclass that were received more than one second ago are declared - # invalid, and marked to expire from the cache in one second. - answers_rrset = set(answers) - for name, type_, class_ in unique_types: - for entry in self.cache.async_all_by_details(name, type_, class_): - if (now - entry.created > _ONE_SECOND) and entry not in answers_rrset: - # Expire in 1s - entry.set_created_ttl(now, 1) - def async_add_listener( self, listener: RecordUpdateListener, question: Optional[Union[DNSQuestion, List[DNSQuestion]]] ) -> None: From 6be0960dcec5397eb923beede033c56bc8d05e7f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Jun 2023 18:29:26 -0500 Subject: [PATCH 2/5] feat: speed up processing incoming records moves async_mark_unique_cached_records_older_than_1s_to_expire since cache has been cythonized and this function alters the cache --- src/zeroconf/_cache.py | 2 +- src/zeroconf/_handlers.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/zeroconf/_cache.py b/src/zeroconf/_cache.py index bb064aaa5..47986fe61 100644 --- a/src/zeroconf/_cache.py +++ b/src/zeroconf/_cache.py @@ -226,7 +226,7 @@ def names(self) -> List[str]: """Return a copy of the list of current cache names.""" return list(self.cache) - def async_mark_unique_cached_records_older_than_1s_to_expire( + def async_mark_unique_records_older_than_1s_to_expire( self, unique_types: Set[Tuple[str, int, int]], answers: Iterable[DNSRecord], now: float ) -> None: # rfc6762#section-10.2 para 2 diff --git a/src/zeroconf/_handlers.py b/src/zeroconf/_handlers.py index 5effaebc2..fb5ed7c71 100644 --- a/src/zeroconf/_handlers.py +++ b/src/zeroconf/_handlers.py @@ -420,9 +420,7 @@ def async_updates_from_response(self, msg: DNSIncoming) -> None: removes.add(record) if unique_types: - self.cache.async_mark_unique_cached_records_older_than_1s_to_expire( - unique_types, msg.answers, now - ) + self.cache.async_mark_unique_records_older_than_1s_to_expire(unique_types, msg.answers, now) if updates: self.async_updates(now, updates) From 04e69e32aa037c5e55582caae854ea544efc4904 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Jun 2023 18:54:05 -0500 Subject: [PATCH 3/5] feat: optimize cython code --- src/zeroconf/_cache.pxd | 11 +++++++++++ src/zeroconf/_cache.py | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/zeroconf/_cache.pxd b/src/zeroconf/_cache.pxd index ea436be70..ff8b45b74 100644 --- a/src/zeroconf/_cache.pxd +++ b/src/zeroconf/_cache.pxd @@ -13,6 +13,7 @@ from ._dns cimport ( cdef object _UNIQUE_RECORD_TYPES cdef object _TYPE_PTR +cdef object _ONE_SECOND cdef _remove_key(cython.dict cache, object key, DNSRecord record) @@ -22,9 +23,19 @@ cdef class DNSCache: cdef public cython.dict cache cdef public cython.dict service_cache + @cython.locals( + records=cython.list, + record=DNSRecord, + ) + cdef _async_all_by_details(self, object name, object type_, object class_) + cdef _async_add(self, DNSRecord record) cdef _async_remove(self, DNSRecord record) + @cython.locals( + record=DNSRecord, + ) + cdef _async_mark_unique_records_older_than_1s_to_expire(self, object unique_types, object answers, object now) cdef _dns_record_matches(DNSRecord record, object key, object type_, object class_) diff --git a/src/zeroconf/_cache.py b/src/zeroconf/_cache.py index 47986fe61..f8aec6166 100644 --- a/src/zeroconf/_cache.py +++ b/src/zeroconf/_cache.py @@ -21,7 +21,7 @@ """ import itertools -from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union, cast +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union, cast from ._dns import ( DNSAddress, @@ -41,6 +41,8 @@ _DNSRecordCacheType = Dict[str, Dict[DNSRecord, DNSRecord]] _DNSRecord = DNSRecord _str = str +_float = float +_int = int def _remove_key(cache: _DNSRecordCacheType, key: _str, record: _DNSRecord) -> None: @@ -134,19 +136,29 @@ def async_get_unique(self, entry: _UniqueRecordsType) -> Optional[DNSRecord]: return None return store.get(entry) - def async_all_by_details(self, name: _str, type_: int, class_: int) -> Iterator[DNSRecord]: + def async_all_by_details(self, name: _str, type_: int, class_: int) -> Iterable[DNSRecord]: """Gets all matching entries by details. - This function is not threadsafe and must be called from + This function is not thread-safe and must be called from + the event loop. + """ + return self._async_all_by_details(name, type_, class_) + + def _async_all_by_details(self, name: _str, type_: int, class_: int) -> list[DNSRecord]: + """Gets all matching entries by details. + + This function is not thread-safe and must be called from the event loop. """ key = name.lower() records = self.cache.get(key) + matches: List[DNSRecord] = [] if records is None: - return - for entry in records: - if _dns_record_matches(entry, key, type_, class_): - yield entry + return matches + for record in records: + if _dns_record_matches(record, key, type_, class_): + matches.append(record) + return matches def async_entries_with_name(self, name: str) -> Dict[DNSRecord, DNSRecord]: """Returns a dict of entries whose key matches the name. @@ -227,7 +239,12 @@ def names(self) -> List[str]: return list(self.cache) def async_mark_unique_records_older_than_1s_to_expire( - self, unique_types: Set[Tuple[str, int, int]], answers: Iterable[DNSRecord], now: float + self, unique_types: Set[Tuple[_str, _int, _int]], answers: Iterable[DNSRecord], now: _float + ) -> None: + self._async_mark_unique_records_older_than_1s_to_expire(unique_types, answers, now) + + def _async_mark_unique_records_older_than_1s_to_expire( + self, unique_types: Set[Tuple[_str, _int, _int]], answers: Iterable[DNSRecord], now: _float ) -> None: # rfc6762#section-10.2 para 2 # Since unique is set, all old records with that name, rrtype, @@ -235,10 +252,10 @@ def async_mark_unique_records_older_than_1s_to_expire( # invalid, and marked to expire from the cache in one second. answers_rrset = set(answers) for name, type_, class_ in unique_types: - for entry in self.async_all_by_details(name, type_, class_): - if (now - entry.created > _ONE_SECOND) and entry not in answers_rrset: + for record in self._async_all_by_details(name, type_, class_): + if (now - record.created > _ONE_SECOND) and record not in answers_rrset: # Expire in 1s - entry.set_created_ttl(now, 1) + record.set_created_ttl(now, 1) def _dns_record_matches(record: _DNSRecord, key: _str, type_: int, class_: int) -> bool: From 116453b3707b8368ce1bd07bde01d7701fd2b90d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Jun 2023 18:55:33 -0500 Subject: [PATCH 4/5] fix: legacy typing --- src/zeroconf/_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zeroconf/_cache.py b/src/zeroconf/_cache.py index f8aec6166..505143b3f 100644 --- a/src/zeroconf/_cache.py +++ b/src/zeroconf/_cache.py @@ -144,7 +144,7 @@ def async_all_by_details(self, name: _str, type_: int, class_: int) -> Iterable[ """ return self._async_all_by_details(name, type_, class_) - def _async_all_by_details(self, name: _str, type_: int, class_: int) -> list[DNSRecord]: + def _async_all_by_details(self, name: _str, type_: int, class_: int) -> List[DNSRecord]: """Gets all matching entries by details. This function is not thread-safe and must be called from From da09a6688d15be783af910fd60f2009ace9b994d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Jun 2023 19:01:44 -0500 Subject: [PATCH 5/5] fix: incorrect type fix --- src/zeroconf/_cache.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zeroconf/_cache.pxd b/src/zeroconf/_cache.pxd index ff8b45b74..07eeb8079 100644 --- a/src/zeroconf/_cache.pxd +++ b/src/zeroconf/_cache.pxd @@ -24,7 +24,7 @@ cdef class DNSCache: cdef public cython.dict service_cache @cython.locals( - records=cython.list, + records=cython.dict, record=DNSRecord, ) cdef _async_all_by_details(self, object name, object type_, object class_)