diff --git a/CHANGELOG.md b/CHANGELOG.md
index c342ca15..745f8df6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 <!-- insertion marker -->
+## [0.5.4](https://github.com/mkdocstrings/python/releases/tag/0.5.4) - 2022-02-13
+
+<small>[Compare with 0.5.3](https://github.com/mkdocstrings/python/compare/0.5.3...0.5.4)</small>
+
+### Bug Fixes
+- Don't load additional modules during fallback ([69b8e25](https://github.com/mkdocstrings/python/commit/69b8e25cddc9e256c5edb8843592a466023aa124) by Timothée Mazzucotelli).
+
+
 ## [0.5.3](https://github.com/mkdocstrings/python/releases/tag/0.5.3) - 2022-02-08
 
 <small>[Compare with 0.5.2](https://github.com/mkdocstrings/python/compare/0.5.2...0.5.3)</small>
diff --git a/src/mkdocstrings_handlers/python/collector.py b/src/mkdocstrings_handlers/python/collector.py
index f552f90e..42e6a371 100644
--- a/src/mkdocstrings_handlers/python/collector.py
+++ b/src/mkdocstrings_handlers/python/collector.py
@@ -31,6 +31,8 @@ class PythonCollector(BaseCollector):
     **`docstring_options`** | `dict[str, Any]` | The options for the docstring parser. | `{}`
     """
 
+    fallback_config: dict = {"fallback": True}
+
     def __init__(self) -> None:
         """Initialize the object."""
         self._modules_collection: ModulesCollection = ModulesCollection()
@@ -49,15 +51,17 @@ def collect(self, identifier: str, config: dict) -> CollectorItem:  # noqa: WPS2
         Returns:
             The collected object-tree.
         """
+        module_name = identifier.split(".", 1)[0]
+        unknown_module = module_name not in self._modules_collection
+        if config.pop("fallback", False) and unknown_module:
+            raise CollectionError("Not loading additional modules during fallback")
+
         final_config = ChainMap(config, self.default_config)
         parser_name = final_config["docstring_style"]
         parser_options = final_config["docstring_options"]
         parser = parser_name and Parser(parser_name)
 
-        just_loaded = False
-        module_name = identifier.split(".", 1)[0]
-        if module_name not in self._modules_collection:
-            just_loaded = True
+        if unknown_module:
             loader = GriffeLoader(
                 extensions=load_extensions(final_config.get("extensions", [])),
                 docstring_parser=parser,
@@ -79,7 +83,7 @@ def collect(self, identifier: str, config: dict) -> CollectorItem:  # noqa: WPS2
         except KeyError as error:  # noqa: WPS440
             raise CollectionError(f"{identifier} could not be found") from error
 
-        if not just_loaded and doc_object.docstring is not None:
+        if not unknown_module and doc_object.docstring is not None:
             doc_object.docstring.parser = parser
             doc_object.docstring.parser_options = parser_options