From 6561ad5e1afa272c7cc0d53e33150fb75fe72e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 14 Nov 2024 13:32:17 +0100 Subject: [PATCH 01/39] chore: Fix docs-deploy duty to deploy to org-pages remote, not upstream --- duties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/duties.py b/duties.py index 0f283217..eae95cc1 100644 --- a/duties.py +++ b/duties.py @@ -137,12 +137,12 @@ def docs_deploy(ctx: Context, *, force: bool = False) -> None: allow_overrides=False, ) ctx.run( - tools.mkdocs.gh_deploy(remote_name="upstream", force=True), + tools.mkdocs.gh_deploy(remote_name="org-pages", force=True), title="Deploying documentation", ) elif force: ctx.run( - tools.mkdocs.gh_deploy(force=True), + tools.mkdocs.gh_deploy(remote_name="org-pages", force=True), title="Deploying documentation", ) else: From 12c8f82e9a959ce32cada09f0d2b5c651a705fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Mon, 25 Nov 2024 15:08:12 +0100 Subject: [PATCH 02/39] fix: Fix broken table of contents when nesting autodoc instructions Issue-348: https://github.com/mkdocstrings/mkdocstrings/issues/348 --- src/mkdocstrings/extension.py | 101 +++++++++++++++++++----------- src/mkdocstrings/handlers/base.py | 17 +++++ tests/fixtures/nesting.py | 10 +++ tests/test_handlers.py | 41 ++++++++++++ 4 files changed, 131 insertions(+), 38 deletions(-) create mode 100644 tests/fixtures/nesting.py diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 266d642f..bd20b48e 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -131,44 +131,9 @@ def run(self, parent: Element, blocks: MutableSequence[str]) -> None: el = Element("div", {"class": "mkdocstrings"}) # The final HTML is inserted as opaque to subsequent processing, and only revealed at the end. el.text = self.md.htmlStash.store(html) - # We need to duplicate the headings directly, just so 'toc' can pick them up, - # otherwise they wouldn't appear in the final table of contents. - # These headings are generated by the `BaseHandler.do_heading` method (Jinja filter), - # which runs in the inner Markdown conversion layer, and not in the outer one where we are now. - headings = handler.get_headings() - el.extend(headings) - # These duplicated headings will later be removed by our `_HeadingsPostProcessor` processor, - # which runs right after 'toc' (see `MkdocstringsExtension.extendMarkdown`). - - page = self._autorefs.current_page - if page is not None: - for heading in headings: - rendered_anchor = heading.attrib["id"] - self._autorefs.register_anchor(page, rendered_anchor) - - if "data-role" in heading.attrib: - self._handlers.inventory.register( - name=rendered_anchor, - domain=handler.domain, - role=heading.attrib["data-role"], - priority=1, # register with standard priority - uri=f"{page}#{rendered_anchor}", - ) - - # also register other anchors for this object in the inventory - try: - data_object = handler.collect(rendered_anchor, handler.fallback_config) - except CollectionError: - continue - for anchor in handler.get_anchors(data_object): - if anchor not in self._handlers.inventory: - self._handlers.inventory.register( - name=anchor, - domain=handler.domain, - role=heading.attrib["data-role"], - priority=2, # register with lower priority - uri=f"{page}#{rendered_anchor}", - ) + + if handler.outer_layer: + self._process_headings(handler, el) parent.append(el) @@ -240,6 +205,66 @@ def _process_block( return rendered, handler, data + def _process_headings(self, handler: BaseHandler, element: Element) -> None: + # We're in the outer handler layer, as well as the outer extension layer. + # + # The "handler layer" tracks the nesting of the autodoc blocks, which can appear in docstrings. + # + # - Render ::: Object1 # Outer handler layer + # - Render Object1's docstring # Outer handler layer + # - Docstring renders ::: Object2 # Inner handler layers + # - etc. # Inner handler layers + # + # The "extension layer" tracks whether we're converting an autodoc instruction + # or nested content within it, like docstrings. Markdown conversion within Markdown conversion. + # + # - Render ::: Object1 # Outer extension layer + # - Render Object1's docstring # Inner extension layer + # + # The generated HTML was just stashed, and the `toc` extension won't be able to see headings. + # We need to duplicate the headings directly, just so `toc` can pick them up, + # otherwise they wouldn't appear in the final table of contents. + # + # These headings are generated by the `BaseHandler.do_heading` method (Jinja filter), + # which runs in the inner extension layer, and not in the outer one where we are now. + headings = handler.get_headings() + element.extend(headings) + # These duplicated headings will later be removed by our `_HeadingsPostProcessor` processor, + # which runs right after `toc` (see `MkdocstringsExtension.extendMarkdown`). + + # If we were in an inner handler layer, we wouldn't do any of this + # and would just let headings bubble up to the outer handler layer. + + page = self._autorefs.current_page + if page is not None: + for heading in headings: + rendered_anchor = heading.attrib["id"] + self._autorefs.register_anchor(page, rendered_anchor) + + if "data-role" in heading.attrib: + self._handlers.inventory.register( + name=rendered_anchor, + domain=handler.domain, + role=heading.attrib["data-role"], + priority=1, # Register with standard priority. + uri=f"{page}#{rendered_anchor}", + ) + + # Also register other anchors for this object in the inventory. + try: + data_object = handler.collect(rendered_anchor, handler.fallback_config) + except CollectionError: + continue + for anchor in handler.get_anchors(data_object): + if anchor not in self._handlers.inventory: + self._handlers.inventory.register( + name=anchor, + domain=handler.domain, + role=heading.attrib["data-role"], + priority=2, # Register with lower priority. + uri=f"{page}#{rendered_anchor}", + ) + class _HeadingsPostProcessor(Treeprocessor): def run(self, root: Element) -> None: diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index d0b9456a..77e10388 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -41,6 +41,15 @@ CollectorItem = Any +# Autodoc instructions can appear in nested Markdown, +# so we need to keep track of the Markdown conversion layer we're in. +# Since any handler can be called from any Markdown conversion layer, +# we need to keep track of the layer globally. +# This global variable is incremented/decremented in `do_convert_markdown`, +# and used in `outer_layer`. +_markdown_conversion_layer: int = 0 + + class CollectionError(Exception): """An exception raised when some collection of data failed.""" @@ -252,6 +261,11 @@ def get_anchors(self, data: CollectorItem) -> tuple[str, ...]: # noqa: ARG002 """ return () + @property + def outer_layer(self) -> bool: + """Whether we're in the outer Markdown conversion layer.""" + return _markdown_conversion_layer == 0 + def do_convert_markdown( self, text: str, @@ -272,6 +286,8 @@ def do_convert_markdown( Returns: An HTML string. """ + global _markdown_conversion_layer # noqa: PLW0603 + _markdown_conversion_layer += 1 treeprocessors = self._md.treeprocessors treeprocessors[HeadingShiftingTreeprocessor.name].shift_by = heading_level # type: ignore[attr-defined] treeprocessors[IdPrependingTreeprocessor.name].id_prefix = html_id and html_id + "--" # type: ignore[attr-defined] @@ -288,6 +304,7 @@ def do_convert_markdown( treeprocessors[ParagraphStrippingTreeprocessor.name].strip = False # type: ignore[attr-defined] self._md.inlinePatterns[AutorefsInlineProcessor.name].hook = None # type: ignore[attr-defined] self._md.reset() + _markdown_conversion_layer -= 1 def do_heading( self, diff --git a/tests/fixtures/nesting.py b/tests/fixtures/nesting.py new file mode 100644 index 00000000..92f7a9ee --- /dev/null +++ b/tests/fixtures/nesting.py @@ -0,0 +1,10 @@ +class Class: + """A class. + + ## ::: tests.fixtures.nesting.Class.method + options: + show_root_heading: true + """ + + def method(self) -> None: + """A method.""" diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 4a07e98b..cea80657 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -2,6 +2,7 @@ from __future__ import annotations +from textwrap import dedent from typing import TYPE_CHECKING import pytest @@ -94,3 +95,43 @@ def test_extended_templates(tmp_path: Path, plugin: MkdocstringsPlugin) -> None: base_theme.mkdir() base_theme.joinpath("new.html").write_text("base new") assert handler.env.get_template("new.html").render() == "base new" + + +@pytest.mark.parametrize( + "ext_markdown", + [{"markdown_extensions": [{"toc": {"permalink": True}}]}], + indirect=["ext_markdown"], +) +def test_nested_autodoc(ext_markdown: Markdown) -> None: + """Assert that nested autodocs render well and do not mess up the TOC.""" + output = ext_markdown.convert( + dedent( + """ + # ::: tests.fixtures.nesting.Class + options: + members: false + show_root_heading: true + """, + ), + ) + assert 'id="tests.fixtures.nesting.Class"' in output + assert 'id="tests.fixtures.nesting.Class.method"' in output + assert ext_markdown.toc_tokens == [ # type: ignore[attr-defined] + { + "level": 1, + "id": "tests.fixtures.nesting.Class", + "html": "", + "name": "Class", + "data-toc-label": "Class", + "children": [ + { + "level": 2, + "id": "tests.fixtures.nesting.Class.method", + "html": "", + "name": "method", + "data-toc-label": "method", + "children": [], + }, + ], + }, + ] From c7b27fbefaf0e98b4b9d20bfa3ab86c35d756fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 17:23:51 +0100 Subject: [PATCH 03/39] docs: Add workaround to hide docstrings from source code blocks Issue-249: https://github.com/mkdocstrings/mkdocstrings/issues/249 --- docs/recipes.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/recipes.md b/docs/recipes.md index cb2d9eb1..72613965 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -405,3 +405,24 @@ Try to select the following code block's text: ... print(word, end=" ") Hello mkdocstrings! ``` + +## Hide documentation strings from source code blocks + +Since documentation strings are rendered by handlers, it can sometimes feel redundant to show these same documentation strings in source code blocks (when handlers render those). + +There is a general workaround to hide these docstrings from source blocks using CSS: + +```css +/* These CSS classes depend on the handler. */ +.doc-contents details .highlight code { + line-height: 0; +} +.doc-contents details .highlight code > * { + line-height: initial; +} +.doc-contents details .highlight code > .sd { /* Literal.String.Doc */ + display: none; +} +``` + +Note that this is considered a workaround and not a proper solution, because it has side-effects like also removing blank lines. From 326cccde2199adedeb9d720e8dd6fef1a958a3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 19:18:49 +0100 Subject: [PATCH 04/39] docs: Hint at default language for syntax highlight of indented code blocks Issue-mkdocstrings/python#187: https://github.com/mkdocstrings/python/issues/187 --- docs/recipes.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/recipes.md b/docs/recipes.md index 72613965..1c5c6823 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -426,3 +426,29 @@ There is a general workaround to hide these docstrings from source blocks using ``` Note that this is considered a workaround and not a proper solution, because it has side-effects like also removing blank lines. + +## Automatic highlighting for indented code blocks in docstrings + +Depending on the language used in your code base and the mkdocstrings handler used to document it, you might want to set a default syntax for code blocks added to your docstrings. For example, to default to the Python syntax: + +```yaml title="mkdocs.yml" +markdown_extensions: +- pymdownx.highlight: + default_lang: python +``` + +Then in your docstrings, indented code blocks will be highlighted as Python code: + +``` +def my_function(): + """This is my function. + + The following code will be highlighted as Python: + + result = my_function() + print(result) + + End of the docstring. + """ + ... +``` From a17fc0316111c080c7e22b0a69b9f38e8641a785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 19:26:45 +0100 Subject: [PATCH 05/39] docs: Document limitation about footnotes in Python docstrings Issue-mkdocstrings/python#199: https://github.com/mkdocstrings/python/issues/199 --- docs/troubleshooting.md | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index bc1da01b..66e93622 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -238,5 +238,56 @@ def my_function(*args, **kwargs): print(*args, **kwargs) ``` +## Footnotes do not render + +The library that parses docstrings, [Griffe](https://mkdocstrings.github.io/griffe/), splits docstrings in several "sections" (example: [Google-style sections syntax](https://mkdocstrings.github.io/griffe/reference/docstrings/#google-syntax)). If a footnote is used in a section, while referenced in another, mkdocstrings won't be able to render it correctly. The footnote and its reference must appear in the same section. + +```python +def my_function(): + """Summary. + + This is the first section[^1]. + + Note: + This is the second section[^2]. + + Note: + This is the third section[^3]. + + References at the end are part of yet another section (fourth here)[^4]. + + [^1]: Some text. + [^2]: Some text. + [^3]: Some text. + [^4]: Some text. + """ +``` + +Here only the fourth footnote will work, because it is the only one that appear in the same section as its reference. To fix this, make sure all footnotes appear in the same section as their references: + +```python +def my_function(): + """Summary. + + This is the first section[^1]. + + [^1]: Some text. + + Note: + This is the second section[^2]. + + [^2]: Some text. + + Note: + This is the third section[^3]. + + [^3]: Some text. + + References at the end are part of yet another section (fourth here)[^4]. + + [^4]: Some text. + """ +``` + [bugtracker]: https://github.com/mkdocstrings/mkdocstrings [markdown-katex]: https://gitlab.com/mbarkhau/markdown-katex From b4a70f041b927b3cee30f8524876ce445fa98699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 19:39:51 +0100 Subject: [PATCH 06/39] docs: Mention mkdocs-autoapi Issue-mkdocstrings/python#199: https://github.com/mkdocstrings/python/issues/199 --- docs/recipes.md | 2 ++ mkdocs.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/recipes.md b/docs/recipes.md index 1c5c6823..7d0ab37d 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -3,6 +3,8 @@ for *mkdocstrings* and more generally Markdown documentation. ## Automatic code reference pages +TIP: **[mkdocs-autoapi](https://github.com/jcayers20/mkdocs-autoapi) is a MkDocs plugin that automatically generates API documentation from your project's source code. It was inspired by the recipe below.** + *mkdocstrings* allows to inject documentation for any object into Markdown pages. But as the project grows, it quickly becomes quite tedious to keep the autodoc instructions, or even the dedicated diff --git a/mkdocs.yml b/mkdocs.yml index 3b4fefb2..610b224f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -100,7 +100,8 @@ extra_javascript: markdown_extensions: - attr_list - admonition -- callouts +- callouts: + strip_period: false - footnotes - pymdownx.details - pymdownx.emoji: From 9ca7ddeecff44b36df16b107d895ff994c636f9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 19:40:05 +0100 Subject: [PATCH 07/39] chore: Update code block in docs --- docs/recipes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index 7d0ab37d..6a81b090 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -441,7 +441,7 @@ markdown_extensions: Then in your docstrings, indented code blocks will be highlighted as Python code: -``` +```python def my_function(): """This is my function. @@ -452,5 +452,5 @@ def my_function(): End of the docstring. """ - ... + pass ``` From 15abd977f88c26f17a0c1f3f65677a536ff4dab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 20:11:50 +0100 Subject: [PATCH 08/39] docs: Fix heading level --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 66e93622..2027a094 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -238,7 +238,7 @@ def my_function(*args, **kwargs): print(*args, **kwargs) ``` -## Footnotes do not render +### Footnotes do not render The library that parses docstrings, [Griffe](https://mkdocstrings.github.io/griffe/), splits docstrings in several "sections" (example: [Google-style sections syntax](https://mkdocstrings.github.io/griffe/reference/docstrings/#google-syntax)). If a footnote is used in a section, while referenced in another, mkdocstrings won't be able to render it correctly. The footnote and its reference must appear in the same section. From 9fb89d829b8dc9f7928e27e16227cec6b7e71488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 3 Dec 2024 20:12:16 +0100 Subject: [PATCH 09/39] docs: Add troubleshooting note about `show_submodules` Issue-mkdocstrings/python#199: https://github.com/mkdocstrings/python/issues/199 --- docs/troubleshooting.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2027a094..7548eafe 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -289,5 +289,18 @@ def my_function(): """ ``` +### Submodules are not rendered + +In previous versions of mkdocstrings-python, submodules were rendered by default. This was changed and you now need to set the following option: + +```yaml title="mkdocs.yml" +plugins: +- mkdocstrings: + handlers: + python: + options: + show_submodules: true +``` + [bugtracker]: https://github.com/mkdocstrings/mkdocstrings [markdown-katex]: https://gitlab.com/mbarkhau/markdown-katex From 2f6ddbe379ea8f1eabad1527f2e8e620c3b973d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 5 Dec 2024 16:05:48 +0100 Subject: [PATCH 10/39] docs: Add items to "Used by" README section --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e9f9fbb2..8d4b8bb0 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,13 @@ Come have a chat or ask questions on our [Gitter channel](https://gitter.im/mkdo [Apache](https://streampipes.apache.org/docs/docs/python/latest/reference/client/client/), [FastAPI](https://fastapi.tiangolo.com/reference/fastapi/), [Google](https://docs.kidger.site/jaxtyping/api/runtime-type-checking/), +[IBM](https://ds4sd.github.io/docling/api_reference/document_converter/), [Jitsi](https://jitsi.github.io/jiwer/reference/alignment/), [Microsoft](https://microsoft.github.io/presidio/api/analyzer_python/), +[NVIDIA](https://nvidia.github.io/bionemo-framework/API_reference/bionemo/core/api/), [Prefect](https://docs.prefect.io/2.10.12/api-ref/prefect/agent/), [Pydantic](https://docs.pydantic.dev/dev-v2/api/main/), +[Textual](https://textual.textualize.io/api/app/), [and more...](https://github.com/mkdocstrings/mkdocstrings/network/dependents) ## Installation From b8e87036e0e1ec5c181b4a2ec5931f1a60636a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Jan 2025 01:20:36 +0100 Subject: [PATCH 11/39] refactor: Clean up data passed down from plugin to extension and handlers These changes straighten up what arguments are passed to the extension, the `Handlers` class, and the handlers' `get_handler` functions. Previously there was a lot of redundancy, building and passing mutable dictionaries around. Now we pass exactly what each class/function needs, as well as the global MkDocs config (renamed "tool config" to decouple ourselves a bit from MkDocs) in case users need it (`get_handler` functions and `update_env` methods, mainly). These changes bring a few deprecations with relevant messages, as well as a few breaking changes in mkdocstrings' API for which we didn't identify any public use on GitHub. PR-726: https://github.com/mkdocstrings/mkdocstrings/pull/726 --- config/pytest.ini | 11 +- src/mkdocstrings/extension.py | 21 +-- src/mkdocstrings/handlers/base.py | 254 ++++++++++++++++++++++-------- src/mkdocstrings/plugin.py | 29 ++-- 4 files changed, 219 insertions(+), 96 deletions(-) diff --git a/config/pytest.ini b/config/pytest.ini index 1a0d99c6..eb7af02a 100644 --- a/config/pytest.ini +++ b/config/pytest.ini @@ -12,8 +12,9 @@ filterwarnings = error # TODO: remove once pytest-xdist 4 is released ignore:.*rsyncdir:DeprecationWarning:xdist - # TODO: remove once griffe and mkdocstrings-python release new versions - ignore:.*`get_logger`:DeprecationWarning:_griffe - ignore:.*`name`:DeprecationWarning:_griffe - ignore:.*Importing from `griffe:DeprecationWarning:mkdocstrings_handlers - ignore:.*`patch_loggers`:DeprecationWarning:_griffe + # TODO: remove once mkdocstrings-python releases a new version + ignore:.*`handler` argument:DeprecationWarning:mkdocstrings_handlers + ignore:.*`mdx` argument:DeprecationWarning:mkdocstrings_handlers + ignore:.*`mdx_config` argument:DeprecationWarning:mkdocstrings_handlers + ignore:.*`update_env\(md\)` parameter:DeprecationWarning:mkdocstrings + ignore:.*`super\(\).update_env\(\)` anymore:DeprecationWarning:mkdocstrings_handlers diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index bd20b48e..b9fc0a79 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -42,7 +42,6 @@ from collections.abc import MutableSequence from markdown import Markdown - from markdown.blockparser import BlockParser from mkdocs_autorefs.plugin import AutorefsPlugin @@ -63,24 +62,20 @@ class AutoDocProcessor(BlockProcessor): def __init__( self, - parser: BlockParser, md: Markdown, - config: dict, + *, handlers: Handlers, autorefs: AutorefsPlugin, ) -> None: """Initialize the object. Arguments: - parser: A `markdown.blockparser.BlockParser` instance. md: A `markdown.Markdown` instance. - config: The [configuration][mkdocstrings.plugin.PluginConfig] of the `mkdocstrings` plugin. handlers: The handlers container. autorefs: The autorefs plugin instance. """ - super().__init__(parser=parser) + super().__init__(parser=md.parser) self.md = md - self._config = config self._handlers = handlers self._autorefs = autorefs self._updated_envs: set = set() @@ -187,19 +182,18 @@ def _process_block( if handler_name not in self._updated_envs: # We haven't seen this handler before on this document. log.debug("Updating handler's rendering env") - handler._update_env(self.md, self._config) + handler._update_env(self.md, config=self._handlers._tool_config) self._updated_envs.add(handler_name) log.debug("Rendering templates") try: rendered = handler.render(data, options) except TemplateNotFound as exc: - theme_name = self._config["theme_name"] log.error( # noqa: TRY400 "Template '%s' not found for '%s' handler and theme '%s'.", exc.name, handler_name, - theme_name, + self._handlers._theme, ) raise @@ -304,18 +298,15 @@ class MkdocstringsExtension(Extension): It cannot work outside of `mkdocstrings`. """ - def __init__(self, config: dict, handlers: Handlers, autorefs: AutorefsPlugin, **kwargs: Any) -> None: + def __init__(self, handlers: Handlers, autorefs: AutorefsPlugin, **kwargs: Any) -> None: """Initialize the object. Arguments: - config: The configuration items from `mkdocs` and `mkdocstrings` that must be passed to the block processor - when instantiated in [`extendMarkdown`][mkdocstrings.extension.MkdocstringsExtension.extendMarkdown]. handlers: The handlers container. autorefs: The autorefs plugin instance. **kwargs: Keyword arguments used by `markdown.extensions.Extension`. """ super().__init__(**kwargs) - self._config = config self._handlers = handlers self._autorefs = autorefs @@ -328,7 +319,7 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me md: A `markdown.Markdown` instance. """ md.parser.blockprocessors.register( - AutoDocProcessor(md.parser, md, self._config, self._handlers, self._autorefs), + AutoDocProcessor(md, handlers=self._handlers, autorefs=self._autorefs), "mkdocstrings", priority=75, # Right before markdown.blockprocessors.HashHeaderProcessor ) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 77e10388..427e5af9 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -6,9 +6,11 @@ from __future__ import annotations import importlib +import inspect import sys from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, cast +from warnings import warn from xml.etree.ElementTree import Element, tostring from jinja2 import Environment, FileSystemLoader @@ -34,11 +36,14 @@ from importlib.metadata import entry_points if TYPE_CHECKING: - from collections.abc import Iterable, Iterator, Mapping, MutableMapping, Sequence + from collections.abc import Iterable, Iterator, Mapping, Sequence + from markdown import Extension from mkdocs_autorefs.references import AutorefsHookInterface + CollectorItem = Any +HandlerConfig = Any # Autodoc instructions can appear in nested Markdown, @@ -89,44 +94,122 @@ class BaseHandler: To add custom CSS, add an `extra_css` variable or create an 'style.css' file beside the templates. """ - # TODO: Make name mandatory? - name: str = "" + # YORE: Bump 1: Replace ` = ""` with `` within line. + name: ClassVar[str] = "" """The handler's name, for example "python".""" - domain: str = "default" + + # YORE: Bump 1: Replace ` = ""` with `` within line. + domain: ClassVar[str] = "" """The handler's domain, used to register objects in the inventory, for example "py".""" - enable_inventory: bool = False + + enable_inventory: ClassVar[bool] = False """Whether the inventory creation is enabled.""" + fallback_config: ClassVar[dict] = {} """Fallback configuration when searching anchors for identifiers.""" - fallback_theme: str = "" + + fallback_theme: ClassVar[str] = "" """Fallback theme to use when a template isn't found in the configured theme.""" - extra_css = "" + + extra_css: str = "" """Extra CSS.""" - def __init__(self, handler: str, theme: str, custom_templates: str | None = None) -> None: + def __init__( + self, + # YORE: Bump 1: Remove line. + *args: Any, + # YORE: Bump 1: Remove line. + **kwargs: Any, + # YORE: Bump 1: Replace `# ` with `` within block. + # *, + # theme: str, + # custom_templates: str | None, + # mdx: Sequence[str | Extension], + # mdx_config: Mapping[str, Any], + ) -> None: """Initialize the object. If the given theme is not supported (it does not exist), it will look for a `fallback_theme` attribute in `self` to use as a fallback theme. - Arguments: - handler: The name of the handler. - theme: The name of theme to use. - custom_templates: Directory containing custom templates. + Keyword Arguments: + theme (str): The theme to use. + custom_templates (str | None): The path to custom templates. + mdx (list[str | Extension]): A list of Markdown extensions to use. + mdx_config (Mapping[str, Mapping[str, Any]]): Configuration for the Markdown extensions. """ + # YORE: Bump 1: Remove block. + handler = "" + theme = "" + custom_templates = None + if args: + handler, args = args[0], args[1:] + if args: + theme, args = args[0], args[1:] + warn( + "The `theme` argument must be passed as a keyword argument.", + DeprecationWarning, + stacklevel=2, + ) + if args: + custom_templates, args = args[0], args[1:] + warn( + "The `custom_templates` argument must be passed as a keyword argument.", + DeprecationWarning, + stacklevel=2, + ) + handler = kwargs.pop("handler", handler) + theme = kwargs.pop("theme", theme) + custom_templates = kwargs.pop("custom_templates", custom_templates) + mdx = kwargs.pop("mdx", None) + mdx_config = kwargs.pop("mdx_config", None) + if handler: + if not self.name: + type(self).name = handler + warn( + "The `handler` argument is deprecated. The handler name must be specified as a class attribute.", + DeprecationWarning, + stacklevel=2, + ) + if not self.domain: + warn( + "The `domain` attribute must be specified as a class attribute.", + DeprecationWarning, + stacklevel=2, + ) + if mdx is None: + warn( + "The `mdx` argument must be provided (as a keyword argument).", + DeprecationWarning, + stacklevel=2, + ) + if mdx_config is None: + warn( + "The `mdx_config` argument must be provided (as a keyword argument).", + DeprecationWarning, + stacklevel=2, + ) + + self.theme = theme + self.custom_templates = custom_templates + self.mdx = mdx + self.mdx_config = mdx_config + self._md: Markdown | None = None + self._headings: list[Element] = [] + paths = [] # add selected theme templates - themes_dir = self.get_templates_dir(handler) - paths.append(themes_dir / theme) + themes_dir = self.get_templates_dir(self.name) + paths.append(themes_dir / self.theme) # add extended theme templates - extended_templates_dirs = self.get_extended_templates_dirs(handler) + extended_templates_dirs = self.get_extended_templates_dirs(self.name) for templates_dir in extended_templates_dirs: - paths.append(templates_dir / theme) + paths.append(templates_dir / self.theme) # add fallback theme templates - if self.fallback_theme and self.fallback_theme != theme: + if self.fallback_theme and self.fallback_theme != self.theme: paths.append(themes_dir / self.fallback_theme) # add fallback theme of extended templates @@ -139,8 +222,8 @@ def __init__(self, handler: str, theme: str, custom_templates: str | None = None self.extra_css += "\n" + css_path.read_text(encoding="utf-8") break - if custom_templates is not None: - paths.insert(0, Path(custom_templates) / handler / theme) + if self.custom_templates is not None: + paths.insert(0, Path(self.custom_templates) / self.name / self.theme) self.env = Environment( autoescape=True, @@ -150,8 +233,16 @@ def __init__(self, handler: str, theme: str, custom_templates: str | None = None self.env.filters["any"] = do_any self.env.globals["log"] = get_template_logger(self.name) - self._headings: list[Element] = [] - self._md: Markdown = None # type: ignore[assignment] # To be populated in `update_env`. + @property + def md(self) -> Markdown: + """The Markdown instance. + + Raises: + RuntimeError: When the Markdown instance is not set yet. + """ + if self._md is None: + raise RuntimeError("Markdown instance not set yet") + return self._md @classmethod def load_inventory( @@ -174,7 +265,7 @@ def load_inventory( """ yield from () - def collect(self, identifier: str, config: MutableMapping[str, Any]) -> CollectorItem: + def collect(self, identifier: str, config: Mapping[str, Any]) -> CollectorItem: """Collect data given an identifier and user configuration. In the implementation, you typically call a subprocess that returns JSON, and load that JSON again into @@ -288,22 +379,22 @@ def do_convert_markdown( """ global _markdown_conversion_layer # noqa: PLW0603 _markdown_conversion_layer += 1 - treeprocessors = self._md.treeprocessors + treeprocessors = self.md.treeprocessors treeprocessors[HeadingShiftingTreeprocessor.name].shift_by = heading_level # type: ignore[attr-defined] treeprocessors[IdPrependingTreeprocessor.name].id_prefix = html_id and html_id + "--" # type: ignore[attr-defined] treeprocessors[ParagraphStrippingTreeprocessor.name].strip = strip_paragraph # type: ignore[attr-defined] if autoref_hook: - self._md.inlinePatterns[AutorefsInlineProcessor.name].hook = autoref_hook # type: ignore[attr-defined] + self.md.inlinePatterns[AutorefsInlineProcessor.name].hook = autoref_hook # type: ignore[attr-defined] try: - return Markup(self._md.convert(text)) + return Markup(self.md.convert(text)) finally: treeprocessors[HeadingShiftingTreeprocessor.name].shift_by = 0 # type: ignore[attr-defined] treeprocessors[IdPrependingTreeprocessor.name].id_prefix = "" # type: ignore[attr-defined] treeprocessors[ParagraphStrippingTreeprocessor.name].strip = False # type: ignore[attr-defined] - self._md.inlinePatterns[AutorefsInlineProcessor.name].hook = None # type: ignore[attr-defined] - self._md.reset() + self.md.inlinePatterns[AutorefsInlineProcessor.name].hook = None # type: ignore[attr-defined] + self.md.reset() _markdown_conversion_layer -= 1 def do_heading( @@ -355,7 +446,7 @@ def do_heading( el = Element(f"h{heading_level}", attributes) el.append(Element("mkdocstrings-placeholder")) # Tell the inner 'toc' extension to make its additions if configured so. - toc = cast(TocTreeprocessor, self._md.treeprocessors["toc"]) + toc = cast(TocTreeprocessor, self.md.treeprocessors["toc"]) if toc.use_anchors: toc.add_anchor(el, attributes["id"]) if toc.use_permalinks: @@ -381,29 +472,45 @@ def get_headings(self) -> Sequence[Element]: self._headings.clear() return result - def update_env(self, md: Markdown, config: dict) -> None: # noqa: ARG002 - """Update the Jinja environment. + # YORE: Bump 1: Replace `*args: Any, **kwargs: Any` with `config: Any`. + def update_env(self, *args: Any, **kwargs: Any) -> None: # noqa: ARG002 + """Update the Jinja environment.""" + # YORE: Bump 1: Remove line. + warn("No need to call `super().update_env()` anymore.", DeprecationWarning, stacklevel=2) - Arguments: - md: The Markdown instance. Useful to add functions able to convert Markdown into the environment filters. - config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code - of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary. - """ - self._md = md - self.env.filters["highlight"] = Highlighter(md).highlight - self.env.filters["convert_markdown"] = self.do_convert_markdown - self.env.filters["heading"] = self.do_heading - - def _update_env(self, md: Markdown, config: dict) -> None: + def _update_env(self, md: Markdown, *, config: Any | None = None) -> None: """Update our handler to point to our configured Markdown instance, grabbing some of the config from `md`.""" - extensions = config["mdx"] + [MkdocstringsInnerExtension(self._headings)] + # YORE: Bump 1: Remove block. + if self.mdx is None and config is not None: + self.mdx = config.get("mdx", None) or config.get("markdown_extensions", None) or () + if self.mdx_config is None and config is not None: + self.mdx_config = config.get("mdx_config", None) or config.get("mdx_configs", None) or {} + + extensions: list[str | Extension] = [*self.mdx, MkdocstringsInnerExtension(self._headings)] + + new_md = Markdown(extensions=extensions, extension_configs=self.mdx_config) - new_md = Markdown(extensions=extensions, extension_configs=config["mdx_configs"]) # MkDocs adds its own (required) extension that's not part of the config. Propagate it. if "relpath" in md.treeprocessors: new_md.treeprocessors.register(md.treeprocessors["relpath"], "relpath", priority=0) - self.update_env(new_md, config) + self._md = new_md + + self.env.filters["highlight"] = Highlighter(new_md).highlight + self.env.filters["convert_markdown"] = self.do_convert_markdown + self.env.filters["heading"] = self.do_heading + + # YORE: Bump 1: Replace block with `self.update_env(config)`. + parameters = inspect.signature(self.update_env).parameters + if "md" in parameters: + warn( + "The `update_env(md)` parameter is deprecated. Use `self.md` instead.", + DeprecationWarning, + stacklevel=1, + ) + self.update_env(new_md, config) + elif "config" in parameters: + self.update_env(config) class Handlers: @@ -413,16 +520,42 @@ class Handlers: this for the purpose of caching. Use [mkdocstrings.plugin.MkdocstringsPlugin.get_handler][] for convenient access. """ - def __init__(self, config: dict) -> None: + def __init__( + self, + *, + theme: str, + default: str, + inventory_project: str, + inventory_version: str = "0.0.0", + handlers_config: dict[str, HandlerConfig] | None = None, + custom_templates: str | None = None, + mdx: Sequence[str | Extension] | None = None, + mdx_config: Mapping[str, Any] | None = None, + tool_config: Any, + ) -> None: """Initialize the object. Arguments: - config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code - of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary. + theme: The theme to use. + default: The default handler to use. + inventory_project: The project name to use in the inventory. + inventory_version: The project version to use in the inventory. + handlers_config: The handlers configuration. + custom_templates: The path to custom templates. + mdx: A list of Markdown extensions to use. + mdx_config: Configuration for the Markdown extensions. + tool_config: Tool configuration to pass down to handlers. """ - self._config = config + self._theme = theme + self._default = default + self._handlers_config = handlers_config or {} + self._custom_templates = custom_templates + self._mdx = mdx or [] + self._mdx_config = mdx_config or {} self._handlers: dict[str, BaseHandler] = {} - self.inventory: Inventory = Inventory(project=self._config["mkdocs"]["site_name"]) + self._tool_config = tool_config + + self.inventory: Inventory = Inventory(project=inventory_project, version=inventory_version) def get_anchors(self, identifier: str) -> tuple[str, ...]: """Return the canonical HTML anchor for the identifier, if any of the seen handlers can collect it. @@ -452,10 +585,7 @@ def get_handler_name(self, config: dict) -> str: Returns: The name of the handler to use. """ - global_config = self._config["mkdocstrings"] - if "handler" in config: - return config["handler"] - return global_config["default_handler"] + return config.get("handler", self._default) def get_handler_config(self, name: str) -> dict: """Return the global configuration of the given handler. @@ -466,10 +596,7 @@ def get_handler_config(self, name: str) -> dict: Returns: The global configuration of the given handler. It can be an empty dictionary. """ - handlers = self._config["mkdocstrings"].get("handlers", {}) - if handlers: - return handlers.get(name, {}) - return {} + return self._handlers_config.get(name, None) or {} def get_handler(self, name: str, handler_config: dict | None = None) -> BaseHandler: """Get a handler thanks to its name. @@ -489,14 +616,15 @@ def get_handler(self, name: str, handler_config: dict | None = None) -> BaseHand """ if name not in self._handlers: if handler_config is None: - handler_config = self.get_handler_config(name) - handler_config.update(self._config) + handler_config = self._handlers_config.get(name, {}) module = importlib.import_module(f"mkdocstrings_handlers.{name}") self._handlers[name] = module.get_handler( - theme=self._config["theme_name"], - custom_templates=self._config["mkdocstrings"]["custom_templates"], - config_file_path=self._config["mkdocs"]["config_file_path"], - **handler_config, + theme=self._theme, + custom_templates=self._custom_templates, + mdx=self._mdx, + mdx_config=self._mdx_config, + handler_config=handler_config, + tool_config=self._tool_config, ) return self._handlers[name] diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 28060b6b..386fece3 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -156,8 +156,6 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: return config log.debug("Adding extension to the list") - theme_name = config.theme.name or os.path.dirname(config.theme.dirs[0]) - to_import: InventoryImportType = [] for handler_name, conf in self.config.handlers.items(): for import_item in conf.pop("import", ()): @@ -165,14 +163,17 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: import_item = {"url": import_item} # noqa: PLW2901 to_import.append((handler_name, import_item)) - extension_config = { - "theme_name": theme_name, - "mdx": config.markdown_extensions, - "mdx_configs": config.mdx_configs, - "mkdocstrings": self.config, - "mkdocs": config, - } - self._handlers = Handlers(extension_config) + handlers = Handlers( + default=self.config.default_handler, + handlers_config=self.config.handlers, + theme=config.theme.name or os.path.dirname(config.theme.dirs[0]), + custom_templates=self.config.custom_templates, + mdx=config.markdown_extensions, + mdx_config=config.mdx_configs, + inventory_project=config.site_name, + inventory_version="0.0.0", # TODO: Find a way to get actual version. + tool_config=config, + ) autorefs: AutorefsPlugin try: @@ -187,18 +188,20 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: config.plugins["autorefs"] = autorefs log.debug("Added a subdued autorefs instance %r", autorefs) # Add collector-based fallback in either case. - autorefs.get_fallback_anchor = self.handlers.get_anchors + autorefs.get_fallback_anchor = handlers.get_anchors - mkdocstrings_extension = MkdocstringsExtension(extension_config, self.handlers, autorefs) + mkdocstrings_extension = MkdocstringsExtension(handlers, autorefs) config.markdown_extensions.append(mkdocstrings_extension) # type: ignore[arg-type] config.extra_css.insert(0, self.css_filename) # So that it has lower priority than user files. + self._handlers = handlers + self._inv_futures = {} if to_import: inv_loader = futures.ThreadPoolExecutor(4) for handler_name, import_item in to_import: - loader = self.get_handler(handler_name).load_inventory + loader = handlers.get_handler(handler_name).load_inventory future = inv_loader.submit( self._load_inventory, # type: ignore[misc] loader, From 4f4a40a564fabfcb7f0b5fda10e9d169888ebbdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Jan 2025 01:21:29 +0100 Subject: [PATCH 12/39] docs: Load Jinja and MarkupSafe object inventories --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 610b224f..0288678d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -143,6 +143,8 @@ plugins: - https://mkdocstrings.github.io/autorefs/objects.inv - https://www.mkdocs.org/objects.inv - https://python-markdown.github.io/objects.inv + - https://jinja.palletsprojects.com/en/stable/objects.inv + - https://markupsafe.palletsprojects.com/en/stable/objects.inv paths: [src] options: docstring_options: From bb87cd833f2333e77cb2c2926aa24a434c97391f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Jan 2025 01:25:09 +0100 Subject: [PATCH 13/39] refactor: Use mkdocs-get-deps' download utility to remove duplicated code --- pyproject.toml | 3 +- src/mkdocstrings/_cache.py | 57 +------------------------------------- src/mkdocstrings/plugin.py | 7 +++-- 3 files changed, 7 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 867747f8..9bb08588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,13 +30,12 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "click>=7.0", "Jinja2>=2.11.1", "Markdown>=3.6", "MarkupSafe>=1.1", "mkdocs>=1.4", "mkdocs-autorefs>=1.2", - "platformdirs>=2.2", + "mkdocs-get-deps>=0.2", # TODO: Remove when we depend on mkdocs>=1.5. "pymdown-extensions>=6.3", "importlib-metadata>=4.6; python_version < '3.10'", "typing-extensions>=4.1; python_version < '3.10'", diff --git a/src/mkdocstrings/_cache.py b/src/mkdocstrings/_cache.py index 0bd0d90e..b9af327d 100644 --- a/src/mkdocstrings/_cache.py +++ b/src/mkdocstrings/_cache.py @@ -1,16 +1,11 @@ import base64 -import datetime import gzip -import hashlib import os import re import urllib.parse import urllib.request from collections.abc import Mapping -from typing import BinaryIO, Callable, Optional - -import click -import platformdirs +from typing import BinaryIO, Optional from mkdocstrings.loggers import get_logger @@ -80,53 +75,3 @@ def _create_auth_header(credential: str, url: str) -> dict[str, str]: log.debug("Using basic authentication for %s", url) credentials = base64.encodebytes(f"{user}:{pwd}".encode()).decode().strip() return {"Authorization": f"Basic {credentials}"} - - -# This is mostly a copy of https://github.com/mkdocs/mkdocs/blob/master/mkdocs/utils/cache.py -# In the future maybe they can be deduplicated. - - -def download_and_cache_url( - url: str, - download: Callable[[str], bytes], - cache_duration: datetime.timedelta, - comment: bytes = b"# ", -) -> bytes: - """Downloads a file from the URL, stores it under ~/.cache/, and returns its content. - - For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime. - - Args: - url: URL to use. - download: Callback that will accept the URL and actually perform the download. - cache_duration: How long to consider the URL content cached. - comment: The appropriate comment prefix for this file format. - """ - directory = os.path.join(platformdirs.user_cache_dir("mkdocs"), "mkdocstrings_url_cache") - name_hash = hashlib.sha256(url.encode()).hexdigest()[:32] - path = os.path.join(directory, name_hash + os.path.splitext(url)[1]) - - now = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) - prefix = b"%s%s downloaded at timestamp " % (comment, url.encode()) - # Check for cached file and try to return it - if os.path.isfile(path): - try: - with open(path, "rb") as f: - line = f.readline() - if line.startswith(prefix): - line = line[len(prefix) :] - timestamp = int(line) - if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration: - log.debug("Using cached '%s' for '%s'", path, url) - return f.read() - except (OSError, ValueError) as e: - log.debug("%s: %s", type(e).__name__, e) - - # Download and cache the file - log.debug("Downloading '%s' to '%s'", url, path) - content = download(url) - os.makedirs(directory, exist_ok=True) - with click.open_file(path, "wb", atomic=True) as f: - f.write(b"%s%d\n" % (prefix, now)) - f.write(content) - return content diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 386fece3..97c7722f 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -29,7 +29,10 @@ from mkdocs.utils import write_file from mkdocs_autorefs.plugin import AutorefsConfig, AutorefsPlugin -from mkdocstrings._cache import download_and_cache_url, download_url_with_gz +# TODO: Replace with `from mkdocs.utils.cache import download_and_cache_url` when we depend on mkdocs>=1.5. +from mkdocs_get_deps.cache import download_and_cache_url + +from mkdocstrings._cache import download_url_with_gz from mkdocstrings.extension import MkdocstringsExtension from mkdocstrings.handlers.base import BaseHandler, Handlers from mkdocstrings.loggers import get_logger @@ -323,7 +326,7 @@ def _load_inventory(cls, loader: InventoryLoaderType, url: str, **kwargs: Any) - A mapping from identifier to absolute URL. """ log.debug("Downloading inventory from %s", url) - content = download_and_cache_url(url, download_url_with_gz, datetime.timedelta(days=1)) + content = download_and_cache_url(url, datetime.timedelta(days=1), download=download_url_with_gz) result = dict(loader(BytesIO(content), url=url, **kwargs)) log.debug("Loaded inventory from %s: %s items", url, len(result)) return result From 310f7002a22f33786303279598a71d4b72277e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Jan 2025 01:25:40 +0100 Subject: [PATCH 14/39] docs: Update troubleshooting entry about Sphinx comments --- docs/troubleshooting.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 7548eafe..d531997a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -167,9 +167,9 @@ def math_function(x, y): ### My docstrings in comments (`#:`) are not picked up -It's because we do not support type annotations in comments. +We only support docstrings in comments through the [griffe-sphinx](https://mkdocstrings.github.io/griffe-sphinx) extension. -So instead of: +Alternatively, instead of: ```python import enum @@ -187,15 +187,11 @@ import enum class MyEnum(enum.Enum): - """My enum. - - Attributes: - v1: The first choice. - v2: The second choice. - """ - v1 = 1 + """The first choice.""" + v2 = 2 + """The second choice.""" ``` Or: @@ -205,11 +201,15 @@ import enum class MyEnum(enum.Enum): - v1 = 1 - """The first choice.""" + """My enum. + + Attributes: + v1: The first choice. + v2: The second choice. + """ + v1 = 1 v2 = 2 - """The second choice.""" ``` ### My wrapped function shows documentation/code for its wrapper instead of its own From 06adb0ce07aed6baaedeb64bf2e9a39d9b3fed64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Jan 2025 01:26:29 +0100 Subject: [PATCH 15/39] style: Move TYPE_CHECKING block below at the end of imports --- src/mkdocstrings/plugin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 97c7722f..03d34df9 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -37,15 +37,16 @@ from mkdocstrings.handlers.base import BaseHandler, Handlers from mkdocstrings.loggers import get_logger -if TYPE_CHECKING: - from jinja2.environment import Environment - from mkdocs.config.defaults import MkDocsConfig - if sys.version_info < (3, 10): from typing_extensions import ParamSpec else: from typing import ParamSpec +if TYPE_CHECKING: + from jinja2.environment import Environment + from mkdocs.config.defaults import MkDocsConfig + + log = get_logger(__name__) InventoryImportType = list[tuple[str, Mapping[str, Any]]] From 434d8c7cd1e3edbdb9d4c45a9b44b290b19d88f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 9 Jan 2025 01:44:21 +0100 Subject: [PATCH 16/39] refactor: Register all identifiers of rendered objects into autorefs Previously we only registered in the inventory, because autorefs didn't let us register an "alias" anchor. See commit c7ac04324d005d9cf7d2c1f3b2c39f212275d451. Now that autorefs supports Markdown anchors, we can actually use that mechanism to register different identifiers that point to the same anchor. See commit a215a97a057b54e11ebec8865c64e93429edde63. --- pyproject.toml | 2 +- src/mkdocstrings/extension.py | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9bb08588..d05d0e35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "Markdown>=3.6", "MarkupSafe>=1.1", "mkdocs>=1.4", - "mkdocs-autorefs>=1.2", + "mkdocs-autorefs>=1.3", "mkdocs-get-deps>=0.2", # TODO: Remove when we depend on mkdocs>=1.5. "pymdown-extensions>=6.3", "importlib-metadata>=4.6; python_version < '3.10'", diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index b9fc0a79..3d2dc3a9 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -233,7 +233,20 @@ def _process_headings(self, handler: BaseHandler, element: Element) -> None: if page is not None: for heading in headings: rendered_anchor = heading.attrib["id"] - self._autorefs.register_anchor(page, rendered_anchor) + self._autorefs.register_anchor(page, rendered_anchor, primary=True) + + # Register all identifiers for this object + # both in the autorefs plugin and in the inventory. + try: + data_object = handler.collect(rendered_anchor, handler.fallback_config) + except CollectionError: + anchors = () + else: + anchors = handler.get_anchors(data_object) + + for anchor in anchors: + if anchor != rendered_anchor: + self._autorefs.register_anchor(page, anchor, rendered_anchor, primary=False) if "data-role" in heading.attrib: self._handlers.inventory.register( @@ -243,13 +256,7 @@ def _process_headings(self, handler: BaseHandler, element: Element) -> None: priority=1, # Register with standard priority. uri=f"{page}#{rendered_anchor}", ) - - # Also register other anchors for this object in the inventory. - try: - data_object = handler.collect(rendered_anchor, handler.fallback_config) - except CollectionError: - continue - for anchor in handler.get_anchors(data_object): + for anchor in anchors: if anchor not in self._handlers.inventory: self._handlers.inventory.register( name=anchor, From d2e9c21318a4ca5c9eaece2f60331b016036d343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 10 Jan 2025 17:27:04 +0100 Subject: [PATCH 17/39] style: Attach comments together --- src/mkdocstrings/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 3d2dc3a9..28b2af13 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -225,7 +225,7 @@ def _process_headings(self, handler: BaseHandler, element: Element) -> None: element.extend(headings) # These duplicated headings will later be removed by our `_HeadingsPostProcessor` processor, # which runs right after `toc` (see `MkdocstringsExtension.extendMarkdown`). - + # # If we were in an inner handler layer, we wouldn't do any of this # and would just let headings bubble up to the outer handler layer. From 7a668f0f731401b07123bd02aafbbfc55cd24c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 10 Jan 2025 17:46:18 +0100 Subject: [PATCH 18/39] refactor: Deprecate base handler's `get_anchors` method in favor of `get_aliases` method First, `get_anchors` accepts a data object as argument. This forces mkdocstrings to (re)collect this data object through the handler's `collect` method just to pass it again to the handler. Why not giving back control to the handler itself? For example, the Python handler would not call `collect` again, but just look into its cached objects collection. Second, "anchors" in this context doesn't make much sense, so we rename the method `get_aliases`, because it really returns aliases of a given identifier, not "HTML anchors". Plus this makes the deprecation easier (method rename instead of signature change). --- config/pytest.ini | 1 + src/mkdocstrings/extension.py | 45 ++++++++++++++++++++----------- src/mkdocstrings/handlers/base.py | 25 +++++++++++------ 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/config/pytest.ini b/config/pytest.ini index eb7af02a..edcaffb5 100644 --- a/config/pytest.ini +++ b/config/pytest.ini @@ -18,3 +18,4 @@ filterwarnings = ignore:.*`mdx_config` argument:DeprecationWarning:mkdocstrings_handlers ignore:.*`update_env\(md\)` parameter:DeprecationWarning:mkdocstrings ignore:.*`super\(\).update_env\(\)` anymore:DeprecationWarning:mkdocstrings_handlers + ignore:.*`get_anchors` method:DeprecationWarning:mkdocstrings diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 28b2af13..74e03530 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -26,6 +26,7 @@ import re from collections import ChainMap from typing import TYPE_CHECKING, Any +from warnings import warn from xml.etree.ElementTree import Element import yaml @@ -232,38 +233,50 @@ def _process_headings(self, handler: BaseHandler, element: Element) -> None: page = self._autorefs.current_page if page is not None: for heading in headings: - rendered_anchor = heading.attrib["id"] - self._autorefs.register_anchor(page, rendered_anchor, primary=True) + rendered_id = heading.attrib["id"] + self._autorefs.register_anchor(page, rendered_id, primary=True) # Register all identifiers for this object # both in the autorefs plugin and in the inventory. - try: - data_object = handler.collect(rendered_anchor, handler.fallback_config) - except CollectionError: - anchors = () + aliases: tuple[str, ...] + # YORE: Bump 1: Replace block with line 16. + if hasattr(handler, "get_anchors"): + warn( + "The `get_anchors` method is deprecated. " + "Declare a `get_aliases` method instead, accepting a string (identifier) " + "instead of a collected object.", + DeprecationWarning, + stacklevel=1, + ) + try: + data_object = handler.collect(rendered_id, handler.fallback_config) + except CollectionError: + aliases = () + else: + aliases = handler.get_anchors(data_object) else: - anchors = handler.get_anchors(data_object) + aliases = handler.get_aliases(rendered_id) - for anchor in anchors: - if anchor != rendered_anchor: - self._autorefs.register_anchor(page, anchor, rendered_anchor, primary=False) + for alias in aliases: + if alias != rendered_id: + self._autorefs.register_anchor(page, alias, rendered_id, primary=False) if "data-role" in heading.attrib: self._handlers.inventory.register( - name=rendered_anchor, + name=rendered_id, domain=handler.domain, role=heading.attrib["data-role"], priority=1, # Register with standard priority. - uri=f"{page}#{rendered_anchor}", + uri=f"{page}#{rendered_id}", ) - for anchor in anchors: - if anchor not in self._handlers.inventory: + for alias in aliases: + if alias not in self._handlers.inventory: self._handlers.inventory.register( - name=anchor, + name=alias, domain=handler.domain, role=heading.attrib["data-role"], priority=2, # Register with lower priority. - uri=f"{page}#{rendered_anchor}", + uri=f"{page}#{rendered_id}", ) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 427e5af9..10f2a9a3 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -341,14 +341,14 @@ def get_extended_templates_dirs(self, handler: str) -> list[Path]: discovered_extensions = entry_points(group=f"mkdocstrings.{handler}.templates") return [extension.load()() for extension in discovered_extensions] - def get_anchors(self, data: CollectorItem) -> tuple[str, ...]: # noqa: ARG002 - """Return the possible identifiers (HTML anchors) for a collected item. + def get_aliases(self, identifier: str) -> tuple[str, ...]: # noqa: ARG002 + """Return the possible aliases for a given identifier. Arguments: - data: The collected data. + identifier: The identifier to get the aliases of. Returns: - The HTML anchors (without '#'), or an empty tuple if this item doesn't have an anchor. + A tuple of strings - aliases. """ return () @@ -567,13 +567,22 @@ def get_anchors(self, identifier: str) -> tuple[str, ...]: A tuple of strings - anchors without '#', or an empty tuple if there isn't any identifier familiar with it. """ for handler in self._handlers.values(): - fallback_config = getattr(handler, "fallback_config", {}) try: - anchors = handler.get_anchors(handler.collect(identifier, fallback_config)) + if hasattr(handler, "get_anchors"): + warn( + "The `get_anchors` method is deprecated. " + "Declare a `get_aliases` method instead, accepting a string (identifier) " + "instead of a collected object.", + DeprecationWarning, + stacklevel=1, + ) + aliases = handler.get_anchors(handler.collect(identifier, handler.fallback_config)) + else: + aliases = handler.get_aliases(identifier) except CollectionError: continue - if anchors: - return anchors + if aliases: + return aliases return () def get_handler_name(self, config: dict) -> str: From f80ef5d98b82d9b97c467825ce606f1e4a7110c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 10 Jan 2025 17:47:07 +0100 Subject: [PATCH 19/39] tests: Assert identifier aliases are registered early in mkdocs-autorefs --- tests/test_extension.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_extension.py b/tests/test_extension.py index 976f376c..5dc84382 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -154,16 +154,16 @@ def test_use_custom_handler(ext_markdown: Markdown) -> None: ext_markdown.convert("::: tests.fixtures.headings\n handler: not_here") -def test_dont_register_every_identifier_as_anchor(plugin: MkdocstringsPlugin, ext_markdown: Markdown) -> None: +def test_register_every_identifier_alias(plugin: MkdocstringsPlugin, ext_markdown: Markdown) -> None: """Assert that we don't preemptively register all identifiers of a rendered object.""" handler = plugin._handlers.get_handler("python") # type: ignore[union-attr] ids = ("id1", "id2", "id3") handler.get_anchors = lambda _: ids # type: ignore[method-assign] - ext_markdown.convert("::: tests.fixtures.headings") autorefs = ext_markdown.parser.blockprocessors["mkdocstrings"]._autorefs # type: ignore[attr-defined] + autorefs.current_page = "foo" + ext_markdown.convert("::: tests.fixtures.headings") for identifier in ids: - assert identifier not in autorefs._url_map - assert identifier not in autorefs._abs_url_map + assert identifier in autorefs._secondary_url_map def test_use_options_yaml_key(ext_markdown: Markdown) -> None: From c00de7a42b9072cbaa47ecbf18e3e15a6d5ab634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 10 Jan 2025 22:30:12 +0100 Subject: [PATCH 20/39] refactor: Give back control to handlers on how they want to handle global/local options Issue-719: https://github.com/mkdocstrings/mkdocstrings/issues/719 --- src/mkdocstrings/extension.py | 29 +++++++++++++++++++---------- src/mkdocstrings/handlers/base.py | 24 ++++++++++++++++++++---- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 74e03530..ab7e12b5 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -24,7 +24,6 @@ from __future__ import annotations import re -from collections import ChainMap from typing import TYPE_CHECKING, Any from warnings import warn from xml.etree.ElementTree import Element @@ -159,20 +158,30 @@ def _process_block( Returns: Rendered HTML, the handler that was used, and the collected item. """ - config = yaml.safe_load(yaml_block) or {} - handler_name = self._handlers.get_handler_name(config) + local_config = yaml.safe_load(yaml_block) or {} + handler_name = self._handlers.get_handler_name(local_config) log.debug("Using handler '%s'", handler_name) - handler_config = self._handlers.get_handler_config(handler_name) - handler = self._handlers.get_handler(handler_name, handler_config) - - global_options = handler_config.get("options", {}) - local_options = config.get("options", {}) - options = ChainMap(local_options, global_options) + handler = self._handlers.get_handler(handler_name) + local_options = local_config.get("options", {}) if heading_level: # Heading level obtained from Markdown (`##`) takes precedence. - options = ChainMap({"heading_level": heading_level}, options) + local_options["heading_level"] = heading_level + + # YORE: Bump 1: Replace block with line 2. + if handler.get_options is not BaseHandler.get_options: + options = handler.get_options(local_options) + else: + warn( + "mkdocstrings v1 will start using your handler's `get_options` method to build options " + "instead of merging the global and local options (dictionaries). ", + DeprecationWarning, + stacklevel=1, + ) + handler_config = self._handlers.get_handler_config(handler_name) + global_options = handler_config.get("options", {}) + options = {**global_options, **local_options} log.debug("Collecting data") try: diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 10f2a9a3..2d2e8c66 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -44,6 +44,7 @@ CollectorItem = Any HandlerConfig = Any +HandlerOptions = Any # Autodoc instructions can appear in nested Markdown, @@ -265,7 +266,22 @@ def load_inventory( """ yield from () - def collect(self, identifier: str, config: Mapping[str, Any]) -> CollectorItem: + def get_options(self, local_options: Mapping[str, Any]) -> HandlerOptions: + """Get combined options. + + Override this method to customize how options are combined, + for example by merging the global options with the local options. + By combining options here, you don't have to do it twice in `collect` and `render`. + + Arguments: + local_options: The local options. + + Returns: + The combined options. + """ + return local_options + + def collect(self, identifier: str, options: HandlerOptions) -> CollectorItem: """Collect data given an identifier and user configuration. In the implementation, you typically call a subprocess that returns JSON, and load that JSON again into @@ -275,19 +291,19 @@ def collect(self, identifier: str, config: Mapping[str, Any]) -> CollectorItem: identifier: An identifier for which to collect data. For example, in Python, it would be 'mkdocstrings.handlers' to collect documentation about the handlers module. It can be anything that you can feed to the tool of your choice. - config: The handler's configuration options. + options: The final configuration options. Returns: Anything you want, as long as you can feed it to the handler's `render` method. """ raise NotImplementedError - def render(self, data: CollectorItem, config: Mapping[str, Any]) -> str: + def render(self, data: CollectorItem, options: HandlerOptions) -> str: """Render a template using provided data and configuration options. Arguments: data: The collected data to render. - config: The handler's configuration options. + options: The final configuration options. Returns: The rendered template as HTML. From b3f4d38da5c23cc4f31b0647464d0dddf6e2dc2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 10 Jan 2025 22:30:51 +0100 Subject: [PATCH 21/39] style: Declare filters earlier, no need to wait for these ones --- src/mkdocstrings/handlers/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 2d2e8c66..7f637700 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -231,6 +231,8 @@ def __init__( loader=FileSystemLoader(paths), auto_reload=False, # Editing a template in the middle of a build is not useful. ) + self.env.filters["convert_markdown"] = self.do_convert_markdown + self.env.filters["heading"] = self.do_heading self.env.filters["any"] = do_any self.env.globals["log"] = get_template_logger(self.name) @@ -513,8 +515,6 @@ def _update_env(self, md: Markdown, *, config: Any | None = None) -> None: self._md = new_md self.env.filters["highlight"] = Highlighter(new_md).highlight - self.env.filters["convert_markdown"] = self.do_convert_markdown - self.env.filters["heading"] = self.do_heading # YORE: Bump 1: Replace block with `self.update_env(config)`. parameters = inspect.signature(self.update_env).parameters From 29b5398de45acfa875428f607b656863c28a83af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 10 Jan 2025 22:31:59 +0100 Subject: [PATCH 22/39] chore: Add Yore comments to stop using autorefs' fallback mechanism in v1 --- src/mkdocstrings/handlers/base.py | 1 + src/mkdocstrings/plugin.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 7f637700..aa924910 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -573,6 +573,7 @@ def __init__( self.inventory: Inventory = Inventory(project=inventory_project, version=inventory_version) + # YORE: Bump 1: Remove block. def get_anchors(self, identifier: str) -> tuple[str, ...]: """Return the canonical HTML anchor for the identifier, if any of the seen handlers can collect it. diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 03d34df9..844ee0f4 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -191,7 +191,7 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: autorefs.scan_toc = False config.plugins["autorefs"] = autorefs log.debug("Added a subdued autorefs instance %r", autorefs) - # Add collector-based fallback in either case. + # YORE: Bump 1: Remove line. autorefs.get_fallback_anchor = handlers.get_anchors mkdocstrings_extension = MkdocstringsExtension(handlers, autorefs) From 8450426aad6f609536b619d08b57fb5e1974ffcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sat, 11 Jan 2025 00:43:30 +0100 Subject: [PATCH 23/39] tests: Update test for previous refactor --- tests/test_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extension.py b/tests/test_extension.py index 5dc84382..c3784b2e 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -158,7 +158,7 @@ def test_register_every_identifier_alias(plugin: MkdocstringsPlugin, ext_markdow """Assert that we don't preemptively register all identifiers of a rendered object.""" handler = plugin._handlers.get_handler("python") # type: ignore[union-attr] ids = ("id1", "id2", "id3") - handler.get_anchors = lambda _: ids # type: ignore[method-assign] + handler.get_aliases = lambda _: ids # type: ignore[method-assign] autorefs = ext_markdown.parser.blockprocessors["mkdocstrings"]._autorefs # type: ignore[attr-defined] autorefs.current_page = "foo" ext_markdown.convert("::: tests.fixtures.headings") From 649c221bc560ef093229f2c6723c08656dcb39f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sat, 11 Jan 2025 00:53:07 +0100 Subject: [PATCH 24/39] chore: Template upgrade --- .copier-answers.yml | 2 +- docs/insiders/index.md | 2 + pyproject.toml | 18 ++-- scripts/gen_credits.py | 2 +- scripts/get_version.py | 27 ++++++ scripts/make | 191 +---------------------------------------- scripts/make.py | 191 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 232 insertions(+), 201 deletions(-) create mode 100644 scripts/get_version.py mode change 100755 => 120000 scripts/make create mode 100755 scripts/make.py diff --git a/.copier-answers.yml b/.copier-answers.yml index 6a772352..b3437df7 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 1.5.2 +_commit: 1.5.6 _src_path: gh:pawamoy/copier-uv author_email: dev@pawamoy.fr author_fullname: Timothée Mazzucotelli diff --git a/docs/insiders/index.md b/docs/insiders/index.md index ccbca99a..daa4731c 100644 --- a/docs/insiders/index.md +++ b/docs/insiders/index.md @@ -97,6 +97,8 @@ else: ``` +Additionally, your sponsorship will give more weight to your upvotes on issues, helping us prioritize work items in our backlog. For more information on how we prioritize work, see this page: [Backlog management](https://pawamoy.github.io/backlog/). + ## How to become a sponsor Thanks for your interest in sponsoring! In order to become an eligible sponsor diff --git a/pyproject.toml b/pyproject.toml index d05d0e35..49b3b818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,12 +59,12 @@ Funding = "https://github.com/sponsors/pawamoy" [project.entry-points."mkdocs.plugins"] mkdocstrings = "mkdocstrings.plugin:MkdocstringsPlugin" -[tool.pdm] -version = {source = "scm"} +[tool.pdm.version] +source = "call" +getter = "scripts.get_version:get_version" [tool.pdm.build] -package-dir = "src" -editable-backend = "editables" +# Include as much as possible in the source distribution, to help redistributors. excludes = ["**/.pytest_cache"] source-includes = [ "config", @@ -79,15 +79,15 @@ source-includes = [ ] [tool.pdm.build.wheel-data] +# Manual pages can be included in the wheel. +# Depending on the installation tool, they will be accessible to users. +# pipx supports it, uv does not yet, see https://github.com/astral-sh/uv/issues/4731. data = [ {path = "share/**/*", relative-to = "."}, ] -[tool.uv] -dev-dependencies = [ - # dev - "editables>=0.5", - +[dependency-groups] +dev = [ # maintenance "build>=1.2", "git-changelog>=2.5", diff --git a/scripts/gen_credits.py b/scripts/gen_credits.py index bd2dcbf2..721ac05d 100644 --- a/scripts/gen_credits.py +++ b/scripts/gen_credits.py @@ -27,7 +27,7 @@ pyproject = tomllib.load(pyproject_file) project = pyproject["project"] project_name = project["name"] -devdeps = [dep for dep in pyproject["tool"]["uv"]["dev-dependencies"] if not dep.startswith("-e")] +devdeps = [dep for dep in pyproject["dependency-groups"]["dev"] if not dep.startswith("-e")] PackageMetadata = dict[str, Union[str, Iterable[str]]] Metadata = dict[str, PackageMetadata] diff --git a/scripts/get_version.py b/scripts/get_version.py new file mode 100644 index 00000000..f4a30a8c --- /dev/null +++ b/scripts/get_version.py @@ -0,0 +1,27 @@ +"""Get current project version from Git tags or changelog.""" + +import re +from contextlib import suppress +from pathlib import Path + +from pdm.backend.hooks.version import SCMVersion, Version, default_version_formatter, get_version_from_scm + +_root = Path(__file__).parent.parent +_changelog = _root / "CHANGELOG.md" +_changelog_version_re = re.compile(r"^## \[(\d+\.\d+\.\d+)\].*$") +_default_scm_version = SCMVersion(Version("0.0.0"), None, False, None, None) # noqa: FBT003 + + +def get_version() -> str: + """Get current project version from Git tags or changelog.""" + scm_version = get_version_from_scm(_root) or _default_scm_version + if scm_version.version <= Version("0.1"): # Missing Git tags? + with suppress(OSError, StopIteration): # noqa: SIM117 + with _changelog.open("r", encoding="utf8") as file: + match = next(filter(None, map(_changelog_version_re.match, file))) + scm_version = scm_version._replace(version=Version(match.group(1))) + return default_version_formatter(scm_version) + + +if __name__ == "__main__": + print(get_version()) diff --git a/scripts/make b/scripts/make deleted file mode 100755 index ac430624..00000000 --- a/scripts/make +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python3 -"""Management commands.""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -from contextlib import contextmanager -from pathlib import Path -from textwrap import dedent -from typing import Any, Iterator - -PYTHON_VERSIONS = os.getenv("PYTHON_VERSIONS", "3.9 3.10 3.11 3.12 3.13 3.14").split() - - -def shell(cmd: str, capture_output: bool = False, **kwargs: Any) -> str | None: - """Run a shell command.""" - if capture_output: - return subprocess.check_output(cmd, shell=True, text=True, **kwargs) # noqa: S602 - subprocess.run(cmd, shell=True, check=True, stderr=subprocess.STDOUT, **kwargs) # noqa: S602 - return None - - -@contextmanager -def environ(**kwargs: str) -> Iterator[None]: - """Temporarily set environment variables.""" - original = dict(os.environ) - os.environ.update(kwargs) - try: - yield - finally: - os.environ.clear() - os.environ.update(original) - - -def uv_install(venv: Path) -> None: - """Install dependencies using uv.""" - with environ(UV_PROJECT_ENVIRONMENT=str(venv), PYO3_USE_ABI3_FORWARD_COMPATIBILITY="1"): - if "CI" in os.environ: - shell("uv sync --no-editable") - else: - shell("uv sync") - - -def setup() -> None: - """Setup the project.""" - if not shutil.which("uv"): - raise ValueError("make: setup: uv must be installed, see https://github.com/astral-sh/uv") - - print("Installing dependencies (default environment)") # noqa: T201 - default_venv = Path(".venv") - if not default_venv.exists(): - shell("uv venv --python python") - uv_install(default_venv) - - if PYTHON_VERSIONS: - for version in PYTHON_VERSIONS: - print(f"\nInstalling dependencies (python{version})") # noqa: T201 - venv_path = Path(f".venvs/{version}") - if not venv_path.exists(): - shell(f"uv venv --python {version} {venv_path}") - with environ(UV_PROJECT_ENVIRONMENT=str(venv_path.resolve())): - uv_install(venv_path) - - -def run(version: str, cmd: str, *args: str, no_sync: bool = False, **kwargs: Any) -> None: - """Run a command in a virtual environment.""" - kwargs = {"check": True, **kwargs} - uv_run = ["uv", "run"] - if no_sync: - uv_run.append("--no-sync") - if version == "default": - with environ(UV_PROJECT_ENVIRONMENT=".venv"): - subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 - else: - with environ(UV_PROJECT_ENVIRONMENT=f".venvs/{version}", MULTIRUN="1"): - subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 - - -def multirun(cmd: str, *args: str, **kwargs: Any) -> None: - """Run a command for all configured Python versions.""" - if PYTHON_VERSIONS: - for version in PYTHON_VERSIONS: - run(version, cmd, *args, **kwargs) - else: - run("default", cmd, *args, **kwargs) - - -def allrun(cmd: str, *args: str, **kwargs: Any) -> None: - """Run a command in all virtual environments.""" - run("default", cmd, *args, **kwargs) - if PYTHON_VERSIONS: - multirun(cmd, *args, **kwargs) - - -def clean() -> None: - """Delete build artifacts and cache files.""" - paths_to_clean = ["build", "dist", "htmlcov", "site", ".coverage*", ".pdm-build"] - for path in paths_to_clean: - shell(f"rm -rf {path}") - - cache_dirs = {".cache", ".pytest_cache", ".mypy_cache", ".ruff_cache", "__pycache__"} - for dirpath in Path(".").rglob("*/"): - if dirpath.parts[0] not in (".venv", ".venvs") and dirpath.name in cache_dirs: - shutil.rmtree(dirpath, ignore_errors=True) - - -def vscode() -> None: - """Configure VSCode to work on this project.""" - Path(".vscode").mkdir(parents=True, exist_ok=True) - shell("cp -v config/vscode/* .vscode") - - -def main() -> int: - """Main entry point.""" - args = list(sys.argv[1:]) - if not args or args[0] == "help": - if len(args) > 1: - run("default", "duty", "--help", args[1]) - else: - print( - dedent( - """ - Available commands - help Print this help. Add task name to print help. - setup Setup all virtual environments (install dependencies). - run Run a command in the default virtual environment. - multirun Run a command for all configured Python versions. - allrun Run a command in all virtual environments. - 3.x Run a command in the virtual environment for Python 3.x. - clean Delete build artifacts and cache files. - vscode Configure VSCode to work on this project. - """ - ), - flush=True, - ) # noqa: T201 - if os.path.exists(".venv"): - print("\nAvailable tasks", flush=True) # noqa: T201 - run("default", "duty", "--list", no_sync=True) - return 0 - - while args: - cmd = args.pop(0) - - if cmd == "run": - run("default", *args) - return 0 - - if cmd == "multirun": - multirun(*args) - return 0 - - if cmd == "allrun": - allrun(*args) - return 0 - - if cmd.startswith("3."): - run(cmd, *args) - return 0 - - opts = [] - while args and (args[0].startswith("-") or "=" in args[0]): - opts.append(args.pop(0)) - - if cmd == "clean": - clean() - elif cmd == "setup": - setup() - elif cmd == "vscode": - vscode() - elif cmd == "check": - multirun("duty", "check-quality", "check-types", "check-docs") - run("default", "duty", "check-api") - elif cmd in {"check-quality", "check-docs", "check-types", "test"}: - multirun("duty", cmd, *opts) - else: - run("default", "duty", cmd, *opts) - - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as process: - if process.output: - print(process.output, file=sys.stderr) # noqa: T201 - sys.exit(process.returncode) diff --git a/scripts/make b/scripts/make new file mode 120000 index 00000000..c2eda0df --- /dev/null +++ b/scripts/make @@ -0,0 +1 @@ +make.py \ No newline at end of file diff --git a/scripts/make.py b/scripts/make.py new file mode 100755 index 00000000..3d427296 --- /dev/null +++ b/scripts/make.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Management commands.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from textwrap import dedent +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + + +PYTHON_VERSIONS = os.getenv("PYTHON_VERSIONS", "3.9 3.10 3.11 3.12 3.13 3.14").split() + + +def shell(cmd: str, *, capture_output: bool = False, **kwargs: Any) -> str | None: + """Run a shell command.""" + if capture_output: + return subprocess.check_output(cmd, shell=True, text=True, **kwargs) # noqa: S602 + subprocess.run(cmd, shell=True, check=True, stderr=subprocess.STDOUT, **kwargs) # noqa: S602 + return None + + +@contextmanager +def environ(**kwargs: str) -> Iterator[None]: + """Temporarily set environment variables.""" + original = dict(os.environ) + os.environ.update(kwargs) + try: + yield + finally: + os.environ.clear() + os.environ.update(original) + + +def uv_install(venv: Path) -> None: + """Install dependencies using uv.""" + with environ(UV_PROJECT_ENVIRONMENT=str(venv), PYO3_USE_ABI3_FORWARD_COMPATIBILITY="1"): + if "CI" in os.environ: + shell("uv sync --no-editable") + else: + shell("uv sync") + + +def setup() -> None: + """Setup the project.""" + if not shutil.which("uv"): + raise ValueError("make: setup: uv must be installed, see https://github.com/astral-sh/uv") + + print("Installing dependencies (default environment)") + default_venv = Path(".venv") + if not default_venv.exists(): + shell("uv venv") + uv_install(default_venv) + + if PYTHON_VERSIONS: + for version in PYTHON_VERSIONS: + print(f"\nInstalling dependencies (python{version})") + venv_path = Path(f".venvs/{version}") + if not venv_path.exists(): + shell(f"uv venv --python {version} {venv_path}") + with environ(UV_PROJECT_ENVIRONMENT=str(venv_path.resolve())): + uv_install(venv_path) + + +def run(version: str, cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command in a virtual environment.""" + kwargs = {"check": True, **kwargs} + uv_run = ["uv", "run", "--no-sync"] + if version == "default": + with environ(UV_PROJECT_ENVIRONMENT=".venv"): + subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 + else: + with environ(UV_PROJECT_ENVIRONMENT=f".venvs/{version}", MULTIRUN="1"): + subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 + + +def multirun(cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command for all configured Python versions.""" + if PYTHON_VERSIONS: + for version in PYTHON_VERSIONS: + run(version, cmd, *args, **kwargs) + else: + run("default", cmd, *args, **kwargs) + + +def allrun(cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command in all virtual environments.""" + run("default", cmd, *args, **kwargs) + if PYTHON_VERSIONS: + multirun(cmd, *args, **kwargs) + + +def clean() -> None: + """Delete build artifacts and cache files.""" + paths_to_clean = ["build", "dist", "htmlcov", "site", ".coverage*", ".pdm-build"] + for path in paths_to_clean: + shutil.rmtree(path, ignore_errors=True) + + cache_dirs = {".cache", ".pytest_cache", ".mypy_cache", ".ruff_cache", "__pycache__"} + for dirpath in Path(".").rglob("*/"): + if dirpath.parts[0] not in (".venv", ".venvs") and dirpath.name in cache_dirs: + shutil.rmtree(dirpath, ignore_errors=True) + + +def vscode() -> None: + """Configure VSCode to work on this project.""" + shutil.copytree("config/vscode", ".vscode", dirs_exist_ok=True) + + +def main() -> int: + """Main entry point.""" + args = list(sys.argv[1:]) + if not args or args[0] == "help": + if len(args) > 1: + run("default", "duty", "--help", args[1]) + else: + print( + dedent( + """ + Available commands + help Print this help. Add task name to print help. + setup Setup all virtual environments (install dependencies). + run Run a command in the default virtual environment. + multirun Run a command for all configured Python versions. + allrun Run a command in all virtual environments. + 3.x Run a command in the virtual environment for Python 3.x. + clean Delete build artifacts and cache files. + vscode Configure VSCode to work on this project. + """, + ), + flush=True, + ) + if os.path.exists(".venv"): + print("\nAvailable tasks", flush=True) + run("default", "duty", "--list") + return 0 + + while args: + cmd = args.pop(0) + + if cmd == "run": + run("default", *args) + return 0 + + if cmd == "multirun": + multirun(*args) + return 0 + + if cmd == "allrun": + allrun(*args) + return 0 + + if cmd.startswith("3."): + run(cmd, *args) + return 0 + + opts = [] + while args and (args[0].startswith("-") or "=" in args[0]): + opts.append(args.pop(0)) + + if cmd == "clean": + clean() + elif cmd == "setup": + setup() + elif cmd == "vscode": + vscode() + elif cmd == "check": + multirun("duty", "check-quality", "check-types", "check-docs") + run("default", "duty", "check-api") + elif cmd in {"check-quality", "check-docs", "check-types", "test"}: + multirun("duty", cmd, *opts) + else: + run("default", "duty", cmd, *opts) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except subprocess.CalledProcessError as process: + if process.output: + print(process.output, file=sys.stderr) + sys.exit(process.returncode) From 095d2f3e42351340aa3759289515ffc13d2f8a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 22:55:50 +0100 Subject: [PATCH 25/39] docs: Update docs for creating custom handlers --- docs/usage/handlers.md | 48 ++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/usage/handlers.md b/docs/usage/handlers.md index 37da4b67..dcf4c5e3 100644 --- a/docs/usage/handlers.md +++ b/docs/usage/handlers.md @@ -14,13 +14,10 @@ A handler is what makes it possible to collect and render documentation for a pa ## About the Python handlers -Since version 0.18, a new, experimental Python handler is available. +Since version 0.18, a new Python handler is available. It is based on [Griffe](https://github.com/mkdocstrings/griffe), which is an improved version of [pytkdocs](https://github.com/mkdocstrings/pytkdocs). -Note that the experimental handler does not yet support all third-party libraries -that the legacy handler supported. - If you want to keep using the legacy handler as long as possible, you can depend on `mkdocstrings-python-legacy` directly, or specify the `python-legacy` extra when depending on *mkdocstrings*: @@ -37,9 +34,9 @@ dependencies = [ The legacy handler will continue to "work" for many releases, as long as the new handler does not cover all previous use-cases. -### Migrate to the experimental Python handler +### Migrate to the new Python handler -To use the new, experimental Python handler, +To use the new Python handler, you can depend on `mkdocstrings-python` directly, or specify the `python` extra when depending on *mkdocstrings*: @@ -131,26 +128,41 @@ NOTE: **Note the absence of `__init__.py` module in `mkdocstrings_handlers`!** ### Code A handler is a subclass of the base handler provided by *mkdocstrings*. - See the documentation for the [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler]. -Subclasses of the base handler must implement the `collect` and `render` methods at least. -The `collect` method is responsible for collecting and returning data (extracting -documentation from source code, loading introspecting objects in memory, other sources? etc.) -while the `render` method is responsible for actually rendering the data to HTML, -using the Jinja templates provided by your package. -You must implement a `get_handler` method at the module level. +Subclasses of the base handler must declare a `name` and `domain` as class attributes, +as well as implement the following methods: + +- `collect(identifier, options)` (**required**): method responsible for collecting and returning data (extracting + documentation from source code, loading introspecting objects in memory, other sources? etc.) +- `render(identifier, options)` (**required**): method responsible for actually rendering the data to HTML, + using the Jinja templates provided by your package. +- `get_options(local_options)` (**required**): method responsible for combining global options with local ones. +- `get_aliases(identifier)` (**recommended**): method responsible for returning known aliases of object identifiers, + in order to register cross-references in the autorefs plugin. +- `get_inventory_urls()` (optional): method responsible for returning a list of URLs to download (object inventories) + along with configuration options (for loading the inventory with `load_inventory`). +- `load_inventory(in_file, url, **options)` (optional): method responsible for loading an inventory (binary file-handle) + and yielding tuples of identifiers and URLs. +- `update_env(config)` (optional): Gives you a chance to customize the Jinja environment used to render templates, + for examples by adding/removing Jinja filters and global context variables. +- `teardown()` (optional): Clean up / teardown anything that needs it at the end of the build. + +You must implement a `get_handler` method at the module level, +which returns an instance of your handler. This function takes the following parameters: - `theme` (string, theme name) - `custom_templates` (optional string, path to custom templates directory) -- `config_file_path` (optional string, path to the config file) +- `mdx` (list, Markdown extensions) +- `mdx_config` (dict, extensions configuration) +- `handler_config` (dict, handle configuration) +- `tool_config` (dict, the whole MkDocs configuration) These arguments are all passed as keyword arguments, so you can ignore them -by adding `**kwargs` or similar to your signature. You can also accept -additional parameters: the handler's global-only options and/or the root -config options. This gives flexibility and access to the mkdocs config, mkdocstring -config etc.. You should never modify the root config but can use it to get +by adding `**kwargs` or similar to your signature. + +You should not modify the MkDocs config but can use it to get information about the MkDocs instance such as where the current `site_dir` lives. See the [Mkdocs Configuration](https://www.mkdocs.org/user-guide/configuration/) for more info about what is accessible from it. From 8eb71e373ba14a6d1964c1a279d3a54373094be6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 22:57:39 +0100 Subject: [PATCH 26/39] chore: Fix detection of `get_options` user implementation --- src/mkdocstrings/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index ab7e12b5..93fd5b20 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -170,7 +170,7 @@ def _process_block( local_options["heading_level"] = heading_level # YORE: Bump 1: Replace block with line 2. - if handler.get_options is not BaseHandler.get_options: + if handler.get_options.__func__ is not BaseHandler.get_options: # type: ignore[attr-defined] options = handler.get_options(local_options) else: warn( From 848d331f981eea085d68345d1592af2abec9d567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 22:59:27 +0100 Subject: [PATCH 27/39] chore: Mark `fallback_config` with Yore removal comment --- src/mkdocstrings/extension.py | 2 +- src/mkdocstrings/handlers/base.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 93fd5b20..4dcbdf99 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -258,7 +258,7 @@ def _process_headings(self, handler: BaseHandler, element: Element) -> None: stacklevel=1, ) try: - data_object = handler.collect(rendered_id, handler.fallback_config) + data_object = handler.collect(rendered_id, getattr(handler, "fallback_config", {})) except CollectionError: aliases = () else: diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index aa924910..d4428d91 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -106,6 +106,7 @@ class BaseHandler: enable_inventory: ClassVar[bool] = False """Whether the inventory creation is enabled.""" + # YORE: Bump 1: Remove block. fallback_config: ClassVar[dict] = {} """Fallback configuration when searching anchors for identifiers.""" @@ -593,7 +594,7 @@ def get_anchors(self, identifier: str) -> tuple[str, ...]: DeprecationWarning, stacklevel=1, ) - aliases = handler.get_anchors(handler.collect(identifier, handler.fallback_config)) + aliases = handler.get_anchors(handler.collect(identifier, getattr(handler, "fallback_config", {}))) else: aliases = handler.get_aliases(identifier) except CollectionError: From b84653f2b175824c73bd0291fafff8343ba80125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 23:26:11 +0100 Subject: [PATCH 28/39] refactor: Give back inventory control to handlers With this change, handlers gain back control on inventory configuration. They can expose which URLs to download, and which options to use when loading inventories through their existing `load_inventory` method, thanks to the new `get_inventory_urls` method. Handlers will have to store inventory configuration instead of relying on mkdocstrings' hardcoded use of an `import` setting. This change also moves inventory-related code into the `Handlers` class, and renames the `_cache` module to `_download`. Related-to-issue-719: https://github.com/mkdocstrings/mkdocstrings/issues/719 --- src/mkdocstrings/{_cache.py => _download.py} | 0 src/mkdocstrings/handlers/base.py | 70 +++++++++++++++++- src/mkdocstrings/plugin.py | 74 ++------------------ tests/{test_cache.py => test_download.py} | 20 +++--- tests/test_inventory.py | 12 ---- 5 files changed, 85 insertions(+), 91 deletions(-) rename src/mkdocstrings/{_cache.py => _download.py} (100%) rename tests/{test_cache.py => test_download.py} (80%) diff --git a/src/mkdocstrings/_cache.py b/src/mkdocstrings/_download.py similarity index 100% rename from src/mkdocstrings/_cache.py rename to src/mkdocstrings/_download.py diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index d4428d91..c245e1b4 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -5,9 +5,12 @@ from __future__ import annotations +import datetime import importlib import inspect import sys +from concurrent import futures +from io import BytesIO from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, cast from warnings import warn @@ -19,6 +22,10 @@ from markupsafe import Markup from mkdocs_autorefs.references import AutorefsInlineProcessor +# TODO: Replace with `from mkdocs.utils.cache import download_and_cache_url` when we depend on mkdocs>=1.5. +from mkdocs_get_deps.cache import download_and_cache_url + +from mkdocstrings._download import download_url_with_gz from mkdocstrings.handlers.rendering import ( HeadingShiftingTreeprocessor, Highlighter, @@ -27,7 +34,7 @@ ParagraphStrippingTreeprocessor, ) from mkdocstrings.inventory import Inventory -from mkdocstrings.loggers import get_template_logger +from mkdocstrings.loggers import get_logger, get_template_logger # TODO: remove once support for Python 3.9 is dropped if sys.version_info < (3, 10): @@ -41,6 +48,7 @@ from markdown import Extension from mkdocs_autorefs.references import AutorefsHookInterface +log = get_logger(__name__) CollectorItem = Any HandlerConfig = Any @@ -248,6 +256,10 @@ def md(self) -> Markdown: raise RuntimeError("Markdown instance not set yet") return self._md + def get_inventory_urls(self) -> list[tuple[str, dict[str, Any]]]: + """Return the URLs (and configuration options) of the inventory files to download.""" + return [] + @classmethod def load_inventory( cls, @@ -574,6 +586,8 @@ def __init__( self.inventory: Inventory = Inventory(project=inventory_project, version=inventory_version) + self._inv_futures: dict[futures.Future, tuple[BaseHandler, str, Any]] = {} + # YORE: Bump 1: Remove block. def get_anchors(self, identifier: str) -> tuple[str, ...]: """Return the canonical HTML anchor for the identifier, if any of the seen handlers can collect it. @@ -655,6 +669,58 @@ def get_handler(self, name: str, handler_config: dict | None = None) -> BaseHand ) return self._handlers[name] + def _download_inventories(self) -> None: + """Download an inventory file from an URL. + + Arguments: + url: The URL of the inventory. + """ + to_download: list[tuple[BaseHandler, str, Any]] = [] + + for handler_name, conf in self._handlers_config.items(): + handler = self.get_handler(handler_name) + + if handler.get_inventory_urls.__func__ is BaseHandler.get_inventory_urls: # type: ignore[attr-defined] + if inv_configs := conf.pop("import", ()): + warn( + "mkdocstrings v1 will stop handling 'import' in handlers configuration. " + "Instead your handler must define a `get_inventory_urls` method " + "that returns a list of URLs to download. ", + DeprecationWarning, + stacklevel=1, + ) + inv_configs = [{"url": inv} if isinstance(inv, str) else inv for inv in inv_configs] + inv_configs = [(inv.pop("url"), inv) for inv in inv_configs] + else: + inv_configs = handler.get_inventory_urls() + + to_download.extend((handler, url, conf) for url, conf in inv_configs) + + if to_download: + thread_pool = futures.ThreadPoolExecutor(4) + for handler, url, conf in to_download: + log.debug("Downloading inventory from %s", url) + future = thread_pool.submit( + download_and_cache_url, + url, + datetime.timedelta(days=1), + download=download_url_with_gz, + ) + self._inv_futures[future] = (handler, url, conf) + thread_pool.shutdown(wait=False) + + def _yield_inventory_items(self) -> Iterator[tuple[str, str]]: + if self._inv_futures: + log.debug("Waiting for %s inventory download(s)", len(self._inv_futures)) + futures.wait(self._inv_futures, timeout=30) + # Reversed order so that pages from first futures take precedence: + for fut, (handler, url, conf) in reversed(self._inv_futures.items()): + try: + yield from handler.load_inventory(BytesIO(fut.result()), url, **conf) + except Exception as error: # noqa: BLE001 + log.error("Couldn't load inventory %s through handler '%s': %s", conf, handler.name, error) # noqa: TRY400 + self._inv_futures = {} + @property def seen_handlers(self) -> Iterable[BaseHandler]: """Get the handlers that were encountered so far throughout the build. @@ -667,6 +733,8 @@ def seen_handlers(self) -> Iterable[BaseHandler]: def teardown(self) -> None: """Teardown all cached handlers and clear the cache.""" + for future in self._inv_futures: + future.cancel() for handler in self.seen_handlers: handler.teardown() self._handlers.clear() diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 844ee0f4..8f931fef 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -14,13 +14,9 @@ from __future__ import annotations -import datetime -import functools import os import sys from collections.abc import Iterable, Mapping -from concurrent import futures -from io import BytesIO from typing import TYPE_CHECKING, Any, Callable, TypeVar from mkdocs.config import Config @@ -29,10 +25,6 @@ from mkdocs.utils import write_file from mkdocs_autorefs.plugin import AutorefsConfig, AutorefsPlugin -# TODO: Replace with `from mkdocs.utils.cache import download_and_cache_url` when we depend on mkdocs>=1.5. -from mkdocs_get_deps.cache import download_and_cache_url - -from mkdocstrings._cache import download_url_with_gz from mkdocstrings.extension import MkdocstringsExtension from mkdocstrings.handlers.base import BaseHandler, Handlers from mkdocstrings.loggers import get_logger @@ -160,13 +152,6 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: return config log.debug("Adding extension to the list") - to_import: InventoryImportType = [] - for handler_name, conf in self.config.handlers.items(): - for import_item in conf.pop("import", ()): - if isinstance(import_item, str): - import_item = {"url": import_item} # noqa: PLW2901 - to_import.append((handler_name, import_item)) - handlers = Handlers( default=self.config.default_handler, handlers_config=self.config.handlers, @@ -179,6 +164,8 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: tool_config=config, ) + handlers._download_inventories() + autorefs: AutorefsPlugin try: # If autorefs plugin is explicitly enabled, just use it. @@ -200,20 +187,6 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: config.extra_css.insert(0, self.css_filename) # So that it has lower priority than user files. self._handlers = handlers - - self._inv_futures = {} - if to_import: - inv_loader = futures.ThreadPoolExecutor(4) - for handler_name, import_item in to_import: - loader = handlers.get_handler(handler_name).load_inventory - future = inv_loader.submit( - self._load_inventory, # type: ignore[misc] - loader, - **import_item, - ) - self._inv_futures[future] = (loader, import_item) - inv_loader.shutdown(wait=False) - return config @property @@ -247,6 +220,7 @@ def on_env(self, env: Environment, config: MkDocsConfig, *args: Any, **kwargs: A """ if not self.plugin_enabled: return + if self._handlers: css_content = "\n".join(handler.extra_css for handler in self.handlers.seen_handlers) write_file(css_content.encode("utf-8"), os.path.join(config.site_dir, self.css_filename)) @@ -256,21 +230,9 @@ def on_env(self, env: Environment, config: MkDocsConfig, *args: Any, **kwargs: A inv_contents = self.handlers.inventory.format_sphinx() write_file(inv_contents, os.path.join(config.site_dir, "objects.inv")) - if self._inv_futures: - log.debug("Waiting for %s inventory download(s)", len(self._inv_futures)) - futures.wait(self._inv_futures, timeout=30) - results = {} - # Reversed order so that pages from first futures take precedence: - for fut in reversed(list(self._inv_futures)): - try: - results.update(fut.result()) - except Exception as error: # noqa: BLE001 - loader, import_item = self._inv_futures[fut] - loader_name = loader.__func__.__qualname__ - log.error("Couldn't load inventory %s through %s: %s", import_item, loader_name, error) # noqa: TRY400 - for page, identifier in results.items(): - config.plugins["autorefs"].register_url(page, identifier) # type: ignore[attr-defined] - self._inv_futures = {} + register = config.plugins["autorefs"].register_url # type: ignore[attr-defined] + for identifier, url in self._handlers._yield_inventory_items(): + register(identifier, url) def on_post_build( self, @@ -293,9 +255,6 @@ def on_post_build( if not self.plugin_enabled: return - for future in self._inv_futures: - future.cancel() - if self._handlers: log.debug("Tearing handlers down") self.handlers.teardown() @@ -310,24 +269,3 @@ def get_handler(self, handler_name: str) -> BaseHandler: An instance of a subclass of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler]. """ return self.handlers.get_handler(handler_name) - - @classmethod - # lru_cache does not allow mutable arguments such lists, but that is what we load from YAML config. - @list_to_tuple - @functools.cache - def _load_inventory(cls, loader: InventoryLoaderType, url: str, **kwargs: Any) -> Mapping[str, str]: - """Download and process inventory files using a handler. - - Arguments: - loader: A function returning a sequence of pairs (identifier, url). - url: The URL to download and process. - **kwargs: Extra arguments to pass to the loader. - - Returns: - A mapping from identifier to absolute URL. - """ - log.debug("Downloading inventory from %s", url) - content = download_and_cache_url(url, datetime.timedelta(days=1), download=download_url_with_gz) - result = dict(loader(BytesIO(content), url=url, **kwargs)) - log.debug("Loaded inventory from %s: %s items", url, len(result)) - return result diff --git a/tests/test_cache.py b/tests/test_download.py similarity index 80% rename from tests/test_cache.py rename to tests/test_download.py index b56e3d3c..95dc0233 100644 --- a/tests/test_cache.py +++ b/tests/test_download.py @@ -1,4 +1,4 @@ -"""Tests for the internal mkdocstrings _cache module.""" +"""Tests for the internal mkdocstrings _download module.""" from __future__ import annotations @@ -7,7 +7,7 @@ import pytest -from mkdocstrings import _cache +from mkdocstrings import _download if TYPE_CHECKING: from collections.abc import Mapping @@ -32,16 +32,16 @@ ) def test_expand_env_vars(credential: str, expected: str, env: Mapping[str, str]) -> None: """Test expanding environment variables.""" - assert _cache._expand_env_vars(credential, url="https://test.example.com", env=env) == expected + assert _download._expand_env_vars(credential, url="https://test.example.com", env=env) == expected def test_expand_env_vars_with_missing_env_var(caplog: pytest.LogCaptureFixture) -> None: """Test expanding environment variables with a missing environment variable.""" - caplog.set_level(logging.WARNING, logger="mkdocs.plugins.mkdocstrings._cache") + caplog.set_level(logging.WARNING, logger="mkdocs.plugins.mkdocstrings._download") credential = "${USER}" env: dict[str, str] = {} - assert _cache._expand_env_vars(credential, url="https://test.example.com", env=env) == "${USER}" + assert _download._expand_env_vars(credential, url="https://test.example.com", env=env) == "${USER}" output = caplog.records[0].getMessage() assert "'USER' is not set" in output @@ -61,20 +61,20 @@ def test_expand_env_vars_with_missing_env_var(caplog: pytest.LogCaptureFixture) ) def test_extract_auth_from_url(monkeypatch: pytest.MonkeyPatch, url: str, expected_url: str) -> None: """Test extracting the auth part from the URL.""" - monkeypatch.setattr(_cache, "_create_auth_header", lambda *args, **kwargs: {}) - result_url, _result_auth_header = _cache._extract_auth_from_url(url) + monkeypatch.setattr(_download, "_create_auth_header", lambda *args, **kwargs: {}) + result_url, _result_auth_header = _download._extract_auth_from_url(url) assert result_url == expected_url def test_create_auth_header_basic_auth() -> None: """Test creating the Authorization header for basic authentication.""" - auth_header = _cache._create_auth_header(credential="testuser:testpass", url="https://test.example.com") + auth_header = _download._create_auth_header(credential="testuser:testpass", url="https://test.example.com") assert auth_header == {"Authorization": "Basic dGVzdHVzZXI6dGVzdHBhc3M="} def test_create_auth_header_bearer_auth() -> None: """Test creating the Authorization header for bearer token authentication.""" - auth_header = _cache._create_auth_header(credential="token123", url="https://test.example.com") + auth_header = _download._create_auth_header(credential="token123", url="https://test.example.com") assert auth_header == {"Authorization": "Bearer token123"} @@ -96,7 +96,7 @@ def test_create_auth_header_bearer_auth() -> None: ) def test_env_var_pattern(var: str, match: str | None) -> None: """Test the environment variable regex pattern.""" - _match = _cache.ENV_VAR_PATTERN.match(var) + _match = _download.ENV_VAR_PATTERN.match(var) if _match is None: assert match is _match else: diff --git a/tests/test_inventory.py b/tests/test_inventory.py index ce707296..ecbb3cd2 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -5,7 +5,6 @@ import sys from io import BytesIO from os.path import join -from typing import TYPE_CHECKING import pytest from mkdocs.commands.build import build @@ -13,8 +12,6 @@ from mkdocstrings.inventory import Inventory, InventoryItem -if TYPE_CHECKING: - from mkdocstrings.plugin import MkdocstringsPlugin sphinx = pytest.importorskip("sphinx.util.inventory", reason="Sphinx is not installed") @@ -58,12 +55,3 @@ def test_sphinx_load_mkdocstrings_inventory_file() -> None: for item in own_inv.values(): assert item.name in sphinx_inv[f"{item.domain}:{item.role}"] - - -def test_load_inventory(plugin: MkdocstringsPlugin) -> None: - """Test the plugin inventory loading method. - - Parameters: - plugin: A mkdocstrings plugin instance. - """ - plugin._load_inventory(loader=lambda *args, **kwargs: (), url="https://example.com", domains=["a", "b"]) # type: ignore[misc,arg-type] From 8fa19891a4813d0b84cf3a0dc86e91a6590af2b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 23:26:26 +0100 Subject: [PATCH 29/39] tests: Fix test while waiting for mkdocstrings-python release --- tests/test_extension.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_extension.py b/tests/test_extension.py index c3784b2e..b7f1685f 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -158,6 +158,8 @@ def test_register_every_identifier_alias(plugin: MkdocstringsPlugin, ext_markdow """Assert that we don't preemptively register all identifiers of a rendered object.""" handler = plugin._handlers.get_handler("python") # type: ignore[union-attr] ids = ("id1", "id2", "id3") + # TODO: Remove line when Python handler removes its `get_anchors` method. + handler.get_anchors = lambda _: ids # type: ignore[union-attr] handler.get_aliases = lambda _: ids # type: ignore[method-assign] autorefs = ext_markdown.parser.blockprocessors["mkdocstrings"]._autorefs # type: ignore[attr-defined] autorefs.current_page = "foo" From 9383fecf8312e032835733b6d406ac14ce00af32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 23:26:34 +0100 Subject: [PATCH 30/39] tests: Ignore deprecation warnings --- config/pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/pytest.ini b/config/pytest.ini index edcaffb5..459c99de 100644 --- a/config/pytest.ini +++ b/config/pytest.ini @@ -19,3 +19,5 @@ filterwarnings = ignore:.*`update_env\(md\)` parameter:DeprecationWarning:mkdocstrings ignore:.*`super\(\).update_env\(\)` anymore:DeprecationWarning:mkdocstrings_handlers ignore:.*`get_anchors` method:DeprecationWarning:mkdocstrings + ignore:.*fallback anchor function:DeprecationWarning:mkdocstrings + ignore:.*v1.*`get_options` method:DeprecationWarning:mkdocstrings \ No newline at end of file From 4105a92b2cb9e0cefd80068ca7d22bf66f5445a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 12 Jan 2025 23:34:36 +0100 Subject: [PATCH 31/39] style: Format --- scripts/insiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/insiders.py b/scripts/insiders.py index 849c6314..a7da99bc 100644 --- a/scripts/insiders.py +++ b/scripts/insiders.py @@ -26,7 +26,7 @@ def human_readable_amount(amount: int) -> str: # noqa: D103 str_amount = str(amount) if len(str_amount) >= 4: # noqa: PLR2004 - return f"{str_amount[:len(str_amount)-3]},{str_amount[-3:]}" + return f"{str_amount[: len(str_amount) - 3]},{str_amount[-3:]}" return str_amount From 060e437485fa5026d70c941a17df4049ebbb904e Mon Sep 17 00:00:00 2001 From: Jeremy Feng <44312563+jeremy-feng@users.noreply.github.com> Date: Tue, 14 Jan 2025 20:30:32 +0800 Subject: [PATCH 32/39] docs: Fix link to arithmatex docs in Material for MkDocs --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d531997a..5e5386e3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -145,7 +145,7 @@ See [Python handler: Finding modules](https://mkdocstrings.github.io/python/usag ### LaTeX in docstrings is not rendered correctly If you are using a Markdown extension like -[Arithmatex Mathjax](https://squidfunk.github.io/mkdocs-material/extensions/pymdown/#arithmatex-mathjax) +[Arithmatex Mathjax](https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown-extensions/#arithmatex) or [`markdown-katex`][markdown-katex] to render LaTeX, add `r` in front of your docstring to make sure nothing is escaped. You'll still maybe have to play with escaping to get things right. From 48625a2d37c90653ac275031dc8a3cf1c607408b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Sun, 19 Jan 2025 14:20:12 +0100 Subject: [PATCH 33/39] chore: Fix inventory loading error log message --- src/mkdocstrings/handlers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index c245e1b4..c9436ab1 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -718,7 +718,7 @@ def _yield_inventory_items(self) -> Iterator[tuple[str, str]]: try: yield from handler.load_inventory(BytesIO(fut.result()), url, **conf) except Exception as error: # noqa: BLE001 - log.error("Couldn't load inventory %s through handler '%s': %s", conf, handler.name, error) # noqa: TRY400 + log.error("Couldn't load inventory %s through handler '%s': %s", url, handler.name, error) # noqa: TRY400 self._inv_futures = {} @property From 8c476ee0b82c09a5b20d7a773ecaf4be17b9e4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 30 Jan 2025 17:46:44 +0100 Subject: [PATCH 34/39] refactor: Pass `config_file_path` to `get_handler` if it expects it --- config/pytest.ini | 3 ++- src/mkdocstrings/handlers/base.py | 37 ++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/config/pytest.ini b/config/pytest.ini index 459c99de..b54cfdfa 100644 --- a/config/pytest.ini +++ b/config/pytest.ini @@ -20,4 +20,5 @@ filterwarnings = ignore:.*`super\(\).update_env\(\)` anymore:DeprecationWarning:mkdocstrings_handlers ignore:.*`get_anchors` method:DeprecationWarning:mkdocstrings ignore:.*fallback anchor function:DeprecationWarning:mkdocstrings - ignore:.*v1.*`get_options` method:DeprecationWarning:mkdocstrings \ No newline at end of file + ignore:.*v1.*`get_options` method:DeprecationWarning:mkdocstrings + ignore:.*`config_file_path` argument:DeprecationWarning:mkdocstrings diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index c9436ab1..6929c0c1 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -659,14 +659,35 @@ def get_handler(self, name: str, handler_config: dict | None = None) -> BaseHand if handler_config is None: handler_config = self._handlers_config.get(name, {}) module = importlib.import_module(f"mkdocstrings_handlers.{name}") - self._handlers[name] = module.get_handler( - theme=self._theme, - custom_templates=self._custom_templates, - mdx=self._mdx, - mdx_config=self._mdx_config, - handler_config=handler_config, - tool_config=self._tool_config, - ) + + # YORE: Bump 1: Remove block. + kwargs = { + "theme": self._theme, + "custom_templates": self._custom_templates, + "mdx": self._mdx, + "mdx_config": self._mdx_config, + "handler_config": handler_config, + "tool_config": self._tool_config, + } + if "config_file_path" in inspect.signature(module.get_handler).parameters: + kwargs["config_file_path"] = self._tool_config.get("config_file_path") + warn( + "The `config_file_path` argument in `get_handler` functions is deprecated. " + "Use `tool_config.get('config_file_path')` instead.", + DeprecationWarning, + stacklevel=1, + ) + self._handlers[name] = module.get_handler(**kwargs) + + # YORE: Bump 1: Replace `# ` with `` within block. + # self._handlers[name] = module.get_handler( + # theme=self._theme, + # custom_templates=self._custom_templates, + # mdx=self._mdx, + # mdx_config=self._mdx_config, + # handler_config=handler_config, + # tool_config=self._tool_config, + # ) return self._handlers[name] def _download_inventories(self) -> None: From 52ea2f216b6481c625c2ee15333cb64ec386b370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Thu, 30 Jan 2025 17:58:30 +0100 Subject: [PATCH 35/39] ci: Upgrade dev-deps to prevent warnings in tests --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 49b3b818..87368ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,8 +115,8 @@ dev = [ "mkdocs-literate-nav>=0.6", "mkdocs-material>=9.5", "mkdocs-minify-plugin>=0.8", - "mkdocs-redirects>=1.2", - "mkdocstrings[python]>=0.25", + "mkdocs-redirects>=1.2.1", + "mkdocstrings-python>=1.13", # YORE: EOL 3.10: Remove line. "tomli>=2.0; python_version < '3.11'", ] \ No newline at end of file From 9618e17a66cd6bf3cdf4445dddde97c1680039a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Fri, 31 Jan 2025 13:11:37 +0100 Subject: [PATCH 36/39] docs: Mention mkdocs-api-autonav in recipe --- docs/recipes.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index 6a81b090..a52347bd 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -3,7 +3,7 @@ for *mkdocstrings* and more generally Markdown documentation. ## Automatic code reference pages -TIP: **[mkdocs-autoapi](https://github.com/jcayers20/mkdocs-autoapi) is a MkDocs plugin that automatically generates API documentation from your project's source code. It was inspired by the recipe below.** +TIP: **[mkdocs-autoapi](https://github.com/jcayers20/mkdocs-autoapi) and [mkdocs-api-autonav](https://github.com/tlambert03/mkdocs-api-autonav) are MkDocs plugins that automatically generate API documentation from your project's source code. They were inspired by the recipe below.** *mkdocstrings* allows to inject documentation for any object into Markdown pages. But as the project grows, it quickly becomes @@ -366,16 +366,16 @@ extra_css: > To target `pycon` code blocks more specifically, you can configure the > `pymdownx.highlight` extension to use Pygments and set language classes > on code blocks: -> +> > ```yaml title="mkdocs.yml" > markdown_extensions: > - pymdownx.highlight: > use_pygments: true > pygments_lang_class: true > ``` -> +> > Then you can update the CSS selector like this: -> +> > ```css title="docs/css/code_select.css" > .language-pycon .gp, .language-pycon .go { /* Generic.Prompt, Generic.Output */ > user-select: none; From f3ecf5868c33944b344a31a2423952d7f4eb1952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Mon, 3 Feb 2025 16:53:26 +0100 Subject: [PATCH 37/39] chore: Ignore autorefs fallback deprecation warning --- src/mkdocstrings/plugin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 8f931fef..9962f48d 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -18,6 +18,7 @@ import sys from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Callable, TypeVar +from warnings import catch_warnings, simplefilter from mkdocs.config import Config from mkdocs.config import config_options as opt @@ -178,8 +179,10 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: autorefs.scan_toc = False config.plugins["autorefs"] = autorefs log.debug("Added a subdued autorefs instance %r", autorefs) - # YORE: Bump 1: Remove line. - autorefs.get_fallback_anchor = handlers.get_anchors + # YORE: Bump 1: Remove block. + with catch_warnings(): + simplefilter("ignore", category=DeprecationWarning) + autorefs.get_fallback_anchor = handlers.get_anchors mkdocstrings_extension = MkdocstringsExtension(handlers, autorefs) config.markdown_extensions.append(mkdocstrings_extension) # type: ignore[arg-type] From 3cf7d51704378adc50d4ea50080aacae39e0e731 Mon Sep 17 00:00:00 2001 From: Matthew Messinger Date: Mon, 3 Feb 2025 10:57:09 -0500 Subject: [PATCH 38/39] fix: Update handlers in JSON schema to be an object instead of an array Issue-733: https://github.com/mkdocstrings/mkdocstrings/issues/733 PR-734: https://github.com/mkdocstrings/mkdocstrings/pull/734 --- docs/schema.json | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/schema.json b/docs/schema.json index a74dabf3..7632af66 100644 --- a/docs/schema.json +++ b/docs/schema.json @@ -37,15 +37,11 @@ "handlers": { "title": "The handlers global configuration.", "markdownDescription": "https://mkdocstrings.github.io/handlers/overview/", - "type": "object", - "default": null, - "items": { - "oneOf": [ - { - "$ref": "https://raw.githubusercontent.com/mkdocstrings/python/master/docs/schema.json" - } - ] - } + "anyOf": [ + { + "$ref": "https://raw.githubusercontent.com/mkdocstrings/python/main/docs/schema.json" + } + ] } }, "additionalProperties": false From 6ef141222d0b5ad47ced9049472243cf5887ec0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Mon, 3 Feb 2025 17:01:00 +0100 Subject: [PATCH 39/39] chore: Prepare release 0.28.0 --- CHANGELOG.md | 197 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a062c06b..e353cf6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,203 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [0.28.0](https://github.com/mkdocstrings/mkdocstrings/releases/tag/0.28.0) - 2025-02-03 + +[Compare with 0.27.0](https://github.com/mkdocstrings/mkdocstrings/compare/0.27.0...0.28.0) + +### Breaking Changes + +Although the following changes are "breaking" in terms of public API, we didn't find any public use of these classes and methods on GitHub. + +- `mkdocstrings.extension.AutoDocProcessor.__init__(parser)`: *Parameter was removed* +- `mkdocstrings.extension.AutoDocProcessor.__init__(md)`: *Positional parameter was moved* +- `mkdocstrings.extension.AutoDocProcessor.__init__(config)`: *Parameter was removed* +- `mkdocstrings.extension.AutoDocProcessor.__init__(handlers)`: *Parameter kind was changed*: `positional or keyword` -> `keyword-only` +- `mkdocstrings.extension.AutoDocProcessor.__init__(autorefs)`: *Parameter kind was changed*: `positional or keyword` -> `keyword-only` +- `mkdocstrings.extension.MkdocstringsExtension.__init__(config)`: *Parameter was removed* +- `mkdocstrings.extension.MkdocstringsExtension.__init__(handlers)`: *Positional parameter was moved* +- `mkdocstrings.extension.MkdocstringsExtension.__init__(autorefs)`: *Positional parameter was moved* +- `mkdocstrings.handlers.base.Handlers.__init__(config)`: *Parameter was removed* +- `mkdocstrings.handlers.base.Handlers.__init__(theme)`: *Parameter was added as required* +- `mkdocstrings.handlers.base.Handlers.__init__(default)`: *Parameter was added as required* +- `mkdocstrings.handlers.base.Handlers.__init__(inventory_project)`: *Parameter was added as required* +- `mkdocstrings.handlers.base.Handlers.__init__(tool_config)`: *Parameter was added as required* + +Similarly, the following parameters were renamed, but the methods are only called from our own code, using positional arguments. + +- `mkdocstrings.handlers.base.BaseHandler.collect(config)`: *Parameter was renamed `options`* +- `mkdocstrings.handlers.base.BaseHandler.render(config)`: *Parameter was renamed `options`* + +Finally, the following method was removed, but this is again taken into account in our own code: + +- `mkdocstrings.handlers.base.BaseHandler.get_anchors`: *Public object was removed* + +For these reasons, and because we're still in v0, we do not bump to v1 yet. See following deprecations. + +### Deprecations + +*mkdocstrings* 0.28 will start emitting these deprecations warnings: + +> The `handler` argument is deprecated. The handler name must be specified as a class attribute. + +Previously, the `get_handler` function would pass a `handler` (name) argument to the handler constructor. This name must now be set on the handler's class directly. + +```python +class MyHandler: + name = "myhandler" +``` + +> The `domain` attribute must be specified as a class attribute. + +The `domain` class attribute on handlers is now mandatory and cannot be an empty string. + +```python +class MyHandler: + domain = "mh" +``` + +> The `theme` argument must be passed as a keyword argument. + +This argument could previously be passed as a positional argument (from the `get_handler` function), and must now be passed as a keyword argument. + +> The `custom_templates` argument must be passed as a keyword argument. + +Same as for `theme`, but with `custom_templates`. + +> The `mdx` argument must be provided (as a keyword argument). + +The `get_handler` function now receives a `mdx` argument, which it must forward to the handler constructor and then to the base handler, either explicitly or through `**kwargs`: + +=== "Explicitly" + + ```python + def get_handler(..., mdx, ...): + return MyHandler(..., mdx=mdx, ...) + + + class MyHandler: + def __init__(self, ..., mdx, ...): + super().__init__(..., mdx=mdx, ...) + ``` + +=== "Through `**kwargs`" + + ```python + def get_handler(..., **kwargs): + return MyHandler(..., **kwargs) + + + class MyHandler: + def __init__(self, ..., **kwargs): + super().__init__(**kwargs) + ``` + +In the meantime we still retrieve this `mdx` value at a different moment, by reading it from the MkDocs configuration. + +> The `mdx_config` argument must be provided (as a keyword argument). + +Same as for `mdx`, but with `mdx_config`. + +> mkdocstrings v1 will stop handling 'import' in handlers configuration. Instead your handler must define a `get_inventory_urls` method that returns a list of URLs to download. + +Previously, mkdocstrings would pop the `import` key from a handler's configuration to download each item (URLs). Items could be strings, or dictionaries with a `url` key. Now mkdocstrings gives back control to handlers, which must store this inventory configuration within them, and expose it again through a `get_inventory_urls` method. This method returns a list of tuples: an URL, and a dictionary of options that will be passed again to their `load_inventory` method. Handlers have now full control over the "inventory" setting. + +```python +from copy import deepcopy + + +def get_handler(..., handler_config, ...): + return MyHandler(..., config=handler_config, ...) + + +class MyHandler: + def __init__(self, ..., config, ...): + self.config = config + + def get_inventory_urls(self): + config = deepcopy(self.config["import"]) + return [(inv, {}) if isinstance(inv, str) else (inv.pop("url"), inv) for inv in config] +``` + +Changing the name of the key (for example from `import` to `inventories`) involves a change in user configuration, and both keys will have to be supported by your handler for some time. + +```python +def get_handler(..., handler_config, ...): + if "inventories" not in handler_config and "import" in handler_config: + warn("The 'import' key is renamed 'inventories'", FutureWarning) + handler_config["inventories"] = handler_config.pop("import") + return MyHandler(..., config=handler_config, ...) +``` + +> Setting a fallback anchor function is deprecated and will be removed in a future release. + +This comes from mkdocstrings and mkdocs-autorefs, and will disappear with mkdocstrings v0.28. + +> mkdocstrings v1 will start using your handler's `get_options` method to build options instead of merging the global and local options (dictionaries). + +Handlers must now store their own global options (in an instance attribute), and implement a `get_options` method that receives `local_options` (a dict) and returns combined options (dict or custom object). These combined options are then passed to `collect` and `render`, so that these methods can use them right away. + +```python +def get_handler(..., handler_config, ...): + return MyHandler(..., config=handler_config, ...) + + +class MyHandler: + def __init__(self, ..., config, ...): + self.config = config + + def get_options(local_options): + return {**self.default_options, **self.config["options"], **local_options} +``` + +> The `update_env(md)` parameter is deprecated. Use `self.md` instead. + +Handlers can remove the `md` parameter from their `update_env` method implementation, and use `self.md` instead, if they need it. + +> No need to call `super().update_env()` anymore. + +Handlers don't have to call the parent `update_env` method from their own implementation anymore, and can just drop the call. + +> The `get_anchors` method is deprecated. Declare a `get_aliases` method instead, accepting a string (identifier) instead of a collected object. + +Previously, handlers would implement a `get_anchors` method that received a data object (typed `CollectorItem`) to return aliases for this object. This forced mkdocstrings to collect this object through the handler's `collect` method, which then required some logic with "fallback config" as to prevent unwanted collection. mkdocstrings gives back control to handlers and now calls `get_aliases` instead, which accepts an `identifier` (string) and lets the handler decide how to return aliases for this identifier. For example, it can replicate previous behavior by calling its own `collect` method with its own "fallback config", or do something different (cache lookup, etc.). + +```python +class MyHandler: + def get_aliases(identifier): + try: + obj = self.collect(identifier, self.fallback_config) + # or obj = self._objects_cache[identifier] + except CollectionError: # or KeyError + return () + return ... # previous logic in `get_anchors` +``` + +> The `config_file_path` argument in `get_handler` functions is deprecated. Use `tool_config.get('config_file_path')` instead. + +The `config_file_path` argument is now deprecated and only passed to `get_handler` functions if they accept it. If you used it to compute a "base directory", you can now use the `tool_config` argument instead, which is the configuration of the SSG tool in use (here MkDocs): + +```python +base_dir = Path(tool_config.config_file_path or "./mkdocs.yml").parent +``` + +**Most of these warnings will disappear with the next version of mkdocstrings-python.** + +### Bug Fixes + +- Update handlers in JSON schema to be an object instead of an array ([3cf7d51](https://github.com/mkdocstrings/mkdocstrings/commit/3cf7d51704378adc50d4ea50080aacae39e0e731) by Matthew Messinger). [Issue-733](https://github.com/mkdocstrings/mkdocstrings/issues/733), [PR-734](https://github.com/mkdocstrings/mkdocstrings/pull/734) +- Fix broken table of contents when nesting autodoc instructions ([12c8f82](https://github.com/mkdocstrings/mkdocstrings/commit/12c8f82e9a959ce32cada09f0d2b5c651a705fdb) by Timothée Mazzucotelli). [Issue-348](https://github.com/mkdocstrings/mkdocstrings/issues/348) + +### Code Refactoring + +- Pass `config_file_path` to `get_handler` if it expects it ([8c476ee](https://github.com/mkdocstrings/mkdocstrings/commit/8c476ee0b82c09a5b20d7a773ecaf4be17b9e4d1) by Timothée Mazzucotelli). +- Give back inventory control to handlers ([b84653f](https://github.com/mkdocstrings/mkdocstrings/commit/b84653f2b175824c73bd0291fafff8343ba80125) by Timothée Mazzucotelli). [Related-to-issue-719](https://github.com/mkdocstrings/mkdocstrings/issues/719) +- Give back control to handlers on how they want to handle global/local options ([c00de7a](https://github.com/mkdocstrings/mkdocstrings/commit/c00de7a42b9072cbaa47ecbf18e3e15a6d5ab634) by Timothée Mazzucotelli). [Issue-719](https://github.com/mkdocstrings/mkdocstrings/issues/719) +- Deprecate base handler's `get_anchors` method in favor of `get_aliases` method ([7a668f0](https://github.com/mkdocstrings/mkdocstrings/commit/7a668f0f731401b07123bd02aafbbfc55cd24c0d) by Timothée Mazzucotelli). +- Register all identifiers of rendered objects into autorefs ([434d8c7](https://github.com/mkdocstrings/mkdocstrings/commit/434d8c7cd1e3edbdb9d4c45a9b44b290b19d88f1) by Timothée Mazzucotelli). +- Use mkdocs-get-deps' download utility to remove duplicated code ([bb87cd8](https://github.com/mkdocstrings/mkdocstrings/commit/bb87cd833f2333e77cb2c2926aa24a434c97391f) by Timothée Mazzucotelli). +- Clean up data passed down from plugin to extension and handlers ([b8e8703](https://github.com/mkdocstrings/mkdocstrings/commit/b8e87036e0e1ec5c181b4a2ec5931f1a60636a32) by Timothée Mazzucotelli). [PR-726](https://github.com/mkdocstrings/mkdocstrings/pull/726) + ## [0.27.0](https://github.com/mkdocstrings/mkdocstrings/releases/tag/0.27.0) - 2024-11-08 [Compare with 0.26.2](https://github.com/mkdocstrings/mkdocstrings/compare/0.26.2...0.27.0)