From 4122610b8f8c554855fa9a23eb8d5aaf0bfebba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 12 Jan 2021 19:29:54 +0100 Subject: [PATCH 01/40] docs: Update credits --- CREDITS.md | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/CREDITS.md b/CREDITS.md index 55579d1d..d6a07fd5 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -12,17 +12,23 @@ These projects were used to build `mkdocstrings`. **Thank you!** ### Direct dependencies [`autoflake`](https://github.com/myint/autoflake) | [`black`](https://github.com/psf/black) | +[`darglint`](None) | [`duty`](https://github.com/pawamoy/duty) | +[`flake8-bandit`](https://github.com/tylerwince/flake8-bandit) | [`flake8-black`](https://github.com/peterjc/flake8-black) | +[`flake8-bugbear`](https://github.com/PyCQA/flake8-bugbear) | [`flake8-builtins`](https://github.com/gforcada/flake8-builtins) | +[`flake8-comprehensions`](https://github.com/adamchainz/flake8-comprehensions) | +[`flake8-docstrings`](https://gitlab.com/pycqa/flake8-docstrings) | [`flake8-pytest-style`](https://pypi.org/project/flake8-pytest-style) | +[`flake8-string-format`](https://github.com/xZise/flake8-string-format) | [`flake8-tidy-imports`](https://github.com/adamchainz/flake8-tidy-imports) | [`flake8-variables-names`](https://github.com/best-doctor/flake8-variables-names) | [`flakehell`](None) | [`git-changelog`](https://github.com/pawamoy/git-changelog) | [`httpx`](https://github.com/encode/httpx) | [`ipython`](https://ipython.org) | -[`isort`](https://github.com/timothycrosley/isort) | +[`isort`](https://pycqa.github.io/isort/) | [`Jinja2`](https://palletsprojects.com/p/jinja/) | [`jinja2-cli`](https://github.com/mattrobenolt/jinja2-cli) | [`Markdown`](https://Python-Markdown.github.io/) | @@ -31,6 +37,7 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`mkdocs`](https://www.mkdocs.org) | [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/) | [`mypy`](http://www.mypy-lang.org/) | +[`pep8-naming`](https://github.com/PyCQA/pep8-naming) | [`pymdown-extensions`](https://github.com/facelessuser/pymdown-extensions) | [`pytest`](https://docs.pytest.org/en/latest/) | [`pytest-cov`](https://github.com/pytest-dev/pytest-cov) | @@ -38,15 +45,13 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`pytest-sugar`](http://pivotfinland.com/pytest-sugar/) | [`pytest-xdist`](https://github.com/pytest-dev/pytest-xdist) | [`pytkdocs`](https://github.com/pawamoy/pytkdocs) | -[`toml`](https://github.com/uiri/toml) | -[`wemake-python-styleguide`](https://wemake-python-stylegui.de) +[`toml`](https://github.com/uiri/toml) ### Indirect dependencies [`ansimarkup`](https://github.com/gvalkov/python-ansimarkup) | [`apipkg`](https://github.com/pytest-dev/apipkg) | [`appdirs`](http://github.com/ActiveState/appdirs) | [`appnope`](http://github.com/minrk/appnope) | -[`astor`](https://github.com/berkerpeksag/astor) | [`astroid`](https://github.com/PyCQA/astroid) | [`atomicwrites`](https://github.com/untitaker/python-atomicwrites) | [`attrs`](https://www.attrs.org/) | @@ -58,29 +63,14 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`colorama`](https://github.com/tartley/colorama) | [`contextvars`](http://github.com/MagicStack/contextvars) | [`coverage`](https://github.com/nedbat/coveragepy) | -[`darglint`](None) | [`dataclasses`](https://github.com/ericvsmith/dataclasses) | [`decorator`](https://github.com/micheles/decorator) | -[`docutils`](http://docutils.sourceforge.net/) | [`entrypoints`](https://github.com/takluyver/entrypoints) | -[`eradicate`](https://github.com/myint/eradicate) | [`execnet`](https://execnet.readthedocs.io/en/latest/) | [`failprint`](https://github.com/pawamoy/failprint) | [`flake8`](https://gitlab.com/pycqa/flake8) | -[`flake8-bandit`](https://github.com/tylerwince/flake8-bandit) | -[`flake8-broken-line`](https://github.com/sobolevn/flake8-broken-line) | -[`flake8-bugbear`](https://github.com/PyCQA/flake8-bugbear) | -[`flake8-commas`](https://github.com/PyCQA/flake8-commas/) | -[`flake8-comprehensions`](https://github.com/adamchainz/flake8-comprehensions) | -[`flake8-debugger`](https://github.com/jbkahn/flake8-debugger) | -[`flake8-docstrings`](https://gitlab.com/pycqa/flake8-docstrings) | -[`flake8-eradicate`](https://github.com/sobolevn/flake8-eradicate) | -[`flake8-isort`](https://github.com/gforcada/flake8-isort) | [`flake8-plugin-utils`](https://pypi.org/project/flake8-plugin-utils) | [`flake8-polyfill`](https://gitlab.com/pycqa/flake8-polyfill) | -[`flake8-quotes`](http://github.com/zheller/flake8-quotes/) | -[`flake8-rst-docstrings`](https://github.com/peterjc/flake8-rst-docstrings) | -[`flake8-string-format`](https://github.com/xZise/flake8-string-format) | [`future`](https://python-future.org) | [`gitdb`](https://github.com/gitpython-developers/gitdb) | [`GitPython`](https://github.com/gitpython-developers/GitPython) | @@ -104,7 +94,6 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`parso`](https://github.com/davidhalter/parso) | [`pathspec`](https://github.com/cpburnz/python-path-specification) | [`pbr`](https://docs.openstack.org/pbr/latest/) | -[`pep8-naming`](https://github.com/PyCQA/pep8-naming) | [`pexpect`](https://pexpect.readthedocs.io/) | [`pickleshare`](https://github.com/pickleshare/pickleshare) | [`pluggy`](https://github.com/pytest-dev/pluggy) | @@ -120,7 +109,6 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`pytest-forked`](https://github.com/pytest-dev/pytest-forked) | [`PyYAML`](https://github.com/yaml/pyyaml) | [`regex`](https://bitbucket.org/mrabarnett/mrab-regex) | -[`restructuredtext-lint`](https://github.com/twolfson/restructuredtext-lint) | [`rfc3986`](http://rfc3986.readthedocs.io) | [`six`](https://github.com/benjaminp/six) | [`smmap`](https://github.com/gitpython-developers/smmap) | @@ -128,7 +116,6 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`snowballstemmer`](https://github.com/snowballstem/snowball) | [`stevedore`](https://docs.openstack.org/stevedore/latest/) | [`termcolor`](http://pypi.python.org/pypi/termcolor) | -[`testfixtures`](https://github.com/Simplistix/testfixtures) | [`tornado`](http://www.tornadoweb.org/) | [`tqdm`](https://github.com/tqdm/tqdm) | [`traitlets`](http://ipython.org) | From 2d66735cf56093cd6b658a4752b256bce0679c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 12 Jan 2021 19:30:50 +0100 Subject: [PATCH 02/40] docs: Add notes about partial RST docstrings support Question #214: https://github.com/pawamoy/mkdocstrings/issues/214 --- README.md | 1 + docs/handlers/python.md | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e3deacea..3a6ed222 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Automatic documentation from sources, for MkDocs. - **reStructuredText-style sections support in docstrings:** `pytkdocs` understands all the [reStructuredText fields](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html?highlight=python%20domain#info-field-lists), and returns structured data for `mkdocstrings` to render them. + *Note: only RST **style** is supported, not the whole markup.* - **Admonition support in docstrings:** blocks like `Note: ` or `Warning: ` will be transformed to their [admonition](https://squidfunk.github.io/mkdocs-material/extensions/admonition/) equivalent. *We do not support nested admonitions in docstrings!* diff --git a/docs/handlers/python.md b/docs/handlers/python.md index b3561314..adb792e5 100644 --- a/docs/handlers/python.md +++ b/docs/handlers/python.md @@ -107,7 +107,7 @@ It stands for *(Python) Take Docs*, and is supposed to be a pun on MkDocs (*Make ### Supported docstrings styles -Right now, `pytkdocs` supports the Google-style and reStrcuturedText-style docstring formats. +Right now, `pytkdocs` supports the Google-style and reStructuredText-style docstring formats. #### Google-style @@ -208,6 +208,10 @@ Type annotations are read both in the code and in the docstrings. #### reStructuredText-style +!!! warning "Partial support" + Only RST-**style** is supported, not the whole RST markup specification. + Docstrings will still be converted as Markdown. + You can see examples of reStructuredText-style docstrings in [Sphinx's documentation](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html). From da9b754f281de05df0e9cbb314c9a49d8e110dd0 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Tue, 29 Dec 2020 20:55:23 +0100 Subject: [PATCH 03/40] refactor: Merge md extensions into one, to reduce boilerplate --- src/mkdocstrings/extension.py | 18 +++++++------ src/mkdocstrings/handlers/base.py | 43 ++++++++++--------------------- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 50021030..0b52a83e 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -277,9 +277,6 @@ class MkdocstringsExtension(Extension): It cannot work outside of `mkdocstrings`. """ - blockprocessor_priority = 75 # Right before markdown.blockprocessors.HashHeaderProcessor - inlineprocessor_priority = 168 # Right after markdown.inlinepatterns.ReferenceInlineProcessor - def __init__(self, config: dict, handlers: Handlers, **kwargs) -> None: """ Initialize the object. @@ -303,8 +300,13 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me Arguments: md: A `markdown.Markdown` instance. """ - md.registerExtension(self) - processor = AutoDocProcessor(md.parser, md, self._config, self._handlers) - md.parser.blockprocessors.register(processor, "mkdocstrings", self.blockprocessor_priority) - ref_processor = AutoRefInlineProcessor(md) - md.inlinePatterns.register(ref_processor, "mkdocstrings", self.inlineprocessor_priority) + md.parser.blockprocessors.register( + AutoDocProcessor(md.parser, md, self._config, self._handlers), + "mkdocstrings", + priority=75, # Right before markdown.blockprocessors.HashHeaderProcessor + ) + md.inlinePatterns.register( + AutoRefInlineProcessor(md), + "mkdocstrings", + priority=168, # Right after markdown.inlinepatterns.ReferenceInlineProcessor + ) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index f2dac1c3..1d8a95a9 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -227,7 +227,7 @@ def update_env(self, md: Markdown, config: dict) -> None: config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary. """ - extensions = config["mdx"] + [ShiftHeadingsExtension(), PrefixIdsExtension()] + extensions = config["mdx"] + [_MkdocstringsInnerExtension()] configs = dict(config["mdx_configs"]) # Prevent a bug that happens due to treeprocessors running on the same fragment both as the inner doc and as # part of the re-integrated doc. Namely, the permalink '¶' would be appended twice. This is the only known @@ -412,28 +412,10 @@ def run(self, root: Element): el.set("for", self.id_prefix + for_attr) -class PrefixIdsExtension(Extension): - """Prepend the configured prefix to IDs of all HTML elements.""" - - treeprocessor_priority = 4 # Right after 'toc' (needed because that extension adds ids to headers). - - def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) - """ - Register the extension, with a treeprocessor under the name 'mkdocstrings_ids'. - - Arguments: - md: A `markdown.Markdown` instance. - """ - md.registerExtension(self) - md.treeprocessors.register( - _IdPrependingTreeprocessor(md, ""), - "mkdocstrings_ids", - self.treeprocessor_priority, - ) - - class _HeadingShiftingTreeprocessor(Treeprocessor): - def __init__(self, md, shift_by: int): + regex = re.compile(r"([Hh])([1-6])") + + def __init__(self, md: Markdown, shift_by: int): super().__init__(md) self.shift_by = shift_by @@ -441,21 +423,17 @@ def run(self, root: Element): if not self.shift_by: return for el in root.iter(): - match = re.fullmatch(r"([Hh])([1-6])", el.tag) + match = self.regex.fullmatch(el.tag) if match: level = int(match[2]) + self.shift_by level = max(1, min(level, 6)) el.tag = f"{match[1]}{level}" -class ShiftHeadingsExtension(Extension): - """Shift levels of all Markdown headings according to the configured base level.""" - - treeprocessor_priority = 12 - +class _MkdocstringsInnerExtension(Extension): def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) """ - Register the extension, with a treeprocessor under the name 'mkdocstrings_headings'. + Register the extension. Arguments: md: A `markdown.Markdown` instance. @@ -464,5 +442,10 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me md.treeprocessors.register( _HeadingShiftingTreeprocessor(md, 0), "mkdocstrings_headings", - self.treeprocessor_priority, + priority=12, + ) + md.treeprocessors.register( + _IdPrependingTreeprocessor(md, ""), + "mkdocstrings_ids", + priority=4, # Right after 'toc' (needed because that extension adds ids to headers). ) From 1ac7f620052d8cae5f0747e62c577208a9aaf282 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Fri, 1 Jan 2021 16:43:10 +0100 Subject: [PATCH 04/40] refactor: Pass the actual used Markdown instance to update_env The fact that we're re-instantiating Markdown used to be seen as a hack, but now it's required and preferred. --- src/mkdocstrings/extension.py | 2 +- src/mkdocstrings/handlers/base.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 0b52a83e..0e416d47 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -197,7 +197,7 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0 if not self._updated_env: log.debug("Updating renderer's env") - handler.renderer.update_env(self.md, self._config) + handler.renderer._update_env(self._config) self._updated_env = True log.debug("Rendering templates") diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 1d8a95a9..6011cc6e 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -218,7 +218,7 @@ def render(self, data: Any, config: dict) -> str: The rendered template as HTML. """ # noqa: DAR202 (excess return section) - def update_env(self, md: Markdown, config: dict) -> None: + def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused argument 'config') """ Update the Jinja environment. @@ -227,6 +227,9 @@ def update_env(self, md: Markdown, config: dict) -> None: config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary. """ + self.env.filters["convert_markdown"] = functools.partial(do_convert_markdown, md) + + def _update_env(self, config: dict): extensions = config["mdx"] + [_MkdocstringsInnerExtension()] configs = dict(config["mdx_configs"]) # Prevent a bug that happens due to treeprocessors running on the same fragment both as the inner doc and as @@ -238,8 +241,7 @@ def update_env(self, md: Markdown, config: dict) -> None: pass md = Markdown(extensions=extensions, extension_configs=configs) - - self.env.filters["convert_markdown"] = functools.partial(do_convert_markdown, md) + self.update_env(md, config) class BaseCollector(ABC): From 1a48dd2def31801973bcf03bf83f986fb293dfe3 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Fri, 1 Jan 2021 17:29:56 +0100 Subject: [PATCH 05/40] refactor: Move do_convert_markdown method into the class --- src/mkdocstrings/handlers/base.py | 51 ++++++++++++++++--------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 6011cc6e..bb370600 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -9,7 +9,6 @@ - `teardown`, that will teardown all the cached handlers, and then clear the cache. """ -import functools import importlib import re import textwrap @@ -128,29 +127,6 @@ def do_any(seq: Sequence, attribute: str = None) -> bool: return any(_[attribute] for _ in seq) -def do_convert_markdown(md: Markdown, text: str, heading_level: int, html_id: str = "") -> Markup: - """ - Render Markdown text; for use inside templates. - - Arguments: - md: A `markdown.Markdown` instance. - text: The text to convert. - heading_level: The base heading level to start all Markdown headings from. - html_id: The HTML id of the element that's considered the parent of this element. - - Returns: - An HTML string. - """ - md.treeprocessors["mkdocstrings_headings"].shift_by = heading_level - md.treeprocessors["mkdocstrings_ids"].id_prefix = html_id and html_id + "--" - try: - return Markup(md.convert(text)) - finally: - md.treeprocessors["mkdocstrings_headings"].shift_by = 0 - md.treeprocessors["mkdocstrings_ids"].id_prefix = "" - md.reset() - - class BaseRenderer(ABC): """ The base renderer class. @@ -205,6 +181,8 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = self.env.filters["highlight"] = highlight_function + self._md = None # To be populated in `update_env`. + @abstractmethod def render(self, data: Any, config: dict) -> str: """ @@ -218,6 +196,28 @@ def render(self, data: Any, config: dict) -> str: The rendered template as HTML. """ # noqa: DAR202 (excess return section) + def do_convert_markdown(self, text: str, heading_level: int, html_id: str = "") -> Markup: + """ + Render Markdown text; for use inside templates. + + Arguments: + text: The text to convert. + heading_level: The base heading level to start all Markdown headings from. + html_id: The HTML id of the element that's considered the parent of this element. + + Returns: + An HTML string. + """ + treeprocessors = self._md.treeprocessors + treeprocessors["mkdocstrings_headings"].shift_by = heading_level + treeprocessors["mkdocstrings_ids"].id_prefix = html_id and html_id + "--" + try: + return Markup(self._md.convert(text)) + finally: + treeprocessors["mkdocstrings_headings"].shift_by = 0 + treeprocessors["mkdocstrings_ids"].id_prefix = "" + self._md.reset() + def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused argument 'config') """ Update the Jinja environment. @@ -227,7 +227,8 @@ def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary. """ - self.env.filters["convert_markdown"] = functools.partial(do_convert_markdown, md) + self._md = md + self.env.filters["convert_markdown"] = self.do_convert_markdown def _update_env(self, config: dict): extensions = config["mdx"] + [_MkdocstringsInnerExtension()] From 15f84f981982c8e2b15498f5c869ac207f3ce5d7 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 4 Jan 2021 14:37:11 +0100 Subject: [PATCH 06/40] refactor: Don't re-parse the whole subdoc, expose only headings Previously the autodoc sub-document (HTML with nested Markdown) was being re-integrated into the top-level document by the way of parsing it as XML and inserting it back into the XML tree of the top-level markdown during its rendering. This gave a lot of side effects that need to be worked around. XML parsing, Markdown double-processing (with only situational workaround possible), etc. Now, the document will be inserted as fully opaque HTML, so those issues are avoided entirely. But now there are new issues to overcome! * The ToC extension can no longer see through this subdocument, so no ToC entries can be generated. I thought a lot how to re-add these properly, and there's no easy solution without parsing the HTML. In the end I decided to manually reinsert just the heading elements back into the document (note that detecting headings is two-pronged: in Markdown with a treeprocessor, and in HTML through requiring the use of a specific callback filter). As those headings have no business there, before finishing the document they get removed. * MkDocs can no longer fix relative paths in this subdocument. That one is much easier: just manually re-add its treeprocessor. --- pyproject.toml | 1 + src/mkdocstrings/extension.py | 116 ++++-------------- src/mkdocstrings/handlers/base.py | 113 ++++++++++++++--- src/mkdocstrings/references.py | 2 +- .../templates/python/material/attribute.html | 21 ++-- .../templates/python/material/children.html | 10 +- .../templates/python/material/class.html | 21 ++-- .../templates/python/material/function.html | 21 ++-- .../templates/python/material/method.html | 21 ++-- .../templates/python/material/module.html | 21 ++-- tests/fixtures/builtin.py | 2 + tests/test_extension.py | 46 ++++++- 12 files changed, 231 insertions(+), 164 deletions(-) create mode 100644 tests/fixtures/builtin.py diff --git a/pyproject.toml b/pyproject.toml index 3bb57576..e6290e50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ exclude = ["tests/fixtures"] "-F821", # redundant with E0602 (undefined variable) "-Q000", # black already deals with quoting "-S101", # use of assert + "-S405", # we are not parsing XML "-W503", # line break before binary operator (incompatible with Black) "-C0103", # two-lowercase-letters variable DO conform to snake_case naming style "-C0116", # redunant with D102 (missing docstring) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 0e416d47..06a88c17 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -24,8 +24,8 @@ """ import re from collections import ChainMap -from typing import Any, Mapping, MutableSequence, Tuple -from xml.etree.ElementTree import XML, Element, ParseError # noqa: S405 (we choose to trust the XML input) +from typing import Any, Mapping, MutableSequence, Sequence, Tuple +from xml.etree.ElementTree import Element import yaml from jinja2.exceptions import TemplateNotFound @@ -33,7 +33,7 @@ from markdown.blockparser import BlockParser from markdown.blockprocessors import BlockProcessor from markdown.extensions import Extension -from markdown.util import AtomicString +from markdown.treeprocessors import Treeprocessor from mkdocstrings.handlers.base import CollectionError, Handlers from mkdocstrings.loggers import get_logger @@ -41,46 +41,6 @@ log = get_logger(__name__) -ENTITIES = """ - - - - - - - - - - - - ]> -""" - - -def atomic_brute_cast(tree: Element) -> Element: - """ - Cast every node's text into an atomic string to prevent further processing on it. - - Since we generate the final HTML with Jinja templates, we do not want other inline or tree processors - to keep modifying the data, so this function is used to mark the complete tree as "do not touch". - - Reference: issue [Python-Markdown/markdown#920](https://github.com/Python-Markdown/markdown/issues/920). - - On a side note: isn't `atomic_brute_cast` such a beautiful function name? - - Arguments: - tree: An XML node, used like the root of an XML tree. - - Returns: - The same node, recursively modified by side-effect. You can skip re-assigning the return value. - """ - if tree.text: - tree.text = AtomicString(tree.text) - for child in tree: - atomic_brute_cast(child) - return tree - class AutoDocProcessor(BlockProcessor): """ @@ -151,8 +111,15 @@ def run(self, parent: Element, blocks: MutableSequence[str]) -> None: identifier = match["name"] heading_level = match["heading"].count("#") log.debug(f"Matched '::: {identifier}'") - xml_element = self.process_block(identifier, block, heading_level) - parent.append(xml_element) + + html, headings = self._process_block(identifier, block, heading_level) + el = Element("div", {"class": "mkdocstrings"}) + # The final HTML is inserted as opaque to subsequent processing, and only revealed at the end. + el.text = self.md.htmlStash.store(html) + # So we need to duplicate the headings directly (and delete later), just so 'toc' can pick them up. + el.extend(headings) + + parent.append(el) if the_rest: # This block contained unindented line(s) after the first indented @@ -160,7 +127,7 @@ def run(self, parent: Element, blocks: MutableSequence[str]) -> None: # list for future processing. blocks.insert(0, the_rest) - def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0) -> Element: + def _process_block(self, identifier: str, yaml_block: str, heading_level: int = 0) -> Tuple[str, Sequence[Element]]: """ Process an autodoc block. @@ -171,11 +138,10 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0 Raises: CollectionError: When something wrong happened during collection. - ParseError: When the generated HTML could not be parsed as XML. TemplateNotFound: When a template used for rendering could not be found. Returns: - A new XML element. + Rendered HTML and the list of heading elements encoutered. """ config = yaml.safe_load(yaml_block) or {} handler_name = self._handlers.get_handler_name(config) @@ -197,7 +163,7 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0 if not self._updated_env: log.debug("Updating renderer's env") - handler.renderer._update_env(self._config) + handler.renderer._update_env(self.md, self._config) # noqa: W0212 (protected member OK) self._updated_env = True log.debug("Rendering templates") @@ -210,15 +176,7 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0 ) raise - log.debug("Loading HTML back into XML tree") - rendered = ENTITIES + rendered - try: - xml_contents = XML(rendered) - except ParseError as error: - log_xml_parse_error(str(error), rendered) - raise - - return atomic_brute_cast(xml_contents) # type: ignore + return (rendered, handler.renderer.get_headings()) def get_item_configs(handler_config: dict, config: dict) -> Tuple[Mapping, Mapping]: @@ -237,37 +195,12 @@ def get_item_configs(handler_config: dict, config: dict) -> Tuple[Mapping, Mappi return item_selection_config, item_rendering_config -def log_xml_parse_error(error: str, xml_text: str) -> None: - """ - Log an XML parsing error. - - If the error is a tag mismatch, augment the log message. - - Arguments: - error: The error message (no traceback). - xml_text: The XML text that generated the parsing error. - """ - message = error - mismatched_tag = "mismatched tag" in error - undefined_entity = "undefined entity" in error - - if mismatched_tag or undefined_entity: - line_column = error[error.rfind(":") + 1 :] - line, column = line_column.split(", ") - lineno = int(line[line.rfind(" ") + 1 :]) - columnno = int(column[column.rfind(" ") + 1 :]) - - line = xml_text.split("\n")[lineno - 1] - if mismatched_tag: - character = line[columnno] - message += ( - f" (character {character}):\n{line}\n" - "If your Markdown contains angle brackets < >, try to wrap them between backticks `< >`, " - "or replace them with < and >" - ) - elif undefined_entity: - message += f":\n{line}\n" - log.error(message) +class _PostProcessor(Treeprocessor): + def run(self, root: Element): + for el in root.iter("div"): + if el.get("class") == "mkdocstrings": + # Delete the duplicated headings from before, but keep the text (i.e. the actual HTML). + del el[:] class MkdocstringsExtension(Extension): @@ -305,6 +238,11 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me "mkdocstrings", priority=75, # Right before markdown.blockprocessors.HashHeaderProcessor ) + md.treeprocessors.register( + _PostProcessor(md.parser), + "mkdocstrings_post", + priority=4, # Right after 'toc'. + ) md.inlinePatterns.register( AutoRefInlineProcessor(md), "mkdocstrings", diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index bb370600..68c19cb2 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -9,13 +9,14 @@ - `teardown`, that will teardown all the cached handlers, and then clear the cache. """ +import copy import importlib import re import textwrap from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Dict, Optional, Sequence -from xml.etree.ElementTree import Element # noqa: S405 (we choose to trust the XML input) +from typing import Any, Dict, List, Optional, Sequence +from xml.etree.ElementTree import Element, tostring from jinja2 import Environment, FileSystemLoader from markdown import Markdown @@ -181,6 +182,7 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = self.env.filters["highlight"] = highlight_function + self._headings = [] self._md = None # To be populated in `update_env`. @abstractmethod @@ -218,6 +220,65 @@ def do_convert_markdown(self, text: str, heading_level: int, html_id: str = "") treeprocessors["mkdocstrings_ids"].id_prefix = "" self._md.reset() + def do_heading( + self, + content: str, + heading_level: int, + *, + toc_label: Optional[str] = None, + **attributes: str, + ) -> Markup: + """ + Render an HTML heading and register it for the table of contents. For use inside templates. + + Arguments: + content: The HTML within the heading. + heading_level: The level of heading (e.g. 3 -> `h3`). + toc_label: The title to use in the table of contents ('data-toc-label' attribute). + attributes: Any extra HTML attributes of the heading. + + Returns: + An HTML string. + """ + # First, produce the "fake" heading, for ToC only. + el = Element(f"h{heading_level}", attributes) + if toc_label is None: + toc_label = content.unescape() if isinstance(el, Markup) else content + el.set("data-toc-label", toc_label) + self._headings.append(el) + + # Now produce the actual HTML to be rendered. The goal is to wrap the HTML content into a heading. + # Start with a heading that has just attributes (no text), and add a placeholder into it. + el = Element(f"h{heading_level}", attributes) + el.append(Element("mkdocstrings-placeholder")) + # Tell the 'toc' extension to make its additions if configured so. + toc = self._md.treeprocessors["toc"] + if toc.use_anchors: + toc.add_anchor(el, attributes["id"]) + if toc.use_permalinks: + toc.add_permalink(el, attributes["id"]) + + # The content we received is HTML, so it can't just be inserted into the tree. We had marked the middle + # of the heading with a placeholder that can never occur (text can't directly contain angle brackets). + # Now this HTML wrapper can be "filled" by replacing the placeholder. + html_with_placeholder = tostring(el, encoding="unicode") + assert ( + html_with_placeholder.count("") == 1 + ), f"Bug in mkdocstrings: failed to replace in {html_with_placeholder!r}" + html = html_with_placeholder.replace("", content) + return Markup(html) + + def get_headings(self) -> Sequence[Element]: + """ + Return and clear the headings gathered so far. + + Returns: + A list of HTML elements. + """ + result = list(self._headings) + self._headings.clear() + return result + def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused argument 'config') """ Update the Jinja environment. @@ -229,20 +290,17 @@ def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused """ self._md = md self.env.filters["convert_markdown"] = self.do_convert_markdown + self.env.filters["heading"] = self.do_heading - def _update_env(self, config: dict): - extensions = config["mdx"] + [_MkdocstringsInnerExtension()] - configs = dict(config["mdx_configs"]) - # Prevent a bug that happens due to treeprocessors running on the same fragment both as the inner doc and as - # part of the re-integrated doc. Namely, the permalink '¶' would be appended twice. This is the only known - # non-idempotent effect of an extension, so specifically prevent it on the inner doc. - try: - configs["toc"] = dict(configs["toc"], permalink=False) - except KeyError: - pass + def _update_env(self, md: Markdown, config: dict): + extensions = config["mdx"] + [_MkdocstringsInnerExtension(self._headings)] + + new_md = Markdown(extensions=extensions, extension_configs=config["mdx_configs"]) + # MkDocs adds its own (required) extension that's not part of the config. Propagate it. + if "relpath" in md.treeprocessors: + new_md.treeprocessors.register(md.treeprocessors["relpath"], "relpath", priority=0) - md = Markdown(extensions=extensions, extension_configs=configs) - self.update_env(md, config) + self.update_env(new_md, config) class BaseCollector(ABC): @@ -433,7 +491,29 @@ def run(self, root: Element): el.tag = f"{match[1]}{level}" +class _HeadingReportingTreeprocessor(Treeprocessor): + regex = re.compile(r"[Hh][1-6]") + + def __init__(self, md: Markdown, headings: List[Element]): + super().__init__(md) + self.headings = headings + + def run(self, root: Element): + for el in root.iter(): + if self.regex.fullmatch(el.tag): + el = copy.copy(el) + # 'toc' extension's first pass (which we require to build heading stubs/ids) also edits the HTML. + # Undo the permalink edit so we can pass this heading to the outer pass of the 'toc' extension. + if len(el) > 0 and el[-1].get("class") == self.md.treeprocessors["toc"].permalink_class: + del el[-1] + self.headings.append(el) + + class _MkdocstringsInnerExtension(Extension): + def __init__(self, headings: List[Element]): + super().__init__() + self.headings = headings + def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) """ Register the extension. @@ -452,3 +532,8 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me "mkdocstrings_ids", priority=4, # Right after 'toc' (needed because that extension adds ids to headers). ) + md.treeprocessors.register( + _HeadingReportingTreeprocessor(md, self.headings), + "mkdocstrings_headings_list", + priority=1, # Close to the end. + ) diff --git a/src/mkdocstrings/references.py b/src/mkdocstrings/references.py index 38957677..6b9e3c1b 100644 --- a/src/mkdocstrings/references.py +++ b/src/mkdocstrings/references.py @@ -3,7 +3,7 @@ import re from html import escape, unescape from typing import Any, Callable, Dict, List, Match, Tuple, Union -from xml.etree.ElementTree import Element # noqa: S405 (input is our own, and Markdown coming from code) +from xml.etree.ElementTree import Element from markdown.inlinepatterns import REFERENCE_RE, ReferenceInlineProcessor diff --git a/src/mkdocstrings/templates/python/material/attribute.html b/src/mkdocstrings/templates/python/material/attribute.html index 3711698b..785b3b41 100644 --- a/src/mkdocstrings/templates/python/material/attribute.html +++ b/src/mkdocstrings/templates/python/material/attribute.html @@ -16,10 +16,10 @@ {% set show_full_path = config.show_object_full_path %} {% endif %} - + {% filter heading(heading_level, + id=html_id, + class="doc doc-heading", + toc_label=attribute.name) %} {% filter highlight(language="python", inline=True) %} {% if show_full_path %}{{ attribute.path }}{% else %}{{ attribute.name }}{% endif %} @@ -30,15 +30,16 @@ {% include "properties.html" with context %} {% endwith %} - + {% endfilter %} {% else %} {% if config.show_root_toc_entry %} - + {% filter heading(heading_level, + class="hidden-toc", + id=html_id, + toc_label=attribute.path, + style="visibility: hidden; position: absolute;") %} + {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} {% endif %} diff --git a/src/mkdocstrings/templates/python/material/children.html b/src/mkdocstrings/templates/python/material/children.html index cb6f072a..967ad493 100644 --- a/src/mkdocstrings/templates/python/material/children.html +++ b/src/mkdocstrings/templates/python/material/children.html @@ -14,7 +14,7 @@ {% endif %} {% if config.show_category_heading and obj.attributes|any("has_contents") %} - Attributes + {% filter heading(heading_level, id=html_id ~ "-attributes") %}Attributes{% endfilter %} {% endif %} {% with heading_level = heading_level + extra_level %} {% for attribute in obj.attributes|sort(attribute="name") %} @@ -23,7 +23,7 @@ {% endwith %} {% if config.show_category_heading and obj.classes|any("has_contents") %} - Classes + {% filter heading(heading_level, id=html_id ~ "-classes") %}Classes{% endfilter %} {% endif %} {% with heading_level = heading_level + extra_level %} {% for class in obj.classes|sort(attribute="name") %} @@ -32,7 +32,7 @@ {% endwith %} {% if config.show_category_heading and obj.functions|any("has_contents") %} - Functions + {% filter heading(heading_level, id=html_id ~ "-functions") %}Functions{% endfilter %} {% endif %} {% with heading_level = heading_level + extra_level %} {% for function in obj.functions|sort(attribute="name") %} @@ -41,7 +41,7 @@ {% endwith %} {% if config.show_category_heading and obj.methods|any("has_contents") %} - Methods + {% filter heading(heading_level, id=html_id ~ "-methods") %}Methods{% endfilter %} {% endif %} {% with heading_level = heading_level + extra_level %} {% for method in obj.methods|sort(attribute="name") %} @@ -50,7 +50,7 @@ {% endwith %} {% if config.show_category_heading and obj.modules|any("has_contents") %} - Modules + {% filter heading(heading_level, id=html_id ~ "-modules") %}Modules{% endfilter %} {% endif %} {% with heading_level = heading_level + extra_level %} {% for module in obj.modules|sort(attribute="name") %} diff --git a/src/mkdocstrings/templates/python/material/class.html b/src/mkdocstrings/templates/python/material/class.html index 76489056..2c043ca3 100644 --- a/src/mkdocstrings/templates/python/material/class.html +++ b/src/mkdocstrings/templates/python/material/class.html @@ -16,10 +16,10 @@ {% set show_full_path = config.show_object_full_path %} {% endif %} - + {% filter heading(heading_level, + id=html_id, + class="doc doc-heading", + toc_label=class.name) %} {% if show_full_path %}{{ class.path }}{% else %}{{ class.name }}{% endif %} @@ -27,15 +27,16 @@ {% include "properties.html" with context %} {% endwith %} - + {% endfilter %} {% else %} {% if config.show_root_toc_entry %} - + {% filter heading(heading_level, + class="hidden-toc", + id=html_id, + toc_label=class.path, + style="visibility: hidden; position: absolute;") %} + {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} {% endif %} diff --git a/src/mkdocstrings/templates/python/material/function.html b/src/mkdocstrings/templates/python/material/function.html index 7b995de8..97f5f483 100644 --- a/src/mkdocstrings/templates/python/material/function.html +++ b/src/mkdocstrings/templates/python/material/function.html @@ -16,10 +16,10 @@ {% set show_full_path = config.show_object_full_path %} {% endif %} - + {% filter heading(heading_level, + id=html_id, + class="doc doc-heading", + toc_label=function.name ~ "()") %} {% filter highlight(language="python", inline=True) %} {% if show_full_path %}{{ function.path }}{% else %}{{ function.name }}{% endif %} @@ -30,15 +30,16 @@ {% include "properties.html" with context %} {% endwith %} - + {% endfilter %} {% else %} {% if config.show_root_toc_entry %} - + {% filter heading(heading_level, + class="hidden-toc", + id=html_id, + toc_label=function.path, + style="visibility: hidden; position: absolute;") %} + {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} {% endif %} diff --git a/src/mkdocstrings/templates/python/material/method.html b/src/mkdocstrings/templates/python/material/method.html index 9473d059..5b35f410 100644 --- a/src/mkdocstrings/templates/python/material/method.html +++ b/src/mkdocstrings/templates/python/material/method.html @@ -16,10 +16,10 @@ {% set show_full_path = config.show_object_full_path %} {% endif %} - + {% filter heading(heading_level, + id=html_id, + class="doc doc-heading", + toc_label=method.name ~ "()") %} {% filter highlight(language="python", inline=True) %} {% if show_full_path %}{{ method.path }}{% else %}{{ method.name }}{% endif %} @@ -30,15 +30,16 @@ {% include "properties.html" with context %} {% endwith %} - + {% endfilter %} {% else %} {% if config.show_root_toc_entry %} - + {% filter heading(heading_level, + class="hidden-toc", + id=html_id, + toc_label=method.path, + style="visibility: hidden; position: absolute;") %} + {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} {% endif %} diff --git a/src/mkdocstrings/templates/python/material/module.html b/src/mkdocstrings/templates/python/material/module.html index b874eabf..efb67a7a 100644 --- a/src/mkdocstrings/templates/python/material/module.html +++ b/src/mkdocstrings/templates/python/material/module.html @@ -16,10 +16,10 @@ {% set show_full_path = config.show_object_full_path %} {% endif %} - + {% filter heading(heading_level, + id=html_id, + class="doc doc-heading", + toc_label=module.name) %} {% if show_full_path %}{{ module.path }}{% else %}{{ module.name }}{% endif %} @@ -27,15 +27,16 @@ {% include "properties.html" with context %} {% endwith %} - + {% endfilter %} {% else %} {% if config.show_root_toc_entry %} - + {% filter heading(heading_level, + class="hidden-toc", + id=html_id, + toc_label=module.path, + style="visibility: hidden; position: absolute;") %} + {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} {% endif %} diff --git a/tests/fixtures/builtin.py b/tests/fixtures/builtin.py new file mode 100644 index 00000000..cab198e3 --- /dev/null +++ b/tests/fixtures/builtin.py @@ -0,0 +1,2 @@ +def func(foo=print): + """test""" diff --git a/tests/test_extension.py b/tests/test_extension.py index 2cb5a818..f47ea8e4 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -27,6 +27,7 @@ def ext_markdown(**kwargs): "mkdocstrings": {"default_handler": "python", "custom_templates": None, "watch": [], "handlers": {}}, } config.update(kwargs) + config["mdx"].append("toc") # Guaranteed to be added by MkDocs. original_config = copy.deepcopy(config) handlers = Handlers(config) @@ -73,9 +74,9 @@ def test_markdown_heading_level(): """Assert that Markdown headings' level doesn't exceed heading_level.""" with ext_markdown() as md: output = md.convert("::: tests.fixtures.headings\n rendering:\n show_root_heading: true") - assert "

Foo

" in output - assert "
Bar
" in output - assert "
Baz
" in output + assert ">Foo" in output + assert ">Bar" in output + assert ">Baz" in output def test_keeps_preceding_text(): @@ -83,7 +84,7 @@ def test_keeps_preceding_text(): with ext_markdown() as md: output = md.convert("**preceding**\n::: tests.fixtures.headings") assert "preceding" in output - assert "

Foo

" in output + assert ">Foo" in output assert ":::" not in output @@ -95,10 +96,19 @@ def test_reference_inside_autodoc(): assert snippet in output +def test_html_inside_heading(): + """Assert that headings don't double-escape HTML.""" + with ext_markdown() as md: + output = md.convert("::: tests.fixtures.builtin") + assert "=<" in output + assert "&" not in output + + @pytest.mark.parametrize( ("permalink_setting", "expect_permalink"), [ ("@@@", "@@@"), + ("TeSt", "TeSt"), (True, "¶"), ], ) @@ -110,7 +120,7 @@ def test_no_double_toc(permalink_setting, expect_permalink): permalink_setting: The 'permalink' setting of 'toc' extension. expect_permalink: Text of the permalink to search for in the output. """ - with ext_markdown(mdx=["toc"], mdx_configs={"toc": {"permalink": permalink_setting}}) as md: + with ext_markdown(mdx_configs={"toc": {"permalink": permalink_setting}}) as md: output = md.convert( dedent( """ @@ -125,3 +135,29 @@ def test_no_double_toc(permalink_setting, expect_permalink): ) ) assert output.count(expect_permalink) == 5 + assert 'id="tests.fixtures.headings--foo"' in output + assert md.toc_tokens == [ # noqa: E1101 (the member gets populated only with 'toc' extension) + { + "level": 1, + "id": "aa", + "name": "aa", + "children": [ + { + "level": 2, + "id": "tests.fixtures.headings--foo", + "name": "Foo", + "children": [ + { + "level": 4, + "id": "tests.fixtures.headings--bar", + "name": "Bar", + "children": [ + {"level": 6, "id": "tests.fixtures.headings--baz", "name": "Baz", "children": []} + ], + } + ], + } + ], + }, + {"level": 1, "id": "bb", "name": "bb", "children": []}, + ] From 0fdb0821867eb0e14a972a603c22301aafecf4f4 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 4 Jan 2021 14:36:54 +0100 Subject: [PATCH 07/40] refactor: Actually exclude hidden headings from the doc Now we separately control which headings appear in the doc and which in the ToC. No need to hide them with CSS now. --- docs/css/mkdocstrings.css | 11 ---------- docs/handlers/python.md | 22 ------------------- src/mkdocstrings/handlers/base.py | 5 +++++ .../templates/python/material/attribute.html | 3 +-- .../templates/python/material/class.html | 3 +-- .../templates/python/material/function.html | 3 +-- .../templates/python/material/method.html | 3 +-- .../templates/python/material/module.html | 3 +-- 8 files changed, 10 insertions(+), 43 deletions(-) diff --git a/docs/css/mkdocstrings.css b/docs/css/mkdocstrings.css index 54950fb2..1ec6bfdf 100644 --- a/docs/css/mkdocstrings.css +++ b/docs/css/mkdocstrings.css @@ -10,17 +10,6 @@ h5.doc-heading { text-transform: none !important; } -/* Don't use vertical space on hidden ToC entries. */ -.hidden-toc::before { - margin-top: 0 !important; - padding-top: 0 !important; -} - -/* Don't show permalink of hidden ToC entries. */ -.hidden-toc a.headerlink { - display: none; -} - /* Avoid breaking parameters name, etc. in table cells. */ td code { word-break: normal !important; diff --git a/docs/handlers/python.md b/docs/handlers/python.md index adb792e5..2be076cc 100644 --- a/docs/handlers/python.md +++ b/docs/handlers/python.md @@ -335,17 +335,6 @@ h5.doc-heading { text-transform: none !important; } -/* Don't use vertical space on hidden ToC entries. */ -.hidden-toc::before { - margin-top: 0 !important; - padding-top: 0 !important; -} - -/* Don't show permalink of hidden ToC entries. */ -.hidden-toc a.headerlink { - display: none; -} - /* Avoid breaking parameters name, etc. in table cells. */ td code { word-break: normal !important; @@ -370,17 +359,6 @@ div.doc-contents:not(.first) { margin-bottom: 60px; } -/* Don't use vertical space on hidden ToC entries. */ -.hidden-toc::before { - margin-top: 0 !important; - padding-top: 0 !important; -} - -/* Don't show permalink of hidden ToC entries. */ -.hidden-toc a.headerlink { - display: none; -} - /* Avoid breaking parameters name, etc. in table cells. */ td code { word-break: normal !important; diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 68c19cb2..31cca04f 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -225,6 +225,7 @@ def do_heading( content: str, heading_level: int, *, + hidden: bool = False, toc_label: Optional[str] = None, **attributes: str, ) -> Markup: @@ -234,6 +235,7 @@ def do_heading( Arguments: content: The HTML within the heading. heading_level: The level of heading (e.g. 3 -> `h3`). + hidden: If True, only register it for the table of contents, don't render anything. toc_label: The title to use in the table of contents ('data-toc-label' attribute). attributes: Any extra HTML attributes of the heading. @@ -247,6 +249,9 @@ def do_heading( el.set("data-toc-label", toc_label) self._headings.append(el) + if hidden: + return Markup('').format(attributes["id"]) + # Now produce the actual HTML to be rendered. The goal is to wrap the HTML content into a heading. # Start with a heading that has just attributes (no text), and add a placeholder into it. el = Element(f"h{heading_level}", attributes) diff --git a/src/mkdocstrings/templates/python/material/attribute.html b/src/mkdocstrings/templates/python/material/attribute.html index 785b3b41..4b742509 100644 --- a/src/mkdocstrings/templates/python/material/attribute.html +++ b/src/mkdocstrings/templates/python/material/attribute.html @@ -35,10 +35,9 @@ {% else %} {% if config.show_root_toc_entry %} {% filter heading(heading_level, - class="hidden-toc", id=html_id, toc_label=attribute.path, - style="visibility: hidden; position: absolute;") %} + hidden=True) %} {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} diff --git a/src/mkdocstrings/templates/python/material/class.html b/src/mkdocstrings/templates/python/material/class.html index 2c043ca3..819edfd4 100644 --- a/src/mkdocstrings/templates/python/material/class.html +++ b/src/mkdocstrings/templates/python/material/class.html @@ -32,10 +32,9 @@ {% else %} {% if config.show_root_toc_entry %} {% filter heading(heading_level, - class="hidden-toc", id=html_id, toc_label=class.path, - style="visibility: hidden; position: absolute;") %} + hidden=True) %} {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} diff --git a/src/mkdocstrings/templates/python/material/function.html b/src/mkdocstrings/templates/python/material/function.html index 97f5f483..a2e0161f 100644 --- a/src/mkdocstrings/templates/python/material/function.html +++ b/src/mkdocstrings/templates/python/material/function.html @@ -35,10 +35,9 @@ {% else %} {% if config.show_root_toc_entry %} {% filter heading(heading_level, - class="hidden-toc", id=html_id, toc_label=function.path, - style="visibility: hidden; position: absolute;") %} + hidden=True) %} {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} diff --git a/src/mkdocstrings/templates/python/material/method.html b/src/mkdocstrings/templates/python/material/method.html index 5b35f410..d73c5ab3 100644 --- a/src/mkdocstrings/templates/python/material/method.html +++ b/src/mkdocstrings/templates/python/material/method.html @@ -35,10 +35,9 @@ {% else %} {% if config.show_root_toc_entry %} {% filter heading(heading_level, - class="hidden-toc", id=html_id, toc_label=method.path, - style="visibility: hidden; position: absolute;") %} + hidden=True) %} {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} diff --git a/src/mkdocstrings/templates/python/material/module.html b/src/mkdocstrings/templates/python/material/module.html index efb67a7a..bff6fdcf 100644 --- a/src/mkdocstrings/templates/python/material/module.html +++ b/src/mkdocstrings/templates/python/material/module.html @@ -32,10 +32,9 @@ {% else %} {% if config.show_root_toc_entry %} {% filter heading(heading_level, - class="hidden-toc", id=html_id, toc_label=module.path, - style="visibility: hidden; position: absolute;") %} + hidden=True) %} {% endfilter %} {% endif %} {% set heading_level = heading_level - 1 %} From 461697167828571075f4a125f157189b21cb4499 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sat, 16 Jan 2021 16:05:41 +0100 Subject: [PATCH 08/40] refactor: Declare a fake type for collector data for posterity --- src/mkdocstrings/extension.py | 6 +++--- src/mkdocstrings/handlers/base.py | 7 ++++--- src/mkdocstrings/handlers/python.py | 6 +++--- src/mkdocstrings/plugin.py | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 50021030..d6d8e2fa 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -24,7 +24,7 @@ """ import re from collections import ChainMap -from typing import Any, Mapping, MutableSequence, Tuple +from typing import Mapping, MutableSequence, Tuple from xml.etree.ElementTree import XML, Element, ParseError # noqa: S405 (we choose to trust the XML input) import yaml @@ -35,7 +35,7 @@ from markdown.extensions import Extension from markdown.util import AtomicString -from mkdocstrings.handlers.base import CollectionError, Handlers +from mkdocstrings.handlers.base import CollectionError, CollectorItem, Handlers from mkdocstrings.loggers import get_logger from mkdocstrings.references import AutoRefInlineProcessor @@ -190,7 +190,7 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0 log.debug("Collecting data") try: - data: Any = handler.collector.collect(identifier, selection) + data: CollectorItem = handler.collector.collect(identifier, selection) except CollectionError: log.error(f"Could not collect '{identifier}'") raise diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index f2dac1c3..1f27ebbd 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -27,7 +27,8 @@ from mkdocstrings.loggers import get_template_logger -handlers_cache: Dict[str, Any] = {} +CollectorItem = Any + TEMPLATES_DIR = Path(__file__).parent.parent / "templates" @@ -206,7 +207,7 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = self.env.filters["highlight"] = highlight_function @abstractmethod - def render(self, data: Any, config: dict) -> str: + def render(self, data: CollectorItem, config: dict) -> str: """ Render a template using provided data and configuration options. @@ -253,7 +254,7 @@ class BaseCollector(ABC): """ @abstractmethod - def collect(self, identifier: str, config: dict) -> Any: + def collect(self, identifier: str, config: dict) -> CollectorItem: """ Collect data given an identifier and selection configuration. diff --git a/src/mkdocstrings/handlers/python.py b/src/mkdocstrings/handlers/python.py index 95ccfc4d..8716d558 100644 --- a/src/mkdocstrings/handlers/python.py +++ b/src/mkdocstrings/handlers/python.py @@ -13,7 +13,7 @@ from markdown import Markdown -from mkdocstrings.handlers.base import BaseCollector, BaseHandler, BaseRenderer, CollectionError +from mkdocstrings.handlers.base import BaseCollector, BaseHandler, BaseRenderer, CollectionError, CollectorItem from mkdocstrings.loggers import get_logger log = get_logger(__name__) @@ -65,7 +65,7 @@ class PythonRenderer(BaseRenderer): **`heading_level`** | `int` | The initial heading level to use. | `2` """ # noqa: E501 - def render(self, data: Any, config: dict) -> str: # noqa: D102 (ignore missing docstring) + def render(self, data: CollectorItem, config: dict) -> str: # noqa: D102 (ignore missing docstring) final_config = ChainMap(config, self.default_config) template = self.env.get_template(f"{data['category']}.html") @@ -161,7 +161,7 @@ def __init__(self, setup_commands: Optional[List[str]] = None) -> None: env=env, ) - def collect(self, identifier: str, config: dict) -> Any: + def collect(self, identifier: str, config: dict) -> CollectorItem: """ Collect the documentation tree given an identifier and selection options. diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 87f3421c..2d5dc964 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -24,7 +24,7 @@ import logging import os -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Callable, Dict, Optional, Tuple from livereload import Server from mkdocs.config import Config @@ -34,7 +34,7 @@ from mkdocs.structure.toc import AnchorLink from mkdocstrings.extension import MkdocstringsExtension -from mkdocstrings.handlers.base import BaseHandler, Handlers +from mkdocstrings.handlers.base import BaseHandler, CollectorItem, Handlers from mkdocstrings.loggers import get_logger from mkdocstrings.references import fix_refs @@ -102,7 +102,7 @@ class MkdocstringsPlugin(BasePlugin): def __init__(self) -> None: """Initialize the object.""" super().__init__() - self.url_map: Dict[Any, str] = {} + self.url_map: Dict[CollectorItem, str] = {} self.handlers: Optional[Handlers] = None def on_serve(self, server: Server, builder: Callable = None, **kwargs) -> Server: # noqa: W0613 (unused arguments) From 05c7a3fc83b67d3244ea3bfe97dab19aa53f2d38 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sun, 17 Jan 2021 00:30:45 +0100 Subject: [PATCH 09/40] feat: Let handlers add CSS to the pages, do so for Python handler --- docs/css/mkdocstrings.css | 27 -------- docs/handlers/python.md | 66 ------------------- src/mkdocstrings/handlers/base.py | 35 +++++++--- src/mkdocstrings/plugin.py | 12 +++- .../templates/python/material/style.css | 26 ++++++++ .../templates/python/readthedocs/style.css | 38 +++++++++++ 6 files changed, 102 insertions(+), 102 deletions(-) create mode 100644 src/mkdocstrings/templates/python/material/style.css create mode 100644 src/mkdocstrings/templates/python/readthedocs/style.css diff --git a/docs/css/mkdocstrings.css b/docs/css/mkdocstrings.css index 54950fb2..42c77416 100644 --- a/docs/css/mkdocstrings.css +++ b/docs/css/mkdocstrings.css @@ -4,30 +4,3 @@ div.doc-contents:not(.first) { border-left: 4px solid rgba(230, 230, 230); margin-bottom: 80px; } - -/* Don't capitalize names. */ -h5.doc-heading { - text-transform: none !important; -} - -/* Don't use vertical space on hidden ToC entries. */ -.hidden-toc::before { - margin-top: 0 !important; - padding-top: 0 !important; -} - -/* Don't show permalink of hidden ToC entries. */ -.hidden-toc a.headerlink { - display: none; -} - -/* Avoid breaking parameters name, etc. in table cells. */ -td code { - word-break: normal !important; -} - -/* For pieces of Markdown rendered in table cells. */ -td p { - margin-top: 0 !important; - margin-bottom: 0 !important; -} \ No newline at end of file diff --git a/docs/handlers/python.md b/docs/handlers/python.md index adb792e5..049ec329 100644 --- a/docs/handlers/python.md +++ b/docs/handlers/python.md @@ -329,33 +329,6 @@ div.doc-contents:not(.first) { border-left: 4px solid rgba(230, 230, 230); margin-bottom: 80px; } - -/* Don't capitalize names. */ -h5.doc-heading { - text-transform: none !important; -} - -/* Don't use vertical space on hidden ToC entries. */ -.hidden-toc::before { - margin-top: 0 !important; - padding-top: 0 !important; -} - -/* Don't show permalink of hidden ToC entries. */ -.hidden-toc a.headerlink { - display: none; -} - -/* Avoid breaking parameters name, etc. in table cells. */ -td code { - word-break: normal !important; -} - -/* For pieces of Markdown rendered in table cells. */ -td p { - margin-top: 0 !important; - margin-bottom: 0 !important; -} ``` ## Recommended style (ReadTheDocs) @@ -369,43 +342,4 @@ div.doc-contents:not(.first) { border-left: 4px solid rgba(230, 230, 230); margin-bottom: 60px; } - -/* Don't use vertical space on hidden ToC entries. */ -.hidden-toc::before { - margin-top: 0 !important; - padding-top: 0 !important; -} - -/* Don't show permalink of hidden ToC entries. */ -.hidden-toc a.headerlink { - display: none; -} - -/* Avoid breaking parameters name, etc. in table cells. */ -td code { - word-break: normal !important; -} - -/* For pieces of Markdown rendered in table cells. */ -td p { - margin-top: 0 !important; - margin-bottom: 0 !important; -} - -/* Avoid breaking code headings. */ -.doc-heading code { - white-space: normal; -} - -/* Improve rendering of parameters, returns and exceptions. */ -.field-name { - min-width: 100px; -} -.field-name, .field-body { - border: none !important; - padding: 0 !important; -} -.field-list { - margin: 0 !important; -} ``` diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index f2dac1c3..d328c9b1 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -15,7 +15,7 @@ import textwrap from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Dict, Optional, Sequence +from typing import Any, Dict, Iterable, Optional, Sequence from xml.etree.ElementTree import Element # noqa: S405 (we choose to trust the XML input) from jinja2 import Environment, FileSystemLoader @@ -161,10 +161,12 @@ class BaseRenderer(ABC): You can also override the `update_env` method, to add more filters to the Jinja environment, making them available in your Jinja templates. - To define a fallback theme, add a `FALLBACK_THEME` class-variable. + To define a fallback theme, add a `fallback_theme` class-variable. + To add custom CSS, add an `extra_css` variable or create an 'style.css' file beside the templates. """ fallback_theme: str = "" + extra_css = "" def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = None) -> None: """ @@ -180,16 +182,22 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = """ paths = [] - if custom_templates is not None: - paths.append(Path(custom_templates) / directory / theme) - themes_dir = TEMPLATES_DIR / directory paths.append(themes_dir / theme) - if self.fallback_theme != "": + if self.fallback_theme: paths.append(themes_dir / self.fallback_theme) + for path in paths: + css_path = path / "style.css" + if css_path.is_file(): + self.extra_css += "\n" + css_path.read_text(encoding="utf-8") + break + + if custom_templates is not None: + paths.insert(0, Path(custom_templates) / directory / theme) + self.env = Environment( autoescape=True, loader=FileSystemLoader(paths), @@ -378,9 +386,20 @@ def get_handler(self, name: str, handler_config: Optional[dict] = None) -> BaseH ) # type: ignore return self._handlers[name] - def teardown(self): + @property + def seen_handlers(self) -> Iterable[BaseHandler]: + """ + Get the handlers that were encountered so far throughout the build. + + Returns: + An iterable of instances of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler] + (usable only to loop through it). + """ + return self._handlers.values() + + def teardown(self) -> None: """Teardown all cached handlers and clear the cache.""" - for handler in self._handlers.values(): + for handler in self.seen_handlers: handler.collector.teardown() self._handlers.clear() diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 87f3421c..a335bdec 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -32,6 +32,7 @@ from mkdocs.plugins import BasePlugin from mkdocs.structure.pages import Page from mkdocs.structure.toc import AnchorLink +from mkdocs.utils import write_file from mkdocstrings.extension import MkdocstringsExtension from mkdocstrings.handlers.base import BaseHandler, Handlers @@ -99,6 +100,8 @@ class MkdocstringsPlugin(BasePlugin): ``` """ + css_filename = "assets/_mkdocstrings.css" + def __init__(self) -> None: """Initialize the object.""" super().__init__() @@ -167,6 +170,9 @@ def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused self.handlers = Handlers(extension_config) mkdocstrings_extension = MkdocstringsExtension(extension_config, self.handlers) config["markdown_extensions"].append(mkdocstrings_extension) + + config["extra_css"].insert(0, self.css_filename) # So that it has lower priority than user files. + return config def on_page_content(self, html: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) @@ -239,7 +245,7 @@ def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 return fixed_output - def on_post_build(self, **kwargs) -> None: # noqa: W0613,R0201 (unused arguments, cannot be static) + def on_post_build(self, config: Config, **kwargs) -> None: # noqa: W0613,R0201 (unused arguments, cannot be static) """ Teardown the handlers. @@ -253,9 +259,13 @@ def on_post_build(self, **kwargs) -> None: # noqa: W0613,R0201 (unused argument this hook. Arguments: + config: The MkDocs config object. kwargs: Additional arguments passed by MkDocs. """ if self.handlers: + css_content = "\n".join(handler.renderer.extra_css for handler in self.handlers.seen_handlers) + write_file(css_content.encode("utf-8"), os.path.join(config["site_dir"], self.css_filename)) + log.debug("Tearing handlers down") self.handlers.teardown() diff --git a/src/mkdocstrings/templates/python/material/style.css b/src/mkdocstrings/templates/python/material/style.css new file mode 100644 index 00000000..c30d885b --- /dev/null +++ b/src/mkdocstrings/templates/python/material/style.css @@ -0,0 +1,26 @@ +/* Don't capitalize names. */ +h5.doc-heading { + text-transform: none !important; +} + +/* Don't use vertical space on hidden ToC entries. */ +.hidden-toc::before { + margin-top: 0 !important; + padding-top: 0 !important; +} + +/* Don't show permalink of hidden ToC entries. */ +.hidden-toc a.headerlink { + display: none; +} + +/* Avoid breaking parameters name, etc. in table cells. */ +.doc-contents td code { + word-break: normal !important; +} + +/* For pieces of Markdown rendered in table cells. */ +.doc-contents td p { + margin-top: 0 !important; + margin-bottom: 0 !important; +} diff --git a/src/mkdocstrings/templates/python/readthedocs/style.css b/src/mkdocstrings/templates/python/readthedocs/style.css new file mode 100644 index 00000000..c16712bc --- /dev/null +++ b/src/mkdocstrings/templates/python/readthedocs/style.css @@ -0,0 +1,38 @@ +/* Don't use vertical space on hidden ToC entries. */ +.hidden-toc::before { + margin-top: 0 !important; + padding-top: 0 !important; +} + +/* Don't show permalink of hidden ToC entries. */ +.hidden-toc a.headerlink { + display: none; +} + +/* Avoid breaking parameters name, etc. in table cells. */ +.doc-contents td code { + word-break: normal !important; +} + +/* For pieces of Markdown rendered in table cells. */ +.doc-contents td p { + margin-top: 0 !important; + margin-bottom: 0 !important; +} + +/* Avoid breaking code headings. */ +.doc-heading code { + white-space: normal; +} + +/* Improve rendering of parameters, returns and exceptions. */ +.doc-contents .field-name { + min-width: 100px; +} +.doc-contents .field-name, .field-body { + border: none !important; + padding: 0 !important; +} +.doc-contents .field-list { + margin: 0 !important; +} From 7fe438c4040a2124b00c39e582ef4c38be7c55c9 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sun, 17 Jan 2021 01:29:05 +0100 Subject: [PATCH 10/40] refactor: Remove the extra wrapper div from the final doc A prior commit introduced
because a wrapper element is required, one can't just insert text into XML as a BlockProcessor. But that's an undesired change for the final doc, it might affect how people use styles. So take care to drop the wrapper. --- src/mkdocstrings/extension.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 06a88c17..50697994 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -197,10 +197,17 @@ def get_item_configs(handler_config: dict, config: dict) -> Tuple[Mapping, Mappi class _PostProcessor(Treeprocessor): def run(self, root: Element): - for el in root.iter("div"): - if el.get("class") == "mkdocstrings": - # Delete the duplicated headings from before, but keep the text (i.e. the actual HTML). - del el[:] + carry_text = "" + for el in reversed(root): # Reversed mainly for the ability to mutate during iteration. + if el.tag == "div" and el.get("class") == "mkdocstrings": + # Delete the duplicated headings along with their container, but keep the text (i.e. the actual HTML). + carry_text = (el.text or "") + carry_text + root.remove(el) + elif carry_text: + el.tail = (el.tail or "") + carry_text + carry_text = "" + if carry_text: + root.text = (root.text or "") + carry_text class MkdocstringsExtension(Extension): From 4789950ff43c354d47afbed5c89d5abb917ffee6 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sat, 16 Jan 2021 20:27:23 +0100 Subject: [PATCH 11/40] feat: Allow linking to an identifier not only by its exact anchor --- src/mkdocstrings/handlers/base.py | 36 +++++++++++++++++++++++++ src/mkdocstrings/handlers/python.py | 3 +++ src/mkdocstrings/plugin.py | 41 ++++++++++++++++++----------- src/mkdocstrings/references.py | 6 ++--- 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 1f27ebbd..ce67d56f 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -219,6 +219,20 @@ def render(self, data: CollectorItem, config: dict) -> str: The rendered template as HTML. """ # noqa: DAR202 (excess return section) + def get_anchor(self, data: Any) -> Optional[str]: + """ + Return the canonical identifier (HTML anchor) for a collected item. + + This must match what the renderer would've actually rendered, + e.g. if rendering the item contains `

...` then the return value should be "foo". + + Arguments: + data: The collected data. + + Returns: + The HTML anchor (without '#') as a string, or None if this item doesn't have an anchor. + """ # noqa: DAR202 (excess return section) + def update_env(self, md: Markdown, config: dict) -> None: """ Update the Jinja environment. @@ -320,6 +334,28 @@ def __init__(self, config: dict) -> None: """ self._config = config self._handlers: Dict[str, BaseHandler] = {} + self._url_map = {} + + def register_anchor(self, page: str, anchor: str): + """ + Register that an anchor corresponding to an identifier was encountered when rendering the page. + + Arguments: + page: The URL of the current page. + anchor: The HTML anchor (without '#') as a string + """ + self._url_map[anchor] = f"{page}#{anchor}" + + def __getitem__(self, identifier: str) -> str: + try: + return self._url_map[identifier] + except KeyError: + for handler in self._handlers.values(): + try: + return self._url_map[handler.renderer.get_anchor(handler.collector.collect(identifier, {}))] + except (CollectionError, KeyError): + continue + raise def get_handler_name(self, config: dict) -> str: """ diff --git a/src/mkdocstrings/handlers/python.py b/src/mkdocstrings/handlers/python.py index 8716d558..4548d0dd 100644 --- a/src/mkdocstrings/handlers/python.py +++ b/src/mkdocstrings/handlers/python.py @@ -79,6 +79,9 @@ def render(self, data: CollectorItem, config: dict) -> str: # noqa: D102 (ignor **{"config": final_config, data["category"]: data, "heading_level": heading_level, "root": True}, ) + def get_anchor(self, data: CollectorItem) -> str: # noqa: D102 (ignore missing docstring) + return data.get("path") + def update_env(self, md: Markdown, config: dict) -> None: # noqa: D102 (ignore missing docstring) super().update_env(md, config) self.env.trim_blocks = True diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 2d5dc964..298fdca6 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -24,7 +24,7 @@ import logging import os -from typing import Callable, Dict, Optional, Tuple +from typing import Callable, Optional, Tuple from livereload import Server from mkdocs.config import Config @@ -34,7 +34,7 @@ from mkdocs.structure.toc import AnchorLink from mkdocstrings.extension import MkdocstringsExtension -from mkdocstrings.handlers.base import BaseHandler, CollectorItem, Handlers +from mkdocstrings.handlers.base import BaseHandler, Handlers from mkdocstrings.loggers import get_logger from mkdocstrings.references import fix_refs @@ -102,8 +102,22 @@ class MkdocstringsPlugin(BasePlugin): def __init__(self) -> None: """Initialize the object.""" super().__init__() - self.url_map: Dict[CollectorItem, str] = {} - self.handlers: Optional[Handlers] = None + self._handlers: Optional[Handlers] = None + + @property + def handlers(self) -> Handlers: + """ + Get the instance of [mkdocstrings.handlers.base.Handlers][] for this plugin/build. + + Raises: + RuntimeError: If the plugin hasn't been initialized with a config. + + Returns: + An instance of [mkdocstrings.handlers.base.Handlers][] (the same throughout the build). + """ + if not self._handlers: + raise RuntimeError("The plugin hasn't been initialized with a config yet") + return self._handlers def on_serve(self, server: Server, builder: Callable = None, **kwargs) -> Server: # noqa: W0613 (unused arguments) """ @@ -164,8 +178,8 @@ def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused "mkdocstrings": self.config, } - self.handlers = Handlers(extension_config) - mkdocstrings_extension = MkdocstringsExtension(extension_config, self.handlers) + self._handlers = Handlers(extension_config) + mkdocstrings_extension = MkdocstringsExtension(extension_config, self._handlers) config["markdown_extensions"].append(mkdocstrings_extension) return config @@ -195,13 +209,13 @@ def map_urls(self, base_url: str, anchor: AnchorLink) -> None: """ Recurse on every anchor to map its ID to its absolute URL. - This method populates `self.url_map` by side-effect. + This method populates `self.handlers.url_map` by side-effect. Arguments: base_url: The base URL to use as a prefix for each anchor's relative URL. anchor: The anchor to process and to recurse on. """ - self.url_map[anchor.id] = base_url + anchor.url + self.handlers.register_anchor(base_url, anchor.id) for child in anchor.children: self.map_urls(base_url, child) @@ -229,7 +243,7 @@ def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 """ log.debug(f"Fixing references in page {page.file.src_path}") - fixed_output, unmapped = fix_refs(output, page.url, self.url_map) + fixed_output, unmapped = fix_refs(output, page.url, self.handlers) if unmapped and log.isEnabledFor(logging.WARNING): for ref in unmapped: @@ -255,9 +269,9 @@ def on_post_build(self, **kwargs) -> None: # noqa: W0613,R0201 (unused argument Arguments: kwargs: Additional arguments passed by MkDocs. """ - if self.handlers: + if self._handlers: log.debug("Tearing handlers down") - self.handlers.teardown() + self._handlers.teardown() def get_handler(self, handler_name: str) -> BaseHandler: """ @@ -266,12 +280,7 @@ def get_handler(self, handler_name: str) -> BaseHandler: Arguments: handler_name: The name of the handler. - Raises: - RuntimeError: If the plugin hasn't been initialized with a config. - Returns: An instance of a subclass of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler]. """ - if not self.handlers: - raise RuntimeError("The plugin hasn't been initialized with a config yet") return self.handlers.get_handler(handler_name) diff --git a/src/mkdocstrings/references.py b/src/mkdocstrings/references.py index 38957677..a71f5971 100644 --- a/src/mkdocstrings/references.py +++ b/src/mkdocstrings/references.py @@ -2,7 +2,7 @@ import re from html import escape, unescape -from typing import Any, Callable, Dict, List, Match, Tuple, Union +from typing import Any, Callable, List, Mapping, Match, Tuple, Union from xml.etree.ElementTree import Element # noqa: S405 (input is our own, and Markdown coming from code) from markdown.inlinepatterns import REFERENCE_RE, ReferenceInlineProcessor @@ -118,7 +118,7 @@ def relative_url(url_a: str, url_b: str) -> str: return f"{relative}#{anchor}" -def fix_ref(url_map: Dict[str, str], from_url: str, unmapped: List[str]) -> Callable: +def fix_ref(url_map: Mapping[str, str], from_url: str, unmapped: List[str]) -> Callable: """ Return a `repl` function for [`re.sub`](https://docs.python.org/3/library/re.html#re.sub). @@ -158,7 +158,7 @@ def inner(match: Match): def fix_refs( html: str, from_url: str, - url_map: Dict[str, str], + url_map: Mapping[str, str], ) -> Tuple[str, List[str]]: """ Fix all references in the given HTML text. From 96c5d004be05f96dbef06708fcff42098dd08b07 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sun, 17 Jan 2021 19:16:37 +0100 Subject: [PATCH 12/40] refactor: Disambiguate the role of Handlers vs url_map --- src/mkdocstrings/handlers/base.py | 14 +++++++++++++- src/mkdocstrings/plugin.py | 2 +- src/mkdocstrings/references.py | 16 +++++++++------- tests/test_references.py | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 0ab267d9..c6518dfd 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -412,7 +412,19 @@ def register_anchor(self, page: str, anchor: str): """ self._url_map[anchor] = f"{page}#{anchor}" - def __getitem__(self, identifier: str) -> str: + def get_item_url(self, identifier: str) -> str: + """ + Return a site-relative URL with anchor to the identifier, if it's present anywhere. + + Arguments: + identifier: The identifier (one that [collect][mkdocstrings.handlers.base.BaseCollector.collect] can accept). + + Returns: + A site-relative URL. + + Raises: + KeyError: If there isn't an item by this identifier anywhere on the site. + """ try: return self._url_map[identifier] except KeyError: diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 298fdca6..6b388c87 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -243,7 +243,7 @@ def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 """ log.debug(f"Fixing references in page {page.file.src_path}") - fixed_output, unmapped = fix_refs(output, page.url, self.handlers) + fixed_output, unmapped = fix_refs(output, page.url, self.handlers.get_item_url) if unmapped and log.isEnabledFor(logging.WARNING): for ref in unmapped: diff --git a/src/mkdocstrings/references.py b/src/mkdocstrings/references.py index 6f1bcb29..b68721f5 100644 --- a/src/mkdocstrings/references.py +++ b/src/mkdocstrings/references.py @@ -2,7 +2,7 @@ import re from html import escape, unescape -from typing import Any, Callable, List, Mapping, Match, Tuple, Union +from typing import Any, Callable, List, Match, Tuple, Union from xml.etree.ElementTree import Element from markdown.inlinepatterns import REFERENCE_RE, ReferenceInlineProcessor @@ -118,7 +118,7 @@ def relative_url(url_a: str, url_b: str) -> str: return f"{relative}#{anchor}" -def fix_ref(url_map: Mapping[str, str], from_url: str, unmapped: List[str]) -> Callable: +def fix_ref(url_mapper: Callable[[str], str], from_url: str, unmapped: List[str]) -> Callable: """ Return a `repl` function for [`re.sub`](https://docs.python.org/3/library/re.html#re.sub). @@ -129,7 +129,8 @@ def fix_ref(url_map: Mapping[str, str], from_url: str, unmapped: List[str]) -> C and rendered, making it impossible to link to it. We catch this exception in the caller to issue a warning. Arguments: - url_map: The mapping of objects and their URLs. + url_mapper: A callable that gets an object's site URL by its identifier, + such as [mkdocstrings.handlers.base.Handlers.get_item_url][]. from_url: The URL of the base page, from which we link towards the targeted pages. unmapped: A list to store unmapped identifiers. @@ -143,7 +144,7 @@ def inner(match: Match): title = match["title"] try: - url = relative_url(from_url, url_map[unescape(identifier)]) + url = relative_url(from_url, url_mapper(unescape(identifier))) except KeyError: unmapped.append(identifier) if title == identifier: @@ -158,7 +159,7 @@ def inner(match: Match): def fix_refs( html: str, from_url: str, - url_map: Mapping[str, str], + url_mapper: Callable[[str], str], ) -> Tuple[str, List[str]]: """ Fix all references in the given HTML text. @@ -166,11 +167,12 @@ def fix_refs( Arguments: html: The text to fix. from_url: The URL at which this HTML is served. - url_map: The mapping of objects and their URLs. + url_mapper: A callable that gets an object's site URL by its identifier, + such as [mkdocstrings.handlers.base.Handlers.get_item_url][]. Returns: The fixed HTML. """ unmapped = [] # type: ignore - html = AUTO_REF_RE.sub(fix_ref(url_map, from_url, unmapped), html) + html = AUTO_REF_RE.sub(fix_ref(url_mapper, from_url, unmapped), html) return html, unmapped diff --git a/tests/test_references.py b/tests/test_references.py index bee270e6..5ba6acca 100644 --- a/tests/test_references.py +++ b/tests/test_references.py @@ -59,7 +59,7 @@ def run_references_test(url_map, source, output, unmapped=None, from_url="page.h ext = MkdocstringsExtension(config, Handlers(config)) md = markdown.Markdown(extensions=[ext]) content = md.convert(source) - actual_output, actual_unmapped = fix_refs(content, from_url, url_map) + actual_output, actual_unmapped = fix_refs(content, from_url, url_map.__getitem__) assert actual_output == output assert actual_unmapped == (unmapped or []) From a16fd52a690527bbd9fceac954740f64b83d0fe6 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sun, 17 Jan 2021 19:53:17 +0100 Subject: [PATCH 13/40] docs: Document the (partly new) ability to link to identifiers --- docs/usage.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 3f95d894..ef634df9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,8 +1,10 @@ # Usage +## Autodoc syntax + MkDocstrings works by processing special expressions in your Markdown files. -The syntax is as follow: +The syntax is as follows: ```md ::: identifier @@ -152,14 +154,24 @@ Cross-references are written as Markdown *reference-style* links: === "HTML Result" ```html

With a custom title: - Object 1

+ Object 1

With the identifier as title: - full.path.object2

+ full.path.object2

``` +Any item that was inserted using the [autodoc syntax](#autodoc-syntax) +(e.g. `::: full.path.object1`) is possible to link to by using the same identifier with the +cross-reference syntax (`[example][full.path.object1]`). +But the cross-references are also applicable to the items' children that get pulled in. + +If you're not sure which exact identifier a doc item uses, you can look at its "anchor", which your +Web browser will show in the URL bar when clicking an item's entry in the table of contents. +If the URL is `https://example.com/some/page.html#full.path.object1` then you know that this item +is possible to link to with `[example][full.path.object1]`, regardless of the current page. + ## Themes -MkDocstrings can support multiple MkDocs theme. +MkDocstrings can support multiple MkDocs themes. It currently supports supports the *[Material for MkDocs](https://squidfunk.github.io/mkdocs-material/)* theme and, partially, the built-in ReadTheDocs theme. From fc676564f9b11269b3e0b0482703ac924069a3fa Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Wed, 20 Jan 2021 00:40:15 +0100 Subject: [PATCH 14/40] refactor: Split out autorefs plugin, make it optional "autorefs" is now a self-sufficient MkDocs plugin, just allowing linking to headings on any page. As for mkdocstrings, unless users opt into using autorefs for any heading (done by scanning the ToC as before), this will now only do autorefs to mkdocstrings identifiers. Opting in is done by just enabling 'autorefs' as a separate plugin, which it now is. The new behavior of linking only to identifiers is implemented by force-including the autorefs plugin even if it's not specified, but in a "subdued" mode. Idea inspired by the interaction between pymdownx.{superfences,highlight}. --- docs/reference/references.md | 2 +- duties.py | 6 +- pyproject.toml | 6 +- src/mkdocs_autorefs/plugin.py | 160 ++++++++++++++++++ .../references.py | 21 +++ src/mkdocstrings/extension.py | 22 ++- src/mkdocstrings/handlers/base.py | 38 ++--- src/mkdocstrings/plugin.py | 90 ++-------- tests/test_extension.py | 42 ++--- tests/test_references.py | 9 +- 10 files changed, 253 insertions(+), 143 deletions(-) create mode 100644 src/mkdocs_autorefs/plugin.py rename src/{mkdocstrings => mkdocs_autorefs}/references.py (89%) diff --git a/docs/reference/references.md b/docs/reference/references.md index 11be1ca9..2a23e10e 100644 --- a/docs/reference/references.md +++ b/docs/reference/references.md @@ -1 +1 @@ -::: mkdocstrings.references \ No newline at end of file +::: mkdocs_autorefs.references diff --git a/duties.py b/duties.py index f21ce190..cde546c8 100644 --- a/duties.py +++ b/duties.py @@ -16,8 +16,7 @@ from jinja2.sandbox import SandboxedEnvironment from pip._internal.commands.show import search_packages_info -PY_SRC_PATHS = (Path(_) for _ in ("src/mkdocstrings", "tests", "duties.py")) -PY_SRC_LIST = tuple(str(_) for _ in PY_SRC_PATHS) +PY_SRC_LIST = ("src/mkdocstrings", "src/mkdocs_autorefs", "tests", "duties.py") PY_SRC = " ".join(PY_SRC_LIST) TESTING = os.environ.get("TESTING", "0") in {"1", "true"} CI = os.environ.get("CI", "0") in {"1", "true", "yes", ""} @@ -214,7 +213,8 @@ def check_types(ctx): Arguments: ctx: The context instance (passed automatically). """ - ctx.run(f"mypy --config-file config/mypy.ini {PY_SRC}", title="Type-checking", pty=PTY) + for f in PY_SRC_LIST: + ctx.run(f"mypy --config-file config/mypy.ini {f}", title="Type-checking", pty=PTY, progress=True) @duty(silent=True) diff --git a/pyproject.toml b/pyproject.toml index e6290e50..ec585772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,10 @@ readme = "README.md" repository = "https://github.com/pawamoy/mkdocstrings" homepage = "https://github.com/pawamoy/mkdocstrings" keywords = ["mkdocs", "mkdocs-plugin", "docstrings", "autodoc", "documentation"] -packages = [ { include = "mkdocstrings", from = "src" } ] +packages = [ + { include = "mkdocstrings", from = "src" }, + { include = "mkdocs_autorefs", from = "src" } +] include = [ "README.md", "pyproject.toml" @@ -61,6 +64,7 @@ pep8-naming = "^0.11.1" [tool.poetry.plugins."mkdocs.plugins"] mkdocstrings = "mkdocstrings.plugin:MkdocstringsPlugin" +autorefs = "mkdocs_autorefs.plugin:AutorefsPlugin" [tool.black] line-length = 120 diff --git a/src/mkdocs_autorefs/plugin.py b/src/mkdocs_autorefs/plugin.py new file mode 100644 index 00000000..a6b1ab4f --- /dev/null +++ b/src/mkdocs_autorefs/plugin.py @@ -0,0 +1,160 @@ +import logging +from typing import Callable, Dict, Optional + +from mkdocs.config import Config +from mkdocs.plugins import BasePlugin +from mkdocs.structure.pages import Page +from mkdocs.structure.toc import AnchorLink +from mkdocs.utils import warning_filter + +from mkdocs_autorefs.references import AutorefsExtension, fix_refs + +log = logging.getLogger(f"mkdocs.plugins.{__name__}") +log.addFilter(warning_filter) + + +class AutorefsPlugin(BasePlugin): + scan_toc: bool = True + current_page: Optional[str] = None + + def __init__(self) -> None: + """Initialize the object.""" + super().__init__() + self._url_map: Dict[str, str] = {} + self.get_fallback_anchor: Callable[[str], Optional[str]] = lambda identifier: None + + def register_anchor(self, page: str, anchor: str): + """ + Register that an anchor corresponding to an identifier was encountered when rendering the page. + + Arguments: + page: The URL of the current page. + anchor: The HTML anchor (without '#') as a string + """ + self._url_map[anchor] = f"{page}#{anchor}" + + def get_item_url(self, anchor: str) -> str: + """ + Return a site-relative URL with anchor to the identifier, if it's present anywhere. + + Arguments: + anchor: The identifier (one that [collect][mkdocstrings.handlers.base.BaseCollector.collect] can accept). + + Returns: + A site-relative URL. + + Raises: + KeyError: If there isn't an item by this identifier anywhere on the site. + """ + try: + return self._url_map[anchor] + except KeyError: + new_anchor = self.get_fallback_anchor(anchor) + if new_anchor and new_anchor in self._url_map: + return self._url_map[new_anchor] + raise + + def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused arguments) + """ + Instantiate our Markdown extension. + + Hook for the [`on_config` event](https://www.mkdocs.org/user-guide/plugins/#on_config). + In this hook, we instantiate our [`AutorefsExtension`][mkdocs_autorefs.references.AutorefsExtension] + and add it to the list of Markdown extensions used by `mkdocs`. + + Arguments: + config: The MkDocs config object. + kwargs: Additional arguments passed by MkDocs. + + Returns: + The modified config. + """ + log.debug("Adding AutorefsExtension to the list") + config["markdown_extensions"].append(AutorefsExtension()) + return config + + def on_page_markdown(self, markdown: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) + """ + Remember which page is the current one. + + Arguments: + markdown: Input Markdown. + page: The related MkDocs page instance. + kwargs: Additional arguments passed by MkDocs. + + Returns: + The same Markdown. We only use this hook to map anchors to URLs. + """ + self.current_page = page.url + return markdown + + def on_page_content(self, html: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) + """ + Map anchors to URLs. + + Hook for the [`on_page_contents` event](https://www.mkdocs.org/user-guide/plugins/#on_page_contents). + In this hook, we map the IDs of every anchor found in the table of contents to the anchors absolute URLs. + This mapping will be used later to fix unresolved reference of the form `[title][identifier]` or + `[identifier][]`. + + Arguments: + html: HTML converted from Markdown. + page: The related MkDocs page instance. + kwargs: Additional arguments passed by MkDocs. + + Returns: + The same HTML. We only use this hook to map anchors to URLs. + """ + if self.scan_toc: + log.debug(f"Mapping identifiers to URLs for page {page.file.src_path}") + for item in page.toc.items: + self.map_urls(page.url, item) + return html + + def map_urls(self, base_url: str, anchor: AnchorLink) -> None: + """ + Recurse on every anchor to map its ID to its absolute URL. + + This method populates `self.url_map` by side-effect. + + Arguments: + base_url: The base URL to use as a prefix for each anchor's relative URL. + anchor: The anchor to process and to recurse on. + """ + self.register_anchor(base_url, anchor.id) + for child in anchor.children: + self.map_urls(base_url, child) + + def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) + """ + Fix cross-references. + + Hook for the [`on_post_page` event](https://www.mkdocs.org/user-guide/plugins/#on_post_page). + In this hook, we try to fix unresolved references of the form `[title][identifier]` or `[identifier][]`. + Doing that allows the user of `mkdocstrings` to cross-reference objects in their documentation strings. + It uses the native Markdown syntax so it's easy to remember and use. + + We log a warning for each reference that we couldn't map to an URL, but try to be smart and ignore identifiers + that do not look legitimate (sometimes documentation can contain strings matching + our [`AUTO_REF_RE`][mkdocs_autorefs.references.AUTO_REF_RE] regular expression that did not intend to reference anything). + We currently ignore references when their identifier contains a space or a slash. + + Arguments: + output: HTML converted from Markdown. + page: The related MkDocs page instance. + kwargs: Additional arguments passed by MkDocs. + + Returns: + Modified HTML. + """ + log.debug(f"Fixing references in page {page.file.src_path}") + + fixed_output, unmapped = fix_refs(output, page.url, self.get_item_url) + + if unmapped and log.isEnabledFor(logging.WARNING): + for ref in unmapped: + log.warning( + f"{page.file.src_path}: Could not find cross-reference target '[{ref}]'", + ) + + return fixed_output diff --git a/src/mkdocstrings/references.py b/src/mkdocs_autorefs/references.py similarity index 89% rename from src/mkdocstrings/references.py rename to src/mkdocs_autorefs/references.py index b68721f5..1d7987bc 100644 --- a/src/mkdocstrings/references.py +++ b/src/mkdocs_autorefs/references.py @@ -5,6 +5,8 @@ from typing import Any, Callable, List, Match, Tuple, Union from xml.etree.ElementTree import Element +from markdown import Markdown +from markdown.extensions import Extension from markdown.inlinepatterns import REFERENCE_RE, ReferenceInlineProcessor AUTO_REF_RE = re.compile(r'[^"<>]*)\1>(?P.*?)</span>') @@ -176,3 +178,22 @@ def fix_refs( unmapped = [] # type: ignore html = AUTO_REF_RE.sub(fix_ref(url_mapper, from_url, unmapped), html) return html, unmapped + + +class AutorefsExtension(Extension): + """Extension that inserts auto-references in Markdown.""" + + def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) + """ + Register the extension. + + Add an instance of our [`AutoRefInlineProcessor`][mkdocs_autorefs.references.AutoRefInlineProcessor] to the Markdown parser. + + Arguments: + md: A `markdown.Markdown` instance. + """ + md.inlinePatterns.register( + AutoRefInlineProcessor(md), + "mkdocstrings", + priority=168, # Right after markdown.inlinepatterns.ReferenceInlineProcessor + ) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 46290461..ec40cf03 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -35,9 +35,9 @@ from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor +from mkdocs_autorefs.plugin import AutorefsPlugin from mkdocstrings.handlers.base import CollectionError, CollectorItem, Handlers from mkdocstrings.loggers import get_logger -from mkdocstrings.references import AutoRefInlineProcessor log = get_logger(__name__) @@ -55,7 +55,9 @@ class AutoDocProcessor(BlockProcessor): regex = re.compile(r"^(?P<heading>#{1,6} *|)::: ?(?P<name>.+?) *$", flags=re.MULTILINE) - def __init__(self, parser: BlockParser, md: Markdown, config: dict, handlers: Handlers) -> None: + def __init__( + self, parser: BlockParser, md: Markdown, config: dict, handlers: Handlers, autorefs: AutorefsPlugin + ) -> None: """ Initialize the object. @@ -65,11 +67,13 @@ def __init__(self, parser: BlockParser, md: Markdown, config: dict, handlers: Ha config: The [configuration][mkdocstrings.plugin.MkdocstringsPlugin.config_scheme] of the `mkdocstrings` plugin. handlers: A [mkdocstrings.handlers.base.Handlers][] instance. + autorefs: A [mkdocs_aurorefs.plugin.AutorefsPlugin][] instance. """ super().__init__(parser=parser) self.md = md self._config = config self._handlers = handlers + self._autorefs = autorefs self._updated_env = False def test(self, parent: Element, block: str) -> bool: @@ -119,6 +123,9 @@ def run(self, parent: Element, blocks: MutableSequence[str]) -> None: # So we need to duplicate the headings directly (and delete later), just so 'toc' can pick them up. el.extend(headings) + for heading in headings: + self._autorefs.register_anchor(self._autorefs.current_page, heading.attrib["id"]) + parent.append(el) if the_rest: @@ -217,7 +224,7 @@ class MkdocstringsExtension(Extension): It cannot work outside of `mkdocstrings`. """ - def __init__(self, config: dict, handlers: Handlers, **kwargs) -> None: + def __init__(self, config: dict, handlers: Handlers, autorefs: AutorefsPlugin, **kwargs) -> None: """ Initialize the object. @@ -225,11 +232,13 @@ def __init__(self, config: dict, handlers: Handlers, **kwargs) -> None: config: The configuration items from `mkdocs` and `mkdocstrings` that must be passed to the block processor when instantiated in [`extendMarkdown`][mkdocstrings.extension.MkdocstringsExtension.extendMarkdown]. handlers: A [mkdocstrings.handlers.base.Handlers][] instance. + autorefs: A [mkdocs_aurorefs.plugin.AutorefsPlugin][] instance. kwargs: Keyword arguments used by `markdown.extensions.Extension`. """ super().__init__(**kwargs) self._config = config self._handlers = handlers + self._autorefs = autorefs def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) """ @@ -241,7 +250,7 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me md: A `markdown.Markdown` instance. """ md.parser.blockprocessors.register( - AutoDocProcessor(md.parser, md, self._config, self._handlers), + AutoDocProcessor(md.parser, md, self._config, self._handlers, self._autorefs), "mkdocstrings", priority=75, # Right before markdown.blockprocessors.HashHeaderProcessor ) @@ -250,8 +259,3 @@ def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent me "mkdocstrings_post", priority=4, # Right after 'toc'. ) - md.inlinePatterns.register( - AutoRefInlineProcessor(md), - "mkdocstrings", - priority=168, # Right after markdown.inlinepatterns.ReferenceInlineProcessor - ) diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index c6518dfd..4f1a22d1 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -400,40 +400,26 @@ def __init__(self, config: dict) -> None: """ self._config = config self._handlers: Dict[str, BaseHandler] = {} - self._url_map = {} - def register_anchor(self, page: str, anchor: str): + def get_anchor(self, identifier: str) -> Optional[str]: """ - Register that an anchor corresponding to an identifier was encountered when rendering the page. - - Arguments: - page: The URL of the current page. - anchor: The HTML anchor (without '#') as a string - """ - self._url_map[anchor] = f"{page}#{anchor}" - - def get_item_url(self, identifier: str) -> str: - """ - Return a site-relative URL with anchor to the identifier, if it's present anywhere. + Return the canonical HTML anchor for the identifier, if any of the seen handlers can collect it. Arguments: identifier: The identifier (one that [collect][mkdocstrings.handlers.base.BaseCollector.collect] can accept). Returns: - A site-relative URL. - - Raises: - KeyError: If there isn't an item by this identifier anywhere on the site. + A string - anchor without '#', or None if there isn't any identifier familiar with it. """ - try: - return self._url_map[identifier] - except KeyError: - for handler in self._handlers.values(): - try: - return self._url_map[handler.renderer.get_anchor(handler.collector.collect(identifier, {}))] - except (CollectionError, KeyError): - continue - raise + for handler in self._handlers.values(): + try: + anchor = handler.renderer.get_anchor(handler.collector.collect(identifier, {})) + except CollectionError: + continue + else: + if anchor is not None: + return anchor + return None def get_handler_name(self, config: dict) -> str: """ diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 6b388c87..eb759446 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -22,7 +22,6 @@ during the [`on_serve` event hook](https://www.mkdocs.org/user-guide/plugins/#on_serve). """ -import logging import os from typing import Callable, Optional, Tuple @@ -30,13 +29,11 @@ from mkdocs.config import Config from mkdocs.config.config_options import Type as MkType from mkdocs.plugins import BasePlugin -from mkdocs.structure.pages import Page -from mkdocs.structure.toc import AnchorLink +from mkdocs_autorefs.plugin import AutorefsPlugin from mkdocstrings.extension import MkdocstringsExtension from mkdocstrings.handlers.base import BaseHandler, Handlers from mkdocstrings.loggers import get_logger -from mkdocstrings.references import fix_refs log = get_logger(__name__) @@ -177,82 +174,23 @@ def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused "mdx_configs": config["mdx_configs"], "mkdocstrings": self.config, } - self._handlers = Handlers(extension_config) - mkdocstrings_extension = MkdocstringsExtension(extension_config, self._handlers) + + try: + # If autorefs plugin is explicitly enabled, just use it. + autorefs = config["plugins"]["autorefs"] + except KeyError: + # Otherwise, add a limited instance of it that acts only on what's added through `register_anchor`. + autorefs = AutorefsPlugin() + autorefs.scan_toc = False + config["plugins"]["autorefs"] = autorefs + # Add collector-based fallback in either case. + autorefs.get_fallback_anchor = self._handlers.get_anchor + + mkdocstrings_extension = MkdocstringsExtension(extension_config, self._handlers, autorefs) config["markdown_extensions"].append(mkdocstrings_extension) return config - def on_page_content(self, html: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) - """ - Map anchors to URLs. - - Hook for the [`on_page_contents` event](https://www.mkdocs.org/user-guide/plugins/#on_page_contents). - In this hook, we map the IDs of every anchor found in the table of contents to the anchors absolute URLs. - This mapping will be used later to fix unresolved reference of the form `[title][identifier]` or - `[identifier][]`. - - Arguments: - html: HTML converted from Markdown. - page: The related MkDocs page instance. - kwargs: Additional arguments passed by MkDocs. - - Returns: - The same HTML. We only use this hook to map anchors to URLs. - """ - log.debug(f"Mapping identifiers to URLs for page {page.file.src_path}") - for item in page.toc.items: - self.map_urls(page.url, item) - return html - - def map_urls(self, base_url: str, anchor: AnchorLink) -> None: - """ - Recurse on every anchor to map its ID to its absolute URL. - - This method populates `self.handlers.url_map` by side-effect. - - Arguments: - base_url: The base URL to use as a prefix for each anchor's relative URL. - anchor: The anchor to process and to recurse on. - """ - self.handlers.register_anchor(base_url, anchor.id) - for child in anchor.children: - self.map_urls(base_url, child) - - def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) - """ - Fix cross-references. - - Hook for the [`on_post_page` event](https://www.mkdocs.org/user-guide/plugins/#on_post_page). - In this hook, we try to fix unresolved references of the form `[title][identifier]` or `[identifier][]`. - Doing that allows the user of `mkdocstrings` to cross-reference objects in their documentation strings. - It uses the native Markdown syntax so it's easy to remember and use. - - We log a warning for each reference that we couldn't map to an URL, but try to be smart and ignore identifiers - that do not look legitimate (sometimes documentation can contain strings matching - our [`AUTO_REF_RE`][mkdocstrings.references.AUTO_REF_RE] regular expression that did not intend to reference anything). - We currently ignore references when their identifier contains a space or a slash. - - Arguments: - output: HTML converted from Markdown. - page: The related MkDocs page instance. - kwargs: Additional arguments passed by MkDocs. - - Returns: - Modified HTML. - """ - log.debug(f"Fixing references in page {page.file.src_path}") - - fixed_output, unmapped = fix_refs(output, page.url, self.handlers.get_item_url) - - if unmapped and log.isEnabledFor(logging.WARNING): - for ref in unmapped: - log.warning( - f"{page.file.src_path}: Could not find cross-reference target '[{ref}]'", - ) - - return fixed_output - def on_post_build(self, **kwargs) -> None: # noqa: W0613,R0201 (unused arguments, cannot be static) """ Teardown the handlers. diff --git a/tests/test_extension.py b/tests/test_extension.py index f47ea8e4..d2058dac 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -1,13 +1,11 @@ """Tests for the extension module.""" -import copy +from collections import ChainMap from contextlib import contextmanager from textwrap import dedent import pytest from markdown import Markdown - -from mkdocstrings.extension import MkdocstringsExtension -from mkdocstrings.handlers.base import Handlers +from mkdocs import config @contextmanager @@ -20,25 +18,27 @@ def ext_markdown(**kwargs): Yields: A `markdown.Markdown` instance. """ - config = { - "theme_name": "material", - "mdx": [], - "mdx_configs": {}, - "mkdocstrings": {"default_handler": "python", "custom_templates": None, "watch": [], "handlers": {}}, + conf = config.Config(schema=config.DEFAULT_SCHEMA) + + conf_dict = { + "site_name": "foo", + "plugins": [{"mkdocstrings": {"default_handler": "python"}}], + **kwargs, } - config.update(kwargs) - config["mdx"].append("toc") # Guaranteed to be added by MkDocs. - original_config = copy.deepcopy(config) + # Re-create it manually as a workaround for https://github.com/mkdocs/mkdocs/issues/2289 + mdx_configs = dict(ChainMap(*conf_dict.get("markdown_extensions", []))) - handlers = Handlers(config) - extension = MkdocstringsExtension(config, handlers) - config["mdx"].append(extension) - original_config["mdx"].append(extension) + conf.load_dict(conf_dict) + assert conf.validate() == ([], []) - yield Markdown(extensions=config["mdx"], extension_configs=config["mdx_configs"]) - handlers.teardown() + conf["mdx_configs"] = mdx_configs + conf["markdown_extensions"].insert(0, "toc") # Guaranteed to be added by MkDocs. - assert config == original_config # Inadvertent mutations would propagate to the outer doc! + conf = conf["plugins"]["mkdocstrings"].on_config(conf) + conf = conf["plugins"]["autorefs"].on_config(conf) + md = Markdown(extensions=conf["markdown_extensions"], extension_configs=conf["mdx_configs"]) + yield md + conf["plugins"]["mkdocstrings"].on_post_build() def test_render_html_escaped_sequences(): @@ -49,7 +49,7 @@ def test_render_html_escaped_sequences(): def test_multiple_footnotes(): """Assert footnotes don't get added to subsequent docstrings.""" - with ext_markdown(mdx=["footnotes"]) as md: + with ext_markdown(markdown_extensions=[{"footnotes": {}}]) as md: output = md.convert( dedent( """ @@ -120,7 +120,7 @@ def test_no_double_toc(permalink_setting, expect_permalink): permalink_setting: The 'permalink' setting of 'toc' extension. expect_permalink: Text of the permalink to search for in the output. """ - with ext_markdown(mdx_configs={"toc": {"permalink": permalink_setting}}) as md: + with ext_markdown(markdown_extensions=[{"toc": {"permalink": permalink_setting}}]) as md: output = md.convert( dedent( """ diff --git a/tests/test_references.py b/tests/test_references.py index 5ba6acca..fd587910 100644 --- a/tests/test_references.py +++ b/tests/test_references.py @@ -2,9 +2,7 @@ import markdown import pytest -from mkdocstrings.extension import MkdocstringsExtension -from mkdocstrings.handlers.base import Handlers -from mkdocstrings.references import fix_refs, relative_url +from mkdocs_autorefs.references import AutorefsExtension, fix_refs, relative_url @pytest.mark.parametrize( @@ -55,10 +53,9 @@ def run_references_test(url_map, source, output, unmapped=None, from_url="page.h unmapped: The expected unmapped list. from_url: The source page URL. """ - config = {} - ext = MkdocstringsExtension(config, Handlers(config)) - md = markdown.Markdown(extensions=[ext]) + md = markdown.Markdown(extensions=[AutorefsExtension()]) content = md.convert(source) + actual_output, actual_unmapped = fix_refs(content, from_url, url_map.__getitem__) assert actual_output == output assert actual_unmapped == (unmapped or []) From 4dd6e98ab44714f65322e4e93c7196ba98d396fa Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Sun, 31 Jan 2021 15:30:56 +0100 Subject: [PATCH 15/40] docs: Add docs of handlers' templates and extra CSS --- docs/handlers/overview.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/handlers/overview.md b/docs/handlers/overview.md index c1c2c4d4..06fb73b8 100644 --- a/docs/handlers/overview.md +++ b/docs/handlers/overview.md @@ -61,6 +61,32 @@ arguments, and you can add any other keyword argument you'd like. The global configuration items (other than `selection` and `rendering`) will be passed to this function when getting your handler. +### Templates + +You renderer's implementation should normally be backed by templates, which go +to the directory `mkdocstrings/handlers/custom_handler/some_theme`. +(`custom_handler` here should be replaced with the actual name of your handler, +and `some_theme` should be the name of an actual MkDocs theme that you support, +e.g. `material`). + +With that structure, you can use `self.env.get_template("foo.html")` inside +your `render` implementation. This already chooses the subdirectory based on +the current MkDocs theme. + +If you wish to support *any* MkDocs theme, rather than a few specifically +selected ones, you can pick one theme's subdirectory to be the fallback for +when an unknown theme is encountered. Then you just need to set the +`fallback_theme` variable on your renderer subclass. The fallback directory can +be used even for themes you explicitly support: you can omit some template from +one of the other theme directories in case they're exactly the same as in the +fallback theme. + +If your theme's HTML requires CSS to go along with it, put it into a file named +`mkdocstrings/handlers/custom_handler/some_theme/style.css`, then this will be +included into the final site automatically if this handler is ever used. +Alternatively, you can put the CSS as a string into the `extra_css` variable of +your renderer. + ### Usage When a custom handler is installed, From f9ea00979545e39983ba377f1930d73ae94165ea Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Thu, 4 Feb 2021 19:40:23 +0100 Subject: [PATCH 16/40] refactor: Theme-agnostic code highlighting, respecting configs Infer the highlighter configs from what the current Markdown config is, trying to mimick the two most typical extensions and how they'd highlight a code block in Markdown normally. This improves theme support, as previously only some assumptions based on each theme's defaults were supported, e.g. the readthedocs theme is assumed to rely only on JS highlighting, when in reality you just need to add a Markdown syntax highlight extension and you can use Pygments. Another improvement stemming from this is support for disabling syntax highlighting altogether. (`use_pygments: false`). For that I also had to edit the template for function signature, to avoid all newlines in it, because previously inline highlighting relied on processing of the syntax highlighter to get rid of those. PR #202: https://github.com/pawamoy/mkdocstrings/pull/202 --- src/mkdocstrings/handlers/base.py | 131 ++++++++---------- .../templates/python/material/class.html | 2 +- .../templates/python/material/examples.html | 2 +- .../templates/python/material/function.html | 2 +- .../templates/python/material/method.html | 2 +- .../templates/python/material/signature.html | 54 ++++---- tests/test_handlers.py | 48 +++++++ 7 files changed, 139 insertions(+), 102 deletions(-) create mode 100644 tests/test_handlers.py diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 4017b076..76a998a4 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -21,9 +21,10 @@ from jinja2 import Environment, FileSystemLoader from markdown import Markdown from markdown.extensions import Extension +from markdown.extensions.codehilite import CodeHiliteExtension from markdown.treeprocessors import Treeprocessor from markupsafe import Markup -from pymdownx.highlight import Highlight +from pymdownx.highlight import Highlight, HighlightExtension from mkdocstrings.loggers import get_template_logger @@ -40,75 +41,69 @@ class ThemeNotSupported(Exception): """An exception raised to tell a theme is not supported.""" -def do_highlight( - src: str, - guess_lang: bool = False, - language: str = None, - inline: bool = False, - dedent: bool = True, - line_nums: bool = False, - line_start: int = 1, -) -> str: - """ - Highlight a code-snippet. +class Highlighter(Highlight): + """Code highlighter that tries to match the Markdown configuration.""" - This function is used as a filter in Jinja templates. + _highlight_config_keys = frozenset( + "use_pygments guess_lang css_class pygments_style noclasses linenums language_prefix".split(), + ) - Arguments: - src: The code to highlight. - guess_lang: Whether to guess the language or not. - language: Explicitly tell what language to use for highlighting. - inline: Whether to do inline highlighting. - dedent: Whether to dedent the code before highlighting it or not. - line_nums: Whether to add line numbers in the result. - line_start: The line number to start with. + def __init__(self, md: Markdown): + """Configure to match a `markdown.Markdown` instance. - Returns: - The highlighted code as HTML text, marked safe (not escaped for HTML). - """ - if dedent: - src = textwrap.dedent(src) - - highlighter = Highlight(use_pygments=True, guess_lang=guess_lang, linenums=line_nums) - result = highlighter.highlight(src=src, language=language, linestart=line_start, inline=inline) - - if inline: - return Markup(f'<code class="highlight language-{language}">{result.text}</code>') - return Markup(result) - - -def do_js_highlight( - src: str, - guess_lang: bool = False, # noqa: W0613 (we must accept the same parameters as do_highlight) - language: str = None, - inline: bool = False, - dedent: bool = True, - line_nums: bool = False, # noqa: W0613 - line_start: int = 1, # noqa: W0613 -) -> str: - """ - Prepare a code-snippet for JS highlighting. + Arguments: + md: The Markdown instance to read configs from. + """ + config = {} + for ext in md.registeredExtensions: + if isinstance(ext, HighlightExtension) and (ext.enabled or not config): + config = ext.getConfigs() + break # This one takes priority, no need to continue looking + if isinstance(ext, CodeHiliteExtension) and not config: + config = ext.getConfigs() + config["language_prefix"] = config["lang_prefix"] + self._css_class = config.pop("css_class", "highlight") + super().__init__(**{k: v for k, v in config.items() if k in self._highlight_config_keys}) + + def highlight( # noqa: W0221 (intentionally different params, we're extending the functionality) + self, + src: str, + language: str = None, + *, + inline: bool = False, + dedent: bool = True, + linenums: Optional[bool] = None, + **kwargs, + ) -> str: + """ + Highlight a code-snippet. + + Arguments: + src: The code to highlight. + language: Explicitly tell what language to use for highlighting. + inline: Whether to highlight as inline. + dedent: Whether to dedent the code before highlighting it or not. + linenums: Whether to add line numbers in the result. + **kwargs: Pass on to `pymdownx.highlight.Highlight.highlight`. - This function is used as a filter in Jinja templates. + Returns: + The highlighted code as HTML text, marked safe (not escaped for HTML). + """ + if dedent: + src = textwrap.dedent(src) - Arguments: - src: The code to highlight. - guess_lang: Whether to guess the language or not. - language: Explicitly tell what language to use for highlighting. - inline: Whether to do inline highlighting. - dedent: Whether to dedent the code before highlighting it or not. - line_nums: Whether to add line numbers in the result. - line_start: The line number to start with. + kwargs.setdefault("css_class", self._css_class) + old_linenums = self.linenums + if linenums is not None: + self.linenums = linenums + try: + result = super().highlight(src, language, inline=inline, **kwargs) + finally: + self.linenums = old_linenums - Returns: - The code properly wrapped for later highlighting by JavaScript. - """ - if dedent: - src = textwrap.dedent(src) - if inline: - src = re.sub(r"\n\s*", "", src) - return Markup(f'<code class="highlight">{src}</code>') - return Markup(f'<div class="highlight {language or ""}"><pre><code>\n{src}\n</code></pre></div>') + if inline: + return Markup(f'<code class="highlight language-{language}">{result.text}</code>') + return Markup(result) def do_any(seq: Sequence, attribute: str = None) -> bool: @@ -184,13 +179,6 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = self.env.filters["any"] = do_any self.env.globals["log"] = get_template_logger() - if theme == "readthedocs": - highlight_function = do_js_highlight - else: - highlight_function = do_highlight - - self.env.filters["highlight"] = highlight_function - self._headings = [] self._md = None # To be populated in `update_env`. @@ -317,6 +305,7 @@ def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary. """ self._md = md + self.env.filters["highlight"] = Highlighter(md).highlight self.env.filters["convert_markdown"] = self.do_convert_markdown self.env.filters["heading"] = self.do_heading diff --git a/src/mkdocstrings/templates/python/material/class.html b/src/mkdocstrings/templates/python/material/class.html index 819edfd4..b3e33f84 100644 --- a/src/mkdocstrings/templates/python/material/class.html +++ b/src/mkdocstrings/templates/python/material/class.html @@ -48,7 +48,7 @@ {% if config.show_source and class.source %} <details class="quote"> <summary>Source code in <code>{{ class.relative_file_path }}</code></summary> - {{ class.source.code|highlight(language="python", line_start=class.source.line_start) }} + {{ class.source.code|highlight(language="python", linestart=class.source.line_start, linenums=False) }} </details> {% endif %} diff --git a/src/mkdocstrings/templates/python/material/examples.html b/src/mkdocstrings/templates/python/material/examples.html index edc210b2..63b6f430 100644 --- a/src/mkdocstrings/templates/python/material/examples.html +++ b/src/mkdocstrings/templates/python/material/examples.html @@ -4,6 +4,6 @@ {% if section_type == "markdown" %} {{ sub_section|convert_markdown(heading_level, html_id) }} {% elif section_type == "examples" %} - {{ sub_section|highlight(language="python", line_nums=False) }} + {{ sub_section|highlight(language="python", linenums=False) }} {% endif %} {% endfor %} diff --git a/src/mkdocstrings/templates/python/material/function.html b/src/mkdocstrings/templates/python/material/function.html index a2e0161f..5ac592b9 100644 --- a/src/mkdocstrings/templates/python/material/function.html +++ b/src/mkdocstrings/templates/python/material/function.html @@ -51,7 +51,7 @@ {% if config.show_source and function.source %} <details class="quote"> <summary>Source code in <code>{{ function.relative_file_path }}</code></summary> - {{ function.source.code|highlight(language="python", line_start=function.source.line_start) }} + {{ function.source.code|highlight(language="python", linestart=function.source.line_start, linenums=False) }} </details> {% endif %} </div> diff --git a/src/mkdocstrings/templates/python/material/method.html b/src/mkdocstrings/templates/python/material/method.html index d73c5ab3..19e9a530 100644 --- a/src/mkdocstrings/templates/python/material/method.html +++ b/src/mkdocstrings/templates/python/material/method.html @@ -51,7 +51,7 @@ {% if config.show_source and method.source %} <details class="quote"> <summary>Source code in <code>{{ method.relative_file_path }}</code></summary> - {{ method.source.code|highlight(language="python", line_start=method.source.line_start) }} + {{ method.source.code|highlight(language="python", linestart=method.source.line_start, linenums=False) }} </details> {% endif %} </div> diff --git a/src/mkdocstrings/templates/python/material/signature.html b/src/mkdocstrings/templates/python/material/signature.html index 1e1a8b27..e1f815da 100644 --- a/src/mkdocstrings/templates/python/material/signature.html +++ b/src/mkdocstrings/templates/python/material/signature.html @@ -1,31 +1,31 @@ {{ log.debug() }} -{% if signature %} - {% with %} - {% set ns = namespace(render_pos_only_separator=True, render_kw_only_separator=True, equal="=") %} +{%- if signature -%} + {%- with -%} + {%- set ns = namespace(render_pos_only_separator=True, render_kw_only_separator=True, equal="=") -%} - {% if config.show_signature_annotations %} - {% set ns.equal = " = " %} - {% endif %} + {%- if config.show_signature_annotations -%} + {%- set ns.equal = " = " -%} + {%- endif -%} - ({% for parameter in signature.parameters %}{% if parameter.kind == "POSITIONAL_ONLY" %} - {% if ns.render_pos_only_separator %} - {% set ns.render_pos_only_separator = False %}/, {% endif %} - {% elif parameter.kind == "KEYWORD_ONLY" %} - {% if ns.render_kw_only_separator %} - {% set ns.render_kw_only_separator = False %}*, {% endif %} - {% endif %} - {% if config.show_signature_annotations and "annotation" in parameter %} - {% set annotation = ": " + parameter.annotation|safe %} - {% endif %} - {% if "default" in parameter %} - {% set default = ns.equal + parameter.default|safe %} - {% endif %} - {% if parameter.kind == "VAR_POSITIONAL" %}* - {% set render_kw_only_separator = False %} - {% elif parameter.kind == "VAR_KEYWORD" %}** - {% endif %}{{ parameter.name }}{{ annotation }}{{ default }}{% if not loop.last %}, {% endif %} - {% endfor %}){% if config.show_signature_annotations and "return_annotation" in signature %} -> {{ signature.return_annotation }} - {% endif %} + ({%- for parameter in signature.parameters %}{% if parameter.kind == "POSITIONAL_ONLY" -%} + {%- if ns.render_pos_only_separator -%} + {%- set ns.render_pos_only_separator = False %}/, {% endif -%} + {%- elif parameter.kind == "KEYWORD_ONLY" -%} + {%- if ns.render_kw_only_separator -%} + {%- set ns.render_kw_only_separator = False %}*, {% endif -%} + {%- endif -%} + {%- if config.show_signature_annotations and "annotation" in parameter -%} + {%- set annotation = ": " + parameter.annotation|safe -%} + {%- endif -%} + {%- if "default" in parameter -%} + {%- set default = ns.equal + parameter.default|safe -%} + {%- endif -%} + {%- if parameter.kind == "VAR_POSITIONAL" %}* + {%- set render_kw_only_separator = False -%} + {%- elif parameter.kind == "VAR_KEYWORD" %}** + {%- endif %}{{ parameter.name }}{{ annotation }}{{ default }}{% if not loop.last %}, {% endif -%} + {%- endfor %}){% if config.show_signature_annotations and "return_annotation" in signature %} -> {{ signature.return_annotation }} + {%- endif -%} - {% endwith %} -{% endif %} + {%- endwith -%} +{%- endif -%} diff --git a/tests/test_handlers.py b/tests/test_handlers.py new file mode 100644 index 00000000..0d0c1b40 --- /dev/null +++ b/tests/test_handlers.py @@ -0,0 +1,48 @@ +"""Tests for the handlers.base module.""" + +import pytest +from markdown import Markdown + +from mkdocstrings.handlers.base import Highlighter + + +@pytest.mark.parametrize("extension_name", ["codehilite", "pymdownx.highlight"]) +def test_highlighter_without_pygments(extension_name): + """ + Assert that it's possible to disable Pygments highlighting. + + Arguments: + extension_name: The "user-chosen" Markdown extension for syntax highlighting. + """ + configs = {extension_name: {"use_pygments": False, "css_class": "hiiii"}} + md = Markdown(extensions=configs, extension_configs=configs) + hl = Highlighter(md) + assert ( + hl.highlight("import foo", language="python") + == '<pre class="hiiii"><code class="language-python">import foo</code></pre>' + ) + assert ( + hl.highlight("import foo", language="python", inline=True) + == '<code class="highlight language-python">import foo</code>' + ) + + +@pytest.mark.parametrize("extension_name", [None, "codehilite", "pymdownx.highlight"]) +@pytest.mark.parametrize("inline", [False, True]) +def test_highlighter_basic(extension_name, inline): + """ + Assert that Pygments syntax highlighting works. + + Arguments: + extension_name: The "user-chosen" Markdown extension for syntax highlighting. + inline: Whether the highlighting was inline. + """ + configs = {} + if extension_name: + configs[extension_name] = {} + md = Markdown(extensions=configs, extension_configs=configs) + hl = Highlighter(md) + + actual = hl.highlight("import foo", language="python", inline=inline) + assert "import" in actual + assert "import foo" not in actual # Highlighting has split it up. From 0d5444594bf278ffc96133311437329eda7b3996 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Thu, 4 Feb 2021 22:08:59 +0100 Subject: [PATCH 17/40] docs: Fix up docs after splitting autorefs plugin --- docs/reference/autorefs/plugin.md | 1 + docs/reference/{ => autorefs}/references.md | 0 mkdocs.yml | 15 +++++---- src/mkdocs_autorefs/plugin.py | 34 ++++++++++++++++++--- src/mkdocs_autorefs/references.py | 6 ++-- src/mkdocstrings/extension.py | 4 +-- src/mkdocstrings/plugin.py | 17 +++-------- 7 files changed, 49 insertions(+), 28 deletions(-) create mode 100644 docs/reference/autorefs/plugin.md rename docs/reference/{ => autorefs}/references.md (100%) diff --git a/docs/reference/autorefs/plugin.md b/docs/reference/autorefs/plugin.md new file mode 100644 index 00000000..fed901e4 --- /dev/null +++ b/docs/reference/autorefs/plugin.md @@ -0,0 +1 @@ +::: mkdocs_autorefs.plugin diff --git a/docs/reference/references.md b/docs/reference/autorefs/references.md similarity index 100% rename from docs/reference/references.md rename to docs/reference/autorefs/references.md diff --git a/mkdocs.yml b/mkdocs.yml index 30f627f8..eafe5773 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,12 +12,15 @@ nav: - Python: handlers/python.md - Code Reference: - - handlers: - - base.py: reference/handlers/base.md - - python.py: reference/handlers/python.md - - extension.py: reference/extension.md - - plugin.py: reference/plugin.md - - references.py: reference/references.md + - mkdocstrings: + - handlers: + - base.py: reference/handlers/base.md + - python.py: reference/handlers/python.md + - extension.py: reference/extension.md + - plugin.py: reference/plugin.md + - mkdocs_autorefs: + - references.py: reference/autorefs/references.md + - plugin.py: reference/autorefs/plugin.md - Troubleshooting: troubleshooting.md - Contributing: contributing.md diff --git a/src/mkdocs_autorefs/plugin.py b/src/mkdocs_autorefs/plugin.py index a6b1ab4f..4ee4e51c 100644 --- a/src/mkdocs_autorefs/plugin.py +++ b/src/mkdocs_autorefs/plugin.py @@ -1,3 +1,16 @@ +""" +This module contains the "mkdocs-autorefs" plugin. + +After each page is processed by the Markdown converter, this plugin stores absolute URLs of every HTML anchors +it finds to later be able to fix unresolved references. +It stores them during the [`on_page_content` event hook](https://www.mkdocs.org/user-guide/plugins/#on_page_content). + +Just before writing the final HTML to the disc, during the +[`on_post_page` event hook](https://www.mkdocs.org/user-guide/plugins/#on_post_page), +this plugin searches for references of the form `[identifier][]` or `[title][identifier]` that were not resolved, +and fixes them using the previously stored identifier-URL mapping. +""" + import logging from typing import Callable, Dict, Optional @@ -14,6 +27,19 @@ class AutorefsPlugin(BasePlugin): + """ + An `mkdocs` plugin. + + This plugin defines the following event hooks: + + - `on_config` + - `on_page_content` + - `on_post_page` + + Check the [Developing Plugins](https://www.mkdocs.org/user-guide/plugins/#developing-plugins) page of `mkdocs` + for more information about its plugin system. + """ + scan_toc: bool = True current_page: Optional[str] = None @@ -28,8 +54,8 @@ def register_anchor(self, page: str, anchor: str): Register that an anchor corresponding to an identifier was encountered when rendering the page. Arguments: - page: The URL of the current page. - anchor: The HTML anchor (without '#') as a string + page: The relative URL of the current page. Examples: `'foo/bar/'`, `'foo/index.html'` + anchor: The HTML anchor (without '#') as a string. """ self._url_map[anchor] = f"{page}#{anchor}" @@ -54,7 +80,7 @@ def get_item_url(self, anchor: str) -> str: return self._url_map[new_anchor] raise - def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused arguments) + def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613,R0201 (unused arguments, cannot be static) """ Instantiate our Markdown extension. @@ -92,7 +118,7 @@ def on_page_content(self, html: str, page: Page, **kwargs) -> str: # noqa: W061 """ Map anchors to URLs. - Hook for the [`on_page_contents` event](https://www.mkdocs.org/user-guide/plugins/#on_page_contents). + Hook for the [`on_page_content` event](https://www.mkdocs.org/user-guide/plugins/#on_page_content). In this hook, we map the IDs of every anchor found in the table of contents to the anchors absolute URLs. This mapping will be used later to fix unresolved reference of the form `[title][identifier]` or `[identifier][]`. diff --git a/src/mkdocs_autorefs/references.py b/src/mkdocs_autorefs/references.py index 1d7987bc..53f5205d 100644 --- a/src/mkdocs_autorefs/references.py +++ b/src/mkdocs_autorefs/references.py @@ -12,7 +12,7 @@ AUTO_REF_RE = re.compile(r'<span data-mkdocstrings-identifier=("?)(?P<identifier>[^"<>]*)\1>(?P<title>.*?)</span>') """ A regular expression to match mkdocstrings' special reference markers -in the [`on_post_page` hook][mkdocstrings.plugin.MkdocstringsPlugin.on_post_page]. +in the [`on_post_page` hook][mkdocs_autorefs.plugin.AutorefsPlugin.on_post_page]. """ EvalIDType = Tuple[Any, Any, Any] @@ -132,7 +132,7 @@ def fix_ref(url_mapper: Callable[[str], str], from_url: str, unmapped: List[str] Arguments: url_mapper: A callable that gets an object's site URL by its identifier, - such as [mkdocstrings.handlers.base.Handlers.get_item_url][]. + such as [mkdocs_autorefs.plugin.AutorefsPlugin.get_item_url][]. from_url: The URL of the base page, from which we link towards the targeted pages. unmapped: A list to store unmapped identifiers. @@ -170,7 +170,7 @@ def fix_refs( html: The text to fix. from_url: The URL at which this HTML is served. url_mapper: A callable that gets an object's site URL by its identifier, - such as [mkdocstrings.handlers.base.Handlers.get_item_url][]. + such as [mkdocs_autorefs.plugin.AutorefsPlugin.get_item_url][]. Returns: The fixed HTML. diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index ec40cf03..ecc4f1a4 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -67,7 +67,7 @@ def __init__( config: The [configuration][mkdocstrings.plugin.MkdocstringsPlugin.config_scheme] of the `mkdocstrings` plugin. handlers: A [mkdocstrings.handlers.base.Handlers][] instance. - autorefs: A [mkdocs_aurorefs.plugin.AutorefsPlugin][] instance. + autorefs: A [mkdocs_autorefs.plugin.AutorefsPlugin][] instance. """ super().__init__(parser=parser) self.md = md @@ -232,7 +232,7 @@ def __init__(self, config: dict, handlers: Handlers, autorefs: AutorefsPlugin, * config: The configuration items from `mkdocs` and `mkdocstrings` that must be passed to the block processor when instantiated in [`extendMarkdown`][mkdocstrings.extension.MkdocstringsExtension.extendMarkdown]. handlers: A [mkdocstrings.handlers.base.Handlers][] instance. - autorefs: A [mkdocs_aurorefs.plugin.AutorefsPlugin][] instance. + autorefs: A [mkdocs_autorefs.plugin.AutorefsPlugin][] instance. kwargs: Keyword arguments used by `markdown.extensions.Extension`. """ super().__init__(**kwargs) diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 9400c205..0eff9ed6 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -1,19 +1,10 @@ """ -This module contains the `mkdocs` plugin. +This module contains the "mkdocstrings" plugin for MkDocs. The plugin instantiates a Markdown extension ([`MkdocstringsExtension`][mkdocstrings.extension.MkdocstringsExtension]), and adds it to the list of Markdown extensions used by `mkdocs` during the [`on_config` event hook](https://www.mkdocs.org/user-guide/plugins/#on_config). -After each page is processed by the Markdown converter, this plugin stores absolute URLs of every HTML anchors -it finds to later be able to fix unresolved references. -It stores them during the [`on_page_contents` event hook](https://www.mkdocs.org/user-guide/plugins/#on_page_contents). - -Just before writing the final HTML to the disc, during the -[`on_post_page` event hook](https://www.mkdocs.org/user-guide/plugins/#on_post_page), -this plugin searches for references of the form `[identifier][]` or `[title][identifier]` that were not resolved, -and fixes them using the previously stored identifier-URL mapping. - Once the documentation is built, the [`on_post_build` event hook](https://www.mkdocs.org/user-guide/plugins/#on_post_build) is triggered and calls the [`handlers.teardown()` method][mkdocstrings.handlers.base.Handlers.teardown]. This method is used to teardown the handlers that were instantiated during documentation buildup. @@ -51,13 +42,11 @@ class MkdocstringsPlugin(BasePlugin): This plugin defines the following event hooks: - `on_config` - - `on_page_contents` - - `on_post_page` - `on_post_build` - `on_serve` Check the [Developing Plugins](https://www.mkdocs.org/user-guide/plugins/#developing-plugins) page of `mkdocs` - for more information about its plugin system.. + for more information about its plugin system. """ config_scheme: Tuple[Tuple[str, MkType]] = ( @@ -182,11 +171,13 @@ def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused try: # If autorefs plugin is explicitly enabled, just use it. autorefs = config["plugins"]["autorefs"] + log.debug(f"Picked up existing autorefs instance {autorefs!r}") except KeyError: # Otherwise, add a limited instance of it that acts only on what's added through `register_anchor`. autorefs = AutorefsPlugin() autorefs.scan_toc = False config["plugins"]["autorefs"] = autorefs + log.debug(f"Added a subdued autorefs instance {autorefs!r}") # Add collector-based fallback in either case. autorefs.get_fallback_anchor = self._handlers.get_anchor From fc2da970385f820ec7e291dccbc991160b95526a Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Tue, 9 Feb 2021 22:52:01 +0100 Subject: [PATCH 18/40] docs: Use `pymdownx.snippets` instead of `markdown_include` PR #229: https://github.com/pawamoy/mkdocstrings/pull/229 --- docs/changelog.md | 2 +- docs/code_of_conduct.md | 2 +- docs/contributing.md | 2 +- docs/credits.md | 2 +- docs/index.md | 2 +- docs/license.md | 16 +--------------- mkdocs.yml | 3 ++- pyproject.toml | 1 - 8 files changed, 8 insertions(+), 22 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index dcdfe2a7..786b75d5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1 +1 @@ -{!CHANGELOG.md!} \ No newline at end of file +--8<-- "CHANGELOG.md" diff --git a/docs/code_of_conduct.md b/docs/code_of_conduct.md index b5d5f94b..01f2ea20 100644 --- a/docs/code_of_conduct.md +++ b/docs/code_of_conduct.md @@ -1 +1 @@ -{!CODE_OF_CONDUCT.md!} \ No newline at end of file +--8<-- "CODE_OF_CONDUCT.md" diff --git a/docs/contributing.md b/docs/contributing.md index 0d1453a3..ea38c9bf 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1 +1 @@ -{!CONTRIBUTING.md!} \ No newline at end of file +--8<-- "CONTRIBUTING.md" diff --git a/docs/credits.md b/docs/credits.md index 343d66d3..0ad20ffb 100644 --- a/docs/credits.md +++ b/docs/credits.md @@ -1 +1 @@ -{!CREDITS.md!} \ No newline at end of file +--8<-- "CREDITS.md" diff --git a/docs/index.md b/docs/index.md index 10882f6c..612c7a5e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1 +1 @@ -{!README.md!} \ No newline at end of file +--8<-- "README.md" diff --git a/docs/license.md b/docs/license.md index 02853682..cdacdfef 100644 --- a/docs/license.md +++ b/docs/license.md @@ -1,17 +1,3 @@ ``` -ISC License - -Copyright (c) 2019, Timothée Mazzucotelli - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +--8<-- "LICENSE" ``` diff --git a/mkdocs.yml b/mkdocs.yml index eafe5773..f5557521 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -41,9 +41,10 @@ extra_css: markdown_extensions: - admonition -- markdown_include.include - pymdownx.emoji - pymdownx.magiclink +- pymdownx.snippets: + check_paths: true - pymdownx.superfences - pymdownx.tabbed - pymdownx.tasklist diff --git a/pyproject.toml b/pyproject.toml index ec585772..04baa3bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ httpx = "^0.14.3" ipython = "^7.2" isort = "^5.7.0" jinja2-cli = "^0.7.0" -markdown-include = "^0.6.0" mkdocs-material = "^5.5.12" mypy = "^0.782" pytest = "^6.0.1" From b2dbfdc9fd9bc1a67afdabb81176dd851f2af3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Tue, 9 Feb 2021 23:39:52 +0100 Subject: [PATCH 19/40] docs: Update credits --- CREDITS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CREDITS.md b/CREDITS.md index d6a07fd5..8357b62e 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -32,7 +32,6 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`Jinja2`](https://palletsprojects.com/p/jinja/) | [`jinja2-cli`](https://github.com/mattrobenolt/jinja2-cli) | [`Markdown`](https://Python-Markdown.github.io/) | -[`markdown-include`](https://github.com/cmacmackin/markdown-include/) | [`MarkupSafe`](https://palletsprojects.com/p/markupsafe/) | [`mkdocs`](https://www.mkdocs.org) | [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/) | @@ -107,7 +106,7 @@ These projects were used to build `mkdocstrings`. **Thank you!** [`pylint`](https://github.com/PyCQA/pylint) | [`pyparsing`](https://github.com/pyparsing/pyparsing/) | [`pytest-forked`](https://github.com/pytest-dev/pytest-forked) | -[`PyYAML`](https://github.com/yaml/pyyaml) | +[`PyYAML`](https://pyyaml.org/) | [`regex`](https://bitbucket.org/mrabarnett/mrab-regex) | [`rfc3986`](http://rfc3986.readthedocs.io) | [`six`](https://github.com/benjaminp/six) | From 6baf720850d359ddb55713553a757fe7b2283e10 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 10 Feb 2021 18:31:39 +0100 Subject: [PATCH 20/40] feat: Nicer-looking error outputs - no tracebacks from mkdocstrings (though definitely not nicer-looking in terms of code) --- src/mkdocstrings/extension.py | 16 ++++++++++++---- src/mkdocstrings/handlers/python.py | 9 ++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index ecc4f1a4..135ffc51 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -39,6 +39,12 @@ from mkdocstrings.handlers.base import CollectionError, CollectorItem, Handlers from mkdocstrings.loggers import get_logger +try: + from mkdocs.exceptions import PluginError # New in MkDocs 1.2 +except ImportError: + PluginError = SystemExit + + log = get_logger(__name__) @@ -144,7 +150,7 @@ def _process_block(self, identifier: str, yaml_block: str, heading_level: int = heading_level: Suggested level of the the heading to insert (0 to ignore). Raises: - CollectionError: When something wrong happened during collection. + PluginError: When something wrong happened during collection. TemplateNotFound: When a template used for rendering could not be found. Returns: @@ -164,9 +170,11 @@ def _process_block(self, identifier: str, yaml_block: str, heading_level: int = log.debug("Collecting data") try: data: CollectorItem = handler.collector.collect(identifier, selection) - except CollectionError: - log.error(f"Could not collect '{identifier}'") - raise + except CollectionError as exception: + log.error(str(exception)) + if PluginError is SystemExit: # When MkDocs 1.2 is sufficiently common, this can be dropped. + log.error(f"Error reading page '{self._autorefs.current_page}':") + raise PluginError(f"Could not collect '{identifier}'") from exception if not self._updated_env: log.debug("Updating renderer's env") diff --git a/src/mkdocstrings/handlers/python.py b/src/mkdocstrings/handlers/python.py index 4548d0dd..75b82db1 100644 --- a/src/mkdocstrings/handlers/python.py +++ b/src/mkdocstrings/handlers/python.py @@ -7,6 +7,7 @@ import json import os import sys +import traceback from collections import ChainMap from subprocess import PIPE, Popen # noqa: S404 (what other option, more secure that PIPE do we have? sockets?) from typing import Any, List, Optional @@ -211,15 +212,13 @@ def collect(self, identifier: str, config: dict) -> CollectorItem: try: result = json.loads(stdout) except json.decoder.JSONDecodeError as exception: - log.error(f"Error while loading JSON: {stdout}") - raise CollectionError(str(exception)) from exception + error = "\n".join(("Error while loading JSON:", stdout, traceback.format_exc())) + raise CollectionError(error) from exception error = result.get("error") if error: - message = f"Collection failed: {error}" if "traceback" in result: - message += f"\n{result['traceback']}" - log.error(message) + error += f"\n{result['traceback']}" raise CollectionError(error) for loading_error in result["loading_errors"]: From 40232a94d1a4266716666d91fb9e8ce022aa55c8 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 10 Feb 2021 19:46:48 +0100 Subject: [PATCH 21/40] fix: Restore log prefixes to autorefs plugin --- src/mkdocs_autorefs/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mkdocs_autorefs/plugin.py b/src/mkdocs_autorefs/plugin.py index 4ee4e51c..5ef8a08d 100644 --- a/src/mkdocs_autorefs/plugin.py +++ b/src/mkdocs_autorefs/plugin.py @@ -95,7 +95,7 @@ def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613,R0201 (u Returns: The modified config. """ - log.debug("Adding AutorefsExtension to the list") + log.debug(f"{__name__}: Adding AutorefsExtension to the list") config["markdown_extensions"].append(AutorefsExtension()) return config @@ -132,7 +132,7 @@ def on_page_content(self, html: str, page: Page, **kwargs) -> str: # noqa: W061 The same HTML. We only use this hook to map anchors to URLs. """ if self.scan_toc: - log.debug(f"Mapping identifiers to URLs for page {page.file.src_path}") + log.debug(f"{__name__}: Mapping identifiers to URLs for page {page.file.src_path}") for item in page.toc.items: self.map_urls(page.url, item) return html @@ -173,14 +173,14 @@ def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 Returns: Modified HTML. """ - log.debug(f"Fixing references in page {page.file.src_path}") + log.debug(f"{__name__}: Fixing references in page {page.file.src_path}") fixed_output, unmapped = fix_refs(output, page.url, self.get_item_url) if unmapped and log.isEnabledFor(logging.WARNING): for ref in unmapped: log.warning( - f"{page.file.src_path}: Could not find cross-reference target '[{ref}]'", + f"{__name__}: {page.file.src_path}: Could not find cross-reference target '[{ref}]'", ) return fixed_output From 63d1599d8a04f4f448f12ea4e6a58b41a891ed8b Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 10 Feb 2021 22:26:19 +0100 Subject: [PATCH 22/40] ci: Don't use Poetry debug mode, it's currently broken https://github.com/python-poetry/poetry/issues/3663 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21ccbc9f..49dca28e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Set up the project run: | pip install poetry - poetry install -vvv || { rm -rf .venv; poetry install -vvv; } + poetry install -v || { rm -rf .venv; poetry install -v; } poetry update - name: Check if the documentation builds correctly @@ -85,7 +85,7 @@ jobs: - name: Set up the project run: | pip install poetry - poetry install -vvv || { rm -rf .venv; poetry install -vvv; } + poetry install -v || { rm -rf .venv; poetry install -v; } poetry update - name: Run the test suite From 7533852e3ac0a378b70a380cef1100421b7d5763 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Thu, 11 Feb 2021 20:48:55 +0100 Subject: [PATCH 23/40] refactor: Split out Markdown extensions from handlers to handlers.rendering --- docs/reference/handlers/rendering.md | 1 + mkdocs.yml | 1 + src/mkdocstrings/handlers/base.py | 183 ++-------------------- src/mkdocstrings/handlers/rendering.py | 209 +++++++++++++++++++++++++ 4 files changed, 223 insertions(+), 171 deletions(-) create mode 100644 docs/reference/handlers/rendering.md create mode 100644 src/mkdocstrings/handlers/rendering.py diff --git a/docs/reference/handlers/rendering.md b/docs/reference/handlers/rendering.md new file mode 100644 index 00000000..d434aaa1 --- /dev/null +++ b/docs/reference/handlers/rendering.md @@ -0,0 +1 @@ +::: mkdocstrings.handlers.rendering diff --git a/mkdocs.yml b/mkdocs.yml index f5557521..bfc95d7c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,6 +15,7 @@ nav: - mkdocstrings: - handlers: - base.py: reference/handlers/base.md + - rendering.py: reference/handlers/rendering.md - python.py: reference/handlers/python.md - extension.py: reference/extension.md - plugin.py: reference/plugin.md diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 13867231..6523bbb7 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -9,23 +9,22 @@ - `teardown`, that will teardown all the cached handlers, and then clear the cache. """ -import copy import importlib -import re -import textwrap from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence +from typing import Any, Dict, Iterable, Optional, Sequence from xml.etree.ElementTree import Element, tostring from jinja2 import Environment, FileSystemLoader from markdown import Markdown -from markdown.extensions import Extension -from markdown.extensions.codehilite import CodeHiliteExtension -from markdown.treeprocessors import Treeprocessor from markupsafe import Markup -from pymdownx.highlight import Highlight, HighlightExtension +from mkdocstrings.handlers.rendering import ( + HeadingShiftingTreeprocessor, + Highlighter, + IdPrependingTreeprocessor, + MkdocstringsInnerExtension, +) from mkdocstrings.loggers import get_template_logger CollectorItem = Any @@ -41,71 +40,6 @@ class ThemeNotSupported(Exception): """An exception raised to tell a theme is not supported.""" -class Highlighter(Highlight): - """Code highlighter that tries to match the Markdown configuration.""" - - _highlight_config_keys = frozenset( - "use_pygments guess_lang css_class pygments_style noclasses linenums language_prefix".split(), - ) - - def __init__(self, md: Markdown): - """Configure to match a `markdown.Markdown` instance. - - Arguments: - md: The Markdown instance to read configs from. - """ - config = {} - for ext in md.registeredExtensions: - if isinstance(ext, HighlightExtension) and (ext.enabled or not config): - config = ext.getConfigs() - break # This one takes priority, no need to continue looking - if isinstance(ext, CodeHiliteExtension) and not config: - config = ext.getConfigs() - config["language_prefix"] = config["lang_prefix"] - self._css_class = config.pop("css_class", "highlight") - super().__init__(**{k: v for k, v in config.items() if k in self._highlight_config_keys}) - - def highlight( # noqa: W0221 (intentionally different params, we're extending the functionality) - self, - src: str, - language: str = None, - *, - inline: bool = False, - dedent: bool = True, - linenums: Optional[bool] = None, - **kwargs, - ) -> str: - """ - Highlight a code-snippet. - - Arguments: - src: The code to highlight. - language: Explicitly tell what language to use for highlighting. - inline: Whether to highlight as inline. - dedent: Whether to dedent the code before highlighting it or not. - linenums: Whether to add line numbers in the result. - **kwargs: Pass on to `pymdownx.highlight.Highlight.highlight`. - - Returns: - The highlighted code as HTML text, marked safe (not escaped for HTML). - """ - if dedent: - src = textwrap.dedent(src) - - kwargs.setdefault("css_class", self._css_class) - old_linenums = self.linenums - if linenums is not None: - self.linenums = linenums - try: - result = super().highlight(src, language, inline=inline, **kwargs) - finally: - self.linenums = old_linenums - - if inline: - return Markup(f'<code class="highlight language-{language}">{result.text}</code>') - return Markup(result) - - def do_any(seq: Sequence, attribute: str = None) -> bool: """ Check if at least one of the item in the sequence evaluates to true. @@ -222,13 +156,13 @@ def do_convert_markdown(self, text: str, heading_level: int, html_id: str = "") An HTML string. """ treeprocessors = self._md.treeprocessors - treeprocessors["mkdocstrings_headings"].shift_by = heading_level - treeprocessors["mkdocstrings_ids"].id_prefix = html_id and html_id + "--" + treeprocessors[HeadingShiftingTreeprocessor.name].shift_by = heading_level + treeprocessors[IdPrependingTreeprocessor.name].id_prefix = html_id and html_id + "--" try: return Markup(self._md.convert(text)) finally: - treeprocessors["mkdocstrings_headings"].shift_by = 0 - treeprocessors["mkdocstrings_ids"].id_prefix = "" + treeprocessors[HeadingShiftingTreeprocessor.name].shift_by = 0 + treeprocessors[IdPrependingTreeprocessor.name].id_prefix = "" self._md.reset() def do_heading( @@ -310,7 +244,7 @@ def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused self.env.filters["heading"] = self.do_heading def _update_env(self, md: Markdown, config: dict): - extensions = config["mdx"] + [_MkdocstringsInnerExtension(self._headings)] + extensions = config["mdx"] + [MkdocstringsInnerExtension(self._headings)] new_md = Markdown(extensions=extensions, extension_configs=config["mdx_configs"]) # MkDocs adds its own (required) extension that's not part of the config. Propagate it. @@ -492,96 +426,3 @@ def teardown(self) -> None: for handler in self.seen_handlers: handler.collector.teardown() self._handlers.clear() - - -class _IdPrependingTreeprocessor(Treeprocessor): - def __init__(self, md, id_prefix: str): - super().__init__(md) - self.id_prefix = id_prefix - - def run(self, root: Element): - if not self.id_prefix: - return - for el in root.iter(): - id_attr = el.get("id") - if id_attr: - el.set("id", self.id_prefix + id_attr) - - href_attr = el.get("href") - if href_attr and href_attr.startswith("#"): - el.set("href", "#" + self.id_prefix + href_attr[1:]) - - name_attr = el.get("name") - if name_attr: - el.set("name", self.id_prefix + name_attr) - - if el.tag == "label": - for_attr = el.get("for") - if for_attr: - el.set("for", self.id_prefix + for_attr) - - -class _HeadingShiftingTreeprocessor(Treeprocessor): - regex = re.compile(r"([Hh])([1-6])") - - def __init__(self, md: Markdown, shift_by: int): - super().__init__(md) - self.shift_by = shift_by - - def run(self, root: Element): - if not self.shift_by: - return - for el in root.iter(): - match = self.regex.fullmatch(el.tag) - if match: - level = int(match[2]) + self.shift_by - level = max(1, min(level, 6)) - el.tag = f"{match[1]}{level}" - - -class _HeadingReportingTreeprocessor(Treeprocessor): - regex = re.compile(r"[Hh][1-6]") - - def __init__(self, md: Markdown, headings: List[Element]): - super().__init__(md) - self.headings = headings - - def run(self, root: Element): - for el in root.iter(): - if self.regex.fullmatch(el.tag): - el = copy.copy(el) - # 'toc' extension's first pass (which we require to build heading stubs/ids) also edits the HTML. - # Undo the permalink edit so we can pass this heading to the outer pass of the 'toc' extension. - if len(el) > 0 and el[-1].get("class") == self.md.treeprocessors["toc"].permalink_class: - del el[-1] - self.headings.append(el) - - -class _MkdocstringsInnerExtension(Extension): - def __init__(self, headings: List[Element]): - super().__init__() - self.headings = headings - - def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) - """ - Register the extension. - - Arguments: - md: A `markdown.Markdown` instance. - """ - md.registerExtension(self) - md.treeprocessors.register( - _HeadingShiftingTreeprocessor(md, 0), - "mkdocstrings_headings", - priority=12, - ) - md.treeprocessors.register( - _IdPrependingTreeprocessor(md, ""), - "mkdocstrings_ids", - priority=4, # Right after 'toc' (needed because that extension adds ids to headers). - ) - md.treeprocessors.register( - _HeadingReportingTreeprocessor(md, self.headings), - "mkdocstrings_headings_list", - priority=1, # Close to the end. - ) diff --git a/src/mkdocstrings/handlers/rendering.py b/src/mkdocstrings/handlers/rendering.py new file mode 100644 index 00000000..970b2413 --- /dev/null +++ b/src/mkdocstrings/handlers/rendering.py @@ -0,0 +1,209 @@ +"""This module holds helpers responsible for augmentations to the Markdown sub-documents produced by handlers.""" + +import copy +import re +import textwrap +from typing import List, Optional +from xml.etree.ElementTree import Element + +from markdown import Markdown +from markdown.extensions import Extension +from markdown.extensions.codehilite import CodeHiliteExtension +from markdown.treeprocessors import Treeprocessor +from markupsafe import Markup +from pymdownx.highlight import Highlight, HighlightExtension + + +class Highlighter(Highlight): + """Code highlighter that tries to match the Markdown configuration.""" + + _highlight_config_keys = frozenset( + "use_pygments guess_lang css_class pygments_style noclasses linenums language_prefix".split(), + ) + + def __init__(self, md: Markdown): + """Configure to match a `markdown.Markdown` instance. + + Arguments: + md: The Markdown instance to read configs from. + """ + config = {} + for ext in md.registeredExtensions: + if isinstance(ext, HighlightExtension) and (ext.enabled or not config): + config = ext.getConfigs() + break # This one takes priority, no need to continue looking + if isinstance(ext, CodeHiliteExtension) and not config: + config = ext.getConfigs() + config["language_prefix"] = config["lang_prefix"] + self._css_class = config.pop("css_class", "highlight") + super().__init__(**{k: v for k, v in config.items() if k in self._highlight_config_keys}) + + def highlight( # noqa: W0221 (intentionally different params, we're extending the functionality) + self, + src: str, + language: str = None, + *, + inline: bool = False, + dedent: bool = True, + linenums: Optional[bool] = None, + **kwargs, + ) -> str: + """ + Highlight a code-snippet. + + Arguments: + src: The code to highlight. + language: Explicitly tell what language to use for highlighting. + inline: Whether to highlight as inline. + dedent: Whether to dedent the code before highlighting it or not. + linenums: Whether to add line numbers in the result. + **kwargs: Pass on to `pymdownx.highlight.Highlight.highlight`. + + Returns: + The highlighted code as HTML text, marked safe (not escaped for HTML). + """ + if dedent: + src = textwrap.dedent(src) + + kwargs.setdefault("css_class", self._css_class) + old_linenums = self.linenums + if linenums is not None: + self.linenums = linenums + try: + result = super().highlight(src, language, inline=inline, **kwargs) + finally: + self.linenums = old_linenums + + if inline: + return Markup(f'<code class="highlight language-{language}">{result.text}</code>') + return Markup(result) + + +class IdPrependingTreeprocessor(Treeprocessor): + """Prepend the configured prefix to IDs of all HTML elements.""" + + name = "mkdocstrings_ids" + + id_prefix: str + """The prefix to add to every ID. It is prepended without any separator; specify your own separator if needed.""" + + def __init__(self, md: Markdown, id_prefix: str): + """Initialize the object. + + Arguments: + md: A `markdown.Markdown` instance. + id_prefix: The prefix to add to every ID. It is prepended without any separator. + """ + super().__init__(md) + self.id_prefix = id_prefix + + def run(self, root: Element): # noqa: D102 (ignore missing docstring) + if not self.id_prefix: + return + for el in root.iter(): + id_attr = el.get("id") + if id_attr: + el.set("id", self.id_prefix + id_attr) + + href_attr = el.get("href") + if href_attr and href_attr.startswith("#"): + el.set("href", "#" + self.id_prefix + href_attr[1:]) + + name_attr = el.get("name") + if name_attr: + el.set("name", self.id_prefix + name_attr) + + if el.tag == "label": + for_attr = el.get("for") + if for_attr: + el.set("for", self.id_prefix + for_attr) + + +class HeadingShiftingTreeprocessor(Treeprocessor): + """Shift levels of all Markdown headings according to the configured base level.""" + + name = "mkdocstrings_headings" + regex = re.compile(r"([Hh])([1-6])") + + shift_by: int + """The number of heading "levels" to add to every heading. `<h2>` with `shift_by = 3` becomes `<h5>`.""" + + def __init__(self, md: Markdown, shift_by: int): + """Initialize the object. + + Arguments: + md: A `markdown.Markdown` instance. + shift_by: The number of heading "levels" to add to every heading. + """ + super().__init__(md) + self.shift_by = shift_by + + def run(self, root: Element): # noqa: D102 (ignore missing docstring) + if not self.shift_by: + return + for el in root.iter(): + match = self.regex.fullmatch(el.tag) + if match: + level = int(match[2]) + self.shift_by + level = max(1, min(level, 6)) + el.tag = f"{match[1]}{level}" + + +class _HeadingReportingTreeprocessor(Treeprocessor): + """Records the heading elements encountered in the document.""" + + name = "mkdocstrings_headings_list" + regex = re.compile(r"[Hh][1-6]") + + headings: List[Element] + """The list (the one passed in the initializer) that is used to record the heading elements (by appending to it).""" + + def __init__(self, md: Markdown, headings: List[Element]): + super().__init__(md) + self.headings = headings + + def run(self, root: Element): + for el in root.iter(): + if self.regex.fullmatch(el.tag): + el = copy.copy(el) + # 'toc' extension's first pass (which we require to build heading stubs/ids) also edits the HTML. + # Undo the permalink edit so we can pass this heading to the outer pass of the 'toc' extension. + if len(el) > 0 and el[-1].get("class") == self.md.treeprocessors["toc"].permalink_class: + del el[-1] + self.headings.append(el) + + +class MkdocstringsInnerExtension(Extension): + """Extension that should always be added to Markdown sub-documents that handlers request (and *only* them).""" + + def __init__(self, headings: List[Element]): + """Initialize the object. + + Arguments: + headings: A list that will be populated with all HTML heading elements encountered in the document. + """ + super().__init__() + self.headings = headings + + def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) + """Register the extension. + + Arguments: + md: A `markdown.Markdown` instance. + """ + md.registerExtension(self) + md.treeprocessors.register( + HeadingShiftingTreeprocessor(md, 0), + HeadingShiftingTreeprocessor.name, + priority=12, + ) + md.treeprocessors.register( + IdPrependingTreeprocessor(md, ""), + IdPrependingTreeprocessor.name, + priority=4, # Right after 'toc' (needed because that extension adds ids to headers). + ) + md.treeprocessors.register( + _HeadingReportingTreeprocessor(md, self.headings), + _HeadingReportingTreeprocessor.name, + priority=1, # Close to the end. + ) From 010bd080d0fcfe7330149fcc09ca808414e6c07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Thu, 11 Feb 2021 00:17:00 +0100 Subject: [PATCH 24/40] chore: Template upgrade - Don't autoescape when rendering changelog - Generate credits on-the-fly, when building docs (for this we're using mkdocs-macros-plugins) - Update docs deploy action to point to org pages - Add cleancov argument to test action, to allow combining coverage for all tested Python versions - Reorganize docs navigation - Add coverage HTML report into docs - Update dev-dependencies, remove ipython --- .copier-answers.yml | 2 +- CONTRIBUTING.md | 2 +- Makefile | 2 +- config/coverage.ini | 5 ++- docs/credits.md | 2 +- docs/macros.py | 81 ++++++++++++++++++++++++++++++++++++++ duties.py | 96 ++++++--------------------------------------- mkdocs.yml | 43 ++++++++++++-------- pyproject.toml | 30 +++++++------- 9 files changed, 144 insertions(+), 119 deletions(-) create mode 100644 docs/macros.py diff --git a/.copier-answers.yml b/.copier-answers.yml index 1b85c204..022b9006 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 0.1.10 +_commit: 0.2.1 _src_path: gh:pawamoy/copier-poetry author_email: pawamoy@pm.me author_fullname: "Timoth\xE9e Mazzucotelli" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 897d976f..75609fd4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ Run `make help` to see all the available actions! ## Tasks -This project uses [Duty](https://github.com/pawamoy/duty) to run tasks. +This project uses [duty](https://github.com/pawamoy/duty) to run tasks. A Makefile is also provided. The Makefile will try to run certain tasks on multiple Python versions. If for some reason you don't want to run the task on multiple Python versions, you can do one of the following: diff --git a/Makefile b/Makefile index 56c7b5f0..28074e01 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ args = $(foreach a,$($(subst -,_,$1)_args),$(if $(value $a),$a="$($a)")) check_code_quality_args = files docs_serve_args = host port release_args = version -test_args = match +test_args = cleancov match BASIC_DUTIES = \ changelog \ diff --git a/config/coverage.ini b/config/coverage.ini index 544ae164..0e7d7d51 100644 --- a/config/coverage.ini +++ b/config/coverage.ini @@ -1,12 +1,15 @@ [coverage:paths] source = src/mkdocstrings + src/mkdocs_autorefs */site-packages/mkdocstrings + */site-packages/mkdocs_autorefs [coverage:run] branch = true source = - mkdocstrings + src/mkdocstrings + src/mkdocs_autorefs tests parallel = true diff --git a/docs/credits.md b/docs/credits.md index 0ad20ffb..76c466aa 100644 --- a/docs/credits.md +++ b/docs/credits.md @@ -1 +1 @@ ---8<-- "CREDITS.md" +[[ credits() ]] diff --git a/docs/macros.py b/docs/macros.py new file mode 100644 index 00000000..cf7f0707 --- /dev/null +++ b/docs/macros.py @@ -0,0 +1,81 @@ +"""Macros and filters made available in Markdown pages.""" + +import functools +from itertools import chain +from pathlib import Path + +import httpx +import toml +from jinja2 import StrictUndefined +from jinja2.sandbox import SandboxedEnvironment +from pip._internal.commands.show import search_packages_info # noqa: WPS436 (no other way?) + + +def get_credits_data() -> dict: + """ + Return data used to generate the credits file. + + Returns: + Data required to render the credits template. + """ + project_dir = Path(__file__).parent.parent + metadata = toml.load(project_dir / "pyproject.toml")["tool"]["poetry"] + lock_data = toml.load(project_dir / "poetry.lock") + project_name = metadata["name"] + + poetry_dependencies = chain(metadata["dependencies"].keys(), metadata["dev-dependencies"].keys()) + direct_dependencies = {dep.lower() for dep in poetry_dependencies} + direct_dependencies.remove("python") + indirect_dependencies = {pkg["name"].lower() for pkg in lock_data["package"]} + indirect_dependencies -= direct_dependencies + dependencies = direct_dependencies | indirect_dependencies + + packages = {} + for pkg in search_packages_info(dependencies): + pkg = {_: pkg[_] for _ in ("name", "home-page")} + packages[pkg["name"].lower()] = pkg + + # all packages might not be credited, + # like the ones that are now part of the standard library + # or the ones that are only used on other operating systems, + # and therefore are not installed, + # but it's not that important + + return { + "project_name": project_name, + "direct_dependencies": sorted(direct_dependencies), + "indirect_dependencies": sorted(indirect_dependencies), + "package_info": packages, + } + + +@functools.lru_cache(maxsize=None) +def get_credits(): + """ + Return credits as Markdown. + + Returns: + The credits page Markdown. + """ + jinja_env = SandboxedEnvironment(undefined=StrictUndefined) + commit = "166758a98d5e544aaa94fda698128e00733497f4" + template_url = f"https://raw.githubusercontent.com/pawamoy/jinja-templates/{commit}/credits.md" + template_data = get_credits_data() + template_text = httpx.get(template_url).text + return jinja_env.from_string(template_text).render(**template_data) + + +def define_env(env): + """ + Add macros and filters into the Jinja2 environment. + + This hook is called by `mkdocs-macros-plugin` + when building the documentation. + + Arguments: + env: An object used to add macros and filters to the environment. + """ + + @env.macro # noqa: WPS430 (nested function) + def credits(): # noqa: W0612,W0622,WPS430 (unused, shadows credits) + return get_credits() diff --git a/duties.py b/duties.py index cde546c8..69234105 100644 --- a/duties.py +++ b/duties.py @@ -3,20 +3,15 @@ import os import re import sys -from itertools import chain -from pathlib import Path from shutil import which from typing import List, Optional, Pattern import httpx -import toml from duty import duty from git_changelog.build import Changelog, Version -from jinja2 import StrictUndefined from jinja2.sandbox import SandboxedEnvironment -from pip._internal.commands.show import search_packages_info -PY_SRC_LIST = ("src/mkdocstrings", "src/mkdocs_autorefs", "tests", "duties.py") +PY_SRC_LIST = ("src/mkdocstrings", "src/mkdocs_autorefs", "tests", "duties.py", "docs/macros.py") PY_SRC = " ".join(PY_SRC_LIST) TESTING = os.environ.get("TESTING", "0") in {"1", "true"} CI = os.environ.get("CI", "0") in {"1", "true", "yes", ""} @@ -102,7 +97,7 @@ def update_changelog( template_url: The URL to the Jinja template used to render contents. commit_style: The style of commit messages to parse. """ - env = SandboxedEnvironment(autoescape=True) + env = SandboxedEnvironment(autoescape=False) template = env.from_string(httpx.get(template_url).text) changelog = Changelog(".", style=commit_style) # noqa: W0621 (shadowing changelog) @@ -236,78 +231,7 @@ def clean(ctx): ctx.run("find . -name '*.rej' -delete") -def get_credits_data() -> dict: - """ - Return data used to generate the credits file. - - Returns: - Data required to render the credits template. - """ - project_dir = Path(__file__).parent.parent - metadata = toml.load(project_dir / "pyproject.toml")["tool"]["poetry"] - lock_data = toml.load(project_dir / "poetry.lock") - project_name = metadata["name"] - - poetry_dependencies = chain(metadata["dependencies"].keys(), metadata["dev-dependencies"].keys()) - direct_dependencies = {dep.lower() for dep in poetry_dependencies} - direct_dependencies.remove("python") - indirect_dependencies = {pkg["name"].lower() for pkg in lock_data["package"]} - indirect_dependencies -= direct_dependencies - dependencies = direct_dependencies | indirect_dependencies - - packages = {} - for pkg in search_packages_info(dependencies): - pkg = {_: pkg[_] for _ in ("name", "home-page")} - packages[pkg["name"].lower()] = pkg - - for dependency in dependencies: - if dependency not in packages: - pkg_data = httpx.get(f"https://pypi.python.org/pypi/{dependency}/json").json()["info"] - home_page = pkg_data["home_page"] or pkg_data["project_url"] or pkg_data["package_url"] - pkg_name = pkg_data["name"] - package = {"name": pkg_name, "home-page": home_page} - packages.update({pkg_name.lower(): package}) - - return { - "project_name": project_name, - "direct_dependencies": sorted(direct_dependencies), - "indirect_dependencies": sorted(indirect_dependencies), - "package_info": packages, - } - - @duty -def docs_regen(ctx): - """ - Regenerate some documentation pages. - - Arguments: - ctx: The context instance (passed automatically). - """ - url_prefix = "https://raw.githubusercontent.com/pawamoy/jinja-templates/master/" - regen_list = (("CREDITS.md", get_credits_data, url_prefix + "credits.md"),) - - def regen() -> int: - """ - Regenerate pages listed in global `REGEN` list. - - Returns: - An exit code. - """ - env = SandboxedEnvironment(undefined=StrictUndefined) - for target, get_data, template in regen_list: - print("Regenerating", target) - template_data = get_data() - template_text = httpx.get(template).text - rendered = env.from_string(template_text).render(**template_data) - with open(target, "w") as stream: - stream.write(rendered) - return 0 - - ctx.run(regen, title="Regenerating docfiles", pty=PTY) - - -@duty(pre=[docs_regen]) def docs(ctx): """ Build the documentation locally. @@ -318,7 +242,7 @@ def docs(ctx): ctx.run("mkdocs build", title="Building documentation") -@duty(pre=[docs_regen]) +@duty def docs_serve(ctx, host="127.0.0.1", port=8000): """ Serve the documentation (localhost:8000). @@ -331,7 +255,7 @@ def docs_serve(ctx, host="127.0.0.1", port=8000): ctx.run(f"mkdocs serve -a {host}:{port}", title="Serving documentation", capture=False) -@duty(pre=[docs_regen]) +@duty def docs_deploy(ctx): """ Deploy the documentation on GitHub pages. @@ -339,7 +263,8 @@ def docs_deploy(ctx): Arguments: ctx: The context instance (passed automatically). """ - ctx.run("mkdocs gh-deploy", title="Deploying documentation") + ctx.run("git remote set-url org-pages git@github.com:mkdocstrings/mkdocstrings.github.io", silent=True) + ctx.run("mkdocs gh-deploy --remote-name org-pages", title="Deploying documentation") @duty @@ -377,7 +302,7 @@ def release(ctx, version): ctx.run("git push --tags", title="Pushing tags", pty=False) ctx.run("poetry build", title="Building dist/wheel", pty=PTY) ctx.run("poetry publish", title="Publishing version", pty=PTY) - ctx.run("mkdocs gh-deploy", title="Deploying documentation", pty=PTY) + docs_deploy.run() @duty(silent=True) @@ -392,15 +317,18 @@ def coverage(ctx): ctx.run("coverage html --rcfile=config/coverage.ini") -@duty(pre=[duty(lambda ctx: ctx.run("rm -f .coverage", silent=True))]) -def test(ctx, match=""): +@duty +def test(ctx, cleancov: bool = True, match: str = ""): """ Run the test suite. Arguments: ctx: The context instance (passed automatically). + cleancov: Whether to remove the `.coverage` file before running the tests. match: A pytest expression to filter selected tests. """ + if cleancov: + ctx.run("rm -f .coverage", silent=True) ctx.run( ["pytest", "-c", "config/pytest.ini", "-n", "auto", "-k", match, "tests"], title="Running tests", diff --git a/mkdocs.yml b/mkdocs.yml index f5557521..52e30ab1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,16 +1,21 @@ site_name: "mkdocstrings" site_description: "Automatic documentation from sources, for MkDocs." -site_url: "https://pawamoy.github.io/mkdocstrings" -repo_url: "https://github.com/pawamoy/mkdocstrings" -repo_name: "pawamoy/mkdocstrings" +site_url: "https://mkdocstrings.github.io/" +repo_url: "https://github.com/mkdocstrings/mkdocstrings" +repo_name: "mkdocstrings/mkdocstrings" nav: -- Overview: index.md -- Usage: usage.md -- Handlers: - - Overview: handlers/overview.md - - Python: handlers/python.md - +- Home: + - Overview: index.md + - Changelog: changelog.md + - Credits: credits.md + - License: license.md +- Usage: + - Usage: usage.md + - Handlers: + - Overview: handlers/overview.md + - Python: handlers/python.md + - Troubleshooting: troubleshooting.md - Code Reference: - mkdocstrings: - handlers: @@ -21,16 +26,16 @@ nav: - mkdocs_autorefs: - references.py: reference/autorefs/references.md - plugin.py: reference/autorefs/plugin.md - -- Troubleshooting: troubleshooting.md -- Contributing: contributing.md -- Code of Conduct: code_of_conduct.md -- Changelog: changelog.md -- Credits: credits.md -- License: license.md +- Development: + - Contributing: contributing.md + - Code of Conduct: code_of_conduct.md + - Coverage report: coverage.md theme: name: material + features: + - navigation.tabs + - navigation.expand palette: scheme: slate primary: teal @@ -53,6 +58,12 @@ markdown_extensions: plugins: - search +- coverage: + html_report_dir: build/coverage +- macros: + module_name: docs/macros + j2_variable_start_string: "[[" + j2_variable_end_string: "]]" - mkdocstrings: handlers: python: diff --git a/pyproject.toml b/pyproject.toml index 04baa3bb..491597e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,26 +33,28 @@ pytkdocs = ">=0.2.0, <0.11.0" [tool.poetry.dev-dependencies] autoflake = "^1.4" black = "^20.8b1" -duty = "^0.5.0" -flakehell = "^0.6.0" +duty = "^0.6.0" +flakehell = "^0.9.0" flake8-black = "^0.2.1" flake8-builtins = "^1.5.3" -flake8-tidy-imports = "^4.1.0" -flake8-variables-names = "^0.0.3" +flake8-tidy-imports = "^4.2.1" +flake8-variables-names = "^0.0.4" flake8-pytest-style = "^1.3.0" -git-changelog = "^0.4.0" -httpx = "^0.14.3" -ipython = "^7.2" -isort = "^5.7.0" +git-changelog = "^0.4.2" +httpx = "^0.16.1" + +isort = {version = "^5.7.0", extras = ["pyproject"]} jinja2-cli = "^0.7.0" -mkdocs-material = "^5.5.12" +mkdocs-coverage = "^0.2.1" +mkdocs-macros-plugin = "^0.5.0" +mkdocs-material = "^6.2.7" mypy = "^0.782" -pytest = "^6.0.1" -pytest-cov = "^2.10.1" -pytest-randomly = "^3.4.1" +pytest = "^6.2.2" +pytest-cov = "^2.11.1" +pytest-randomly = "^3.5.0" pytest-sugar = "^0.9.4" -pytest-xdist = "^2.1.0" -toml = "^0.10.1" +pytest-xdist = "^2.2.0" +toml = "^0.10.2" darglint = "^1.5.8" flake8-bandit = "^2.1.2" flake8-bugbear = "^20.11.1" From 60b8b698d070a5aab8296d9a472575c17860edd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Fri, 12 Feb 2021 18:51:13 +0100 Subject: [PATCH 25/40] ci: Ignore tornado CVE (unclear responsibility) --- duties.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/duties.py b/duties.py index 69234105..69e91b5b 100644 --- a/duties.py +++ b/duties.py @@ -179,8 +179,14 @@ def check_dependencies(ctx): else: safety = "safety" nofail = True + + # Ignore tornado/39462 as there is currently no fix + # See https://github.com/tornadoweb/tornado/issues/2981 + ignored_cves = "39462" + ctx.run( - f"poetry export -f requirements.txt --without-hashes | {safety} check --stdin --full-report", + "poetry export -f requirements.txt --without-hashes | " + f"{safety} check --stdin --full-report -i {ignored_cves}", title="Checking dependencies", pty=PTY, nofail=nofail, From db039c8f3b72cc67496f4b80902c3ad7fb2df007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Tue, 16 Feb 2021 21:03:53 +0100 Subject: [PATCH 26/40] ci: Fix check-docs job --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49dca28e..4aa606c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,10 @@ jobs: poetry update - name: Check if the documentation builds correctly - run: poetry run duty check-docs + run: | + mkdir -p build/coverage + touch build/coverage/index.html + poetry run duty check-docs - name: Check the code quality run: poetry run duty check-code-quality From 61e4e29c17e5a23699cad41b6ba96e8f03ee69ce Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 17 Feb 2021 21:09:02 +0100 Subject: [PATCH 27/40] docs: Details about autorefs, reorganize nav, update links & more * Document more about autorefs, warn about the breaking change. * Linkify *Features* on the home page. * Mention Crystal handler (+add CSS for external link). * Split out a "theming" page from "usage" page. * Linkify sections via *mkdocs-section-index* plugin. * Update links after the site has moved into an org. * Indent some examples into an admonition. * Change "`mkdocstrings`" and "MkDocstrings" to "*mkdocstrings*". * Some minor rewordings. * Fix internal links to be MkDocs-native instead of assuming `use_directory_urls`. --- .github/ISSUE_TEMPLATE/question.md | 2 +- CHANGELOG.md | 2 +- README.md | 77 ++++++---- docs/css/mkdocstrings.css | 6 - docs/css/style.css | 17 +++ docs/handlers/overview.md | 48 +++---- docs/handlers/python.md | 34 ++--- docs/theming.md | 130 +++++++++++++++++ docs/troubleshooting.md | 112 ++++++++------- docs/usage.md | 222 +++++++++-------------------- mkdocs.yml | 26 ++-- pyproject.toml | 6 +- 12 files changed, 372 insertions(+), 310 deletions(-) delete mode 100644 docs/css/mkdocstrings.css create mode 100644 docs/css/style.css create mode 100644 docs/theming.md diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index a9c1cf98..c65012f1 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -10,4 +10,4 @@ assignees: '' **Add detailed information, like** - project folder structure (`tree -L 2`) - `mkdocs.yml` configuration file contents -- `mkdocstrings` version: [e.g. 0.10.2] +- *mkdocstrings* version: [e.g. 0.10.2] diff --git a/CHANGELOG.md b/CHANGELOG.md index a960b050..258115b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. <small>[Compare with 0.13.6](https://github.com/pawamoy/mkdocstrings/compare/0.13.6...0.14.0)</small> Special thanks to Oleh [@oprypin](https://github.com/oprypin) Prypin who did an amazing job (this is a euphemism) -at improving MkDocstrings, fixing hard-to-fix bugs with clever solutions, implementing great new features +at improving *mkdocstrings*, fixing hard-to-fix bugs with clever solutions, implementing great new features and refactoring the code for better performance and readability! Thanks Oleh! ### Bug Fixes diff --git a/README.md b/README.md index 3a6ed222..67ea0f29 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # mkdocstrings -[![ci](https://github.com/pawamoy/mkdocstrings/workflows/ci/badge.svg)](https://github.com/pawamoy/mkdocstrings/actions?query=workflow%3Aci) -[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://pawamoy.github.io/mkdocstrings/) +[![ci](https://github.com/mkdocstrings/mkdocstrings/workflows/ci/badge.svg)](https://github.com/mkdocstrings/mkdocstrings/actions?query=workflow%3Aci) +[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://mkdocstrings.github.io/) [![pypi version](https://img.shields.io/pypi/v/mkdocstrings.svg)](https://pypi.org/project/mkdocstrings/) [![conda version](https://img.shields.io/conda/vn/conda-forge/mkdocstrings)](https://anaconda.org/conda-forge/mkdocstrings) [![gitter](https://badges.gitter.im/join%20chat.svg)](https://gitter.im/mkdocstrings/community) -Automatic documentation from sources, for MkDocs. +Automatic documentation from sources, for [MkDocs](https://mkdocs.org/). --- @@ -22,33 +22,50 @@ Automatic documentation from sources, for MkDocs. ## Features -- **Language agnostic:** just like `mkdocs`, `mkdocstrings` is written in Python but is language-agnostic. - It means you can use it for any language, as long as you implement a - [`handler`](https://pawamoy.github.io/mkdocstrings/reference/handlers/__init__/) for it. - Currently, we only have a [Python handler](https://pawamoy.github.io/mkdocstrings/reference/handlers/python/). - Maybe you'd like to contribute another one :wink:? -- **Multiple themes support:** each handler can offer multiple themes. Currently, we offer the +- [**Language-agnostic:**](https://mkdocstrings.github.io/handlers/overview/) + just like *MkDocs*, *mkdocstrings* is written in Python but is language-agnostic. + It means you can use it with any programming language, as long as there is a + [**handler**](https://mkdocstrings.github.io/reference/handlers/base/) for it. + The [Python handler](https://mkdocstrings.github.io/handlers/python/) is built-in. + [Others](https://mkdocstrings.github.io/handlers/overview/) are external. + Maybe you'd like to add another one to the list? :wink: + +- [**Multiple themes support:**](https://mkdocstrings.github.io/theming/) + each handler can offer multiple themes. Currently, we offer the :star: [Material theme](https://squidfunk.github.io/mkdocs-material/) :star: as well as basic support for the ReadTheDocs theme for the Python handler. -- **Cross-references to other objects:** `mkdocstrings` makes it possible to reference other headings from your - Markdown files with the classic Markdown syntax: `[identifier][]` or `[title][identifier]`. This feature is language - agnostic as well: you can cross-reference any heading that appear in your Markdown pages. - If the handler for a particular language renders headings for documented objects, you'll be able to reference them! -- **Inline injection in Markdown:** instead of generating Markdown files, `mkdocstrings` allows you to inject + +- [**Cross-links across pages:**](https://mkdocstrings.github.io/usage/#cross-references) + *mkdocstrings* makes it possible to reference headings in other Markdown files with the classic Markdown linking + syntax: `[identifier][]` or `[title][identifier]` -- and you don't need to remember which exact page this object was + on. This works for any heading that's produced by a *mkdocstrings* language handler, and you can opt to include + *any* Markdown heading into the global referencing scheme. + + **Note**: in versions prior to 0.15 *all* Markdown headers were included, but now you need to + [opt in](https://mkdocstrings.github.io/usage/#cross-references). + +- [**Inline injection in Markdown:**](https://mkdocstrings.github.io/usage/) + instead of generating Markdown files, *mkdocstrings* allows you to inject documentation anywhere in your Markdown contents. The syntax is simple: `::: identifier` followed by a 4-spaces indented YAML block. The identifier and YAML configuration will be passed to the appropriate handler to collect and render documentation. -- **Global and local configuration:** each handler can be configured globally in `mkdocs.yml`, and locally for each + +- [**Global and local configuration:**](https://mkdocstrings.github.io/usage/#global-options) + each handler can be configured globally in `mkdocs.yml`, and locally for each "autodoc" instruction. -- **Watch source code directories:** you can tell `mkdocstrings` to add directories to be watched by `mkdocs` when + +- [**Watch source code directories:**](https://mkdocstrings.github.io/usage/#watch-directories) + you can tell *mkdocstrings* to add directories to be watched by *MkDocs* when serving the documentation, for auto-reload. -- **Sane defaults:** you should be able to just drop the plugin in your configuration and enjoy your auto-generated docs. + +- **Reasonable defaults:** + you should be able to just drop the plugin in your configuration and enjoy your auto-generated docs. ### Python handler features - **Data collection from source code**: collection of the object-tree and the docstrings is done by [`pytkdocs`](https://github.com/pawamoy/pytkdocs). The following features are possible thanks to it: - - **Support for type annotations:** `pytkdocs` collects your type annotations and `mkdocstrings` uses them + - **Support for type annotations:** `pytkdocs` collects your type annotations and *mkdocstrings* uses them to display parameters types or return types. - **Recursive documentation of Python objects:** just use the module dotted-path as identifier, and you get the full module docs. You don't need to inject documentation for each class, function, etc. @@ -56,37 +73,37 @@ Automatic documentation from sources, for MkDocs. be recognized by `pytkdocs` in modules, classes and even in `__init__` methods. - **Support for objects properties:** `pytkdocs` detects if a method is a `staticmethod`, a `classmethod`, etc., it also detects if a property is read-only or writable, and more! These properties will be displayed - next to the object signature by `mkdocstrings`. + next to the object signature by *mkdocstrings*. - **Google-style sections support in docstrings:** `pytkdocs` understands `Arguments:`, `Raises:` - and `Returns:` sections, and returns structured data for `mkdocstrings` to render them. + and `Returns:` sections, and returns structured data for *mkdocstrings* to render them. - **reStructuredText-style sections support in docstrings:** `pytkdocs` understands all the [reStructuredText fields](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html?highlight=python%20domain#info-field-lists), - and returns structured data for `mkdocstrings` to render them. + and returns structured data for *mkdocstrings* to render them. *Note: only RST **style** is supported, not the whole markup.* - **Admonition support in docstrings:** blocks like `Note: ` or `Warning: ` will be transformed to their [admonition](https://squidfunk.github.io/mkdocs-material/extensions/admonition/) equivalent. *We do not support nested admonitions in docstrings!* - **Support for reStructuredText in docstrings:** `pytkdocs` can parse simple RST. -- **Every object has a TOC entry:** we render a heading for each object, meaning `mkdocs` picks them into the Table - of Contents, which is nicely display by the Material theme. Thanks to `mkdocstrings` cross-reference ability, +- **Every object has a TOC entry:** we render a heading for each object, meaning *MkDocs* picks them into the Table + of Contents, which is nicely display by the Material theme. Thanks to *mkdocstrings* cross-reference ability, you can even reference other objects within your docstrings, with the classic Markdown syntax: `[this object][package.module.object]` or directly with `[package.module.object][]` -- **Source code display:** `mkdocstrings` can add a collapsible div containing the highlighted source code +- **Source code display:** *mkdocstrings* can add a collapsible div containing the highlighted source code of the Python object. -To get an example of what is possible, check `mkdocstrings`' -own [documentation](https://pawamoy.github.io/mkdocstrings), auto-generated from sources by itself of course, +To get an example of what is possible, check *mkdocstrings*' +own [documentation](https://mkdocstrings.github.io/), auto-generated from sources by itself of course, and the following GIF: ![mkdocstrings_gif2](https://user-images.githubusercontent.com/3999221/77157838-7184db80-6aa2-11ea-9f9a-fe77405202de.gif) ## Roadmap -See the [Feature Roadmap issue](https://github.com/pawamoy/mkdocstrings/issues/183) on the bugtracker. +See the [Feature Roadmap issue](https://github.com/mkdocstrings/mkdocstrings/issues/183) on the bugtracker. ## Requirements -`mkdocstrings` requires Python 3.6 or above. +*mkdocstrings* requires Python 3.6 or above. <details> <summary>To install Python 3.6, I recommend using <a href="https://github.com/pyenv/pyenv"><code>pyenv</code></a>.</summary> @@ -141,10 +158,10 @@ plugins: In one of your markdown files: -```yaml +```markdown # Reference ::: my_library.my_module.my_class ``` -See the [Usage](https://pawamoy.github.io/mkdocstrings/usage) section of the docs for more examples! +See the [Usage](https://mkdocstrings.github.io/usage) section of the docs for more examples! diff --git a/docs/css/mkdocstrings.css b/docs/css/mkdocstrings.css deleted file mode 100644 index 42c77416..00000000 --- a/docs/css/mkdocstrings.css +++ /dev/null @@ -1,6 +0,0 @@ -/* Indentation. */ -div.doc-contents:not(.first) { - padding-left: 25px; - border-left: 4px solid rgba(230, 230, 230); - margin-bottom: 80px; -} diff --git a/docs/css/style.css b/docs/css/style.css new file mode 100644 index 00000000..27265bb0 --- /dev/null +++ b/docs/css/style.css @@ -0,0 +1,17 @@ +/* Indentation for mkdocstrings items. */ +div.doc-contents:not(.first) { + padding-left: 25px; + border-left: 4px solid rgba(230, 230, 230); + margin-bottom: 80px; +} + +/* Mark external links as such (also in nav) */ +a.external:hover::after, a.md-nav__link[href^="https:"]:hover::after { + /* https://primer.style/octicons/link-external-16 */ + background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill="rgb(233, 235, 252)" d="M10.604 1h4.146a.25.25 0 01.25.25v4.146a.25.25 0 01-.427.177L13.03 4.03 9.28 7.78a.75.75 0 01-1.06-1.06l3.75-3.75-1.543-1.543A.25.25 0 0110.604 1zM3.75 2A1.75 1.75 0 002 3.75v8.5c0 .966.784 1.75 1.75 1.75h8.5A1.75 1.75 0 0014 12.25v-3.5a.75.75 0 00-1.5 0v3.5a.25.25 0 01-.25.25h-8.5a.25.25 0 01-.25-.25v-8.5a.25.25 0 01.25-.25h3.5a.75.75 0 000-1.5h-3.5z"></path></svg>'); + height: 0.8em; + width: 0.8em; + margin-left: 0.2em; + content: ' '; + display: inline-block; +} diff --git a/docs/handlers/overview.md b/docs/handlers/overview.md index 06fb73b8..351a356e 100644 --- a/docs/handlers/overview.md +++ b/docs/handlers/overview.md @@ -4,7 +4,8 @@ A handler is what makes it possible to collect and render documentation for a pa ## Available handlers -- [Python](../python) +- [Python](python.md) +- <a class="external" href="https://mkdocstrings.github.io/crystal/">Crystal</a> ## Custom handlers @@ -14,23 +15,13 @@ thanks to namespace packages. For more information about namespace packages, ### Packaging -For MkDocstrings, a custom handler package would have the following structure: +For *mkdocstrings*, a custom handler package would have the following structure: ``` 📁 your_repository -└── 📁 mkdocstrings -    └── 📁 handlers - └── 📄 custom_handler.py -``` - -Or with a `src` layout: - -``` -📁 your_repository -└── 📁 src - └── 📁 mkdocstrings -    └── 📁 handlers - └── 📄 custom_handler.py +└─╴📁 mkdocstrings +   └─╴📁 handlers + └─╴📄 custom_handler.py ``` **Note the absence of `__init__.py` modules!** @@ -89,22 +80,21 @@ your renderer. ### Usage -When a custom handler is installed, -it is then available to MkDocstrings. +When a custom handler is installed, it is then available to *mkdocstrings*. You can configure it as usual: -```yaml -# mkdocs.yml -plugins: -- mkdocstrings: - handlers: - custom_handler: - selection: - some_config_option: "a" - rendering: - other_config_option: 0 - handler_config_option: yes -``` +!!! example "mkdocs.yml" + ```yaml + plugins: + - mkdocstrings: + handlers: + custom_handler: + selection: + some_config_option: "a" + rendering: + other_config_option: 0 + handler_config_option: yes + ``` ...and use it in your autodoc instructions: diff --git a/docs/handlers/python.md b/docs/handlers/python.md index 049ec329..4ad1077b 100644 --- a/docs/handlers/python.md +++ b/docs/handlers/python.md @@ -116,7 +116,7 @@ in [Napoleon's documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/l ##### Sections -Docstrings sections are parsed by `pytkdocs` and rendered by MkDocstrings. +Docstrings sections are parsed by `pytkdocs` and rendered by *mkdocstrings*. Supported sections are: - `Arguments` (or `Args`, `Parameters`, `Params`) @@ -217,7 +217,7 @@ in [Sphinx's documentation](https://sphinx-rtd-tutorial.readthedocs.io/en/latest ##### Sections -Docstrings directives are parsed by `pytkdocs` and rendered by MkDocstrings. +Docstrings directives are parsed by `pytkdocs` and rendered by *mkdocstrings*. Supported directives are: - `param` (or `parameter`, `arg`, `argument`, `key`, `keyword`) @@ -301,21 +301,21 @@ You may want to to generate documentation for a package while its dependencies a The Python handler provides itself no builtin way to mock libraries, but you can use the `setup_commands` to mock them manually: -```yaml -# mkdocs.yml -plugins: - - mkdocstrings: - handlers: - python: - setup_commands: - - import sys - - from unittest.mock import MagicMock as mock - - sys.modules["lib1"] = mock() - - sys.modules["lib2"] = mock() - - sys.modules["lib2.module1"] = mock() - - sys.modules["lib2.module1.moduleB"] = mock() - # etc -``` +!!! example "mkdocs.yml" + ```yaml + plugins: + - mkdocstrings: + handlers: + python: + setup_commands: + - import sys + - from unittest.mock import MagicMock as mock + - sys.modules["lib1"] = mock() + - sys.modules["lib2"] = mock() + - sys.modules["lib2.module1"] = mock() + - sys.modules["lib2.module1.moduleB"] = mock() + # etc + ``` ## Recommended style (Material) diff --git a/docs/theming.md b/docs/theming.md new file mode 100644 index 00000000..39159713 --- /dev/null +++ b/docs/theming.md @@ -0,0 +1,130 @@ +# Themes + +*mkdocstrings* can support multiple MkDocs themes. +It currently supports supports the +*[Material for MkDocs](https://squidfunk.github.io/mkdocs-material/)* +theme and, partially, the built-in ReadTheDocs theme. + +Each renderer can fallback to a particular theme when the user selected theme is not supported. +For example, the Python renderer will fallback to the *Material for MkDocs* templates. + +## Customization + +There is some degree of customization possible in *mkdocstrings*. +First, you can write custom templates to override the theme templates. +Second, the provided templates make use of CSS classes, +so you can tweak the look and feel with extra CSS rules. + +### Templates + +To use custom templates and override the theme ones, +specify the relative path to your templates directory +with the `custom_templates` global configuration option: + +!!! example "mkdocs.yml" + ```yaml + plugins: + - mkdocstrings: + custom_templates: templates + ``` + +You directory structure must be identical to the provided templates one: + +``` +templates +├─╴<HANDLER 1> +│ ├── <THEME 1> +│ └── <THEME 2> +└── <HANDLER 2> + ├── <THEME 1> + └── <THEME 2> +``` + +(*[Check out the template tree on GitHub](https://github.com/mkdocstrings/mkdocstrings/tree/master/src/mkdocstrings/templates/)*) + +You don't have to replicate the whole tree, +only the handlers, themes or templates you want to override. +For example, to override some templates of the *Material* theme for Python: + +``` +templates +└── python + └── material + ├── parameters.html + └── exceptions.html +``` + +In the HTML files, replace the original contents with your modified version. +In the future, the templates will use Jinja blocks, so it will be easier +to modify a small part of the template without copy-pasting the whole file. + +The *Material* theme provides the following template structure: + +- `children.html`: where the recursion happen, to render all children of an object + - `attribute.html`: to render attributes (class-attributes, etc.) + - `class.html`: to render classes + - `function.html`: to render functions + - `method.html`: to render methods + - `module.html`: to render modules +- `docstring.html`: to render docstrings + - `attributes.html`: to render attributes sections of docstrings + - `examples.html`: to render examples sections of docstrings + - `exceptions.html`: to render exceptions/"raises" sections of docstrings + - `parameters.html`: to render parameters/arguments sections of docstrings + - `return.html`: to render "return" sections of docstrings +- `properties.html`: to render the properties of an object (`staticmethod`, `read-only`, etc.) +- `signature.html`: to render functions and methods signatures + +#### Debugging + +Every template has access to a `log` function, allowing to log messages as usual: + +```jinja +{{ log.debug("A DEBUG message.") }} +{{ log.info("An INFO message.") }} +{{ log.warning("A WARNING message.") }} +{{ log.error("An ERROR message.") }} +{{ log.critical("A CRITICAL message.") }} +``` + +### CSS classes + +The *Material* theme uses the following CSS classes in the HTML: + +- `doc`: on all the following elements +- `doc-children`: on `div`s containing the children of an object +- `doc-object`: on `div`s containing an object + - `doc-attribute`: on `div`s containing an attribute + - `doc-class`: on `div`s containing a class + - `doc-function`: on `div`s containing a function + - `doc-method`: on `div`s containing a method + - `doc-module`: on `div`s containing a module +- `doc-heading`: on objects headings +- `doc-contents`: on `div`s wrapping the docstring then the children (if any) + - `first`: same, but only on the root object's contents `div` +- `doc-properties`: on `span`s wrapping the object's properties + - `doc-property`: on `small` elements containing a property + - `doc-property-PROPERTY`: same, where `PROPERTY` is replaced by the actual property + +!!! example "Example with colorful properties" + === "CSS" + ```css + .doc-property { border-radius: 15px; padding: 0 5px; } + .doc-property-special { background-color: blue; color: white; } + .doc-property-private { background-color: red; color: white; } + .doc-property-property { background-color: green; color: white; } + .doc-property-read-only { background-color: yellow; color: black; } + ``` + + === "Result" + <style> + .prop { border-radius: 15px; padding: 0 5px; } + </style> + <h3 style="margin: 0;"><span> + <small class="prop" style="background-color: blue; color: white !important;">special</small> + <small class="prop" style="background-color: red; color: white !important;">private</small> + <small class="prop" style="background-color: green; color: white !important;">property</small> + <small class="prop" style="background-color: yellow; color: black !important;">read-only</small> + </span></h3> + + As you can see, CSS is not my field of predilection... diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 97de5c8b..25ed85dd 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -29,7 +29,7 @@ Please upgrade to version 0.14 or higher. See also: -- [Issue #186](https://github.com/pawamoy/mkdocstrings/issues/186) +- [Issue #186](https://github.com/mkdocstrings/mkdocstrings/issues/186) - [Tabs in docstrings (from `pymdownx.tabbed`) are not working properly](#tabs-in-docstrings-from-pymdownxtabbed-are-not-working-properly). ## MkDocs warns me about links to unfound documentation files @@ -46,80 +46,72 @@ It shows that it's probably a cross-reference, not a direct link. It's probably written like `[Section](pytkdocs.parsers.docstrings.Section)` in the docs, when it should be `[Section][pytkdocs.parsers.docstrings.Section]`. -## Nothing is rendered at all - -Python? - -- "No": we only support Python right now. -- "Yes": is your package available in the Python path? - See [Python handler: Finding module](../handlers/python/#finding-modules). - ## Some objects are not rendered (they do not appear in the generated docs) - Make sure the configuration options of the handler for both selection and rendering are correct. - Check the documentation for [Handlers](../handlers/overview) to see the available options for each handler. + Check the documentation for [Handlers](handlers/overview.md) to see the available options for each handler. - Also make sure your documentation in your source code is formatted correctly. - For Python code, check the [supported docstring styles](../handlers/python/#supported-docstrings-styles) page. + For Python code, check the [supported docstring styles](handlers/python.md#supported-docstrings-styles) page. - Re-run the Mkdocs command with `-v`, and carefully read any traceback. ## Tabs in docstrings (from `pymdownx.tabbed`) are not working properly -Before version 0.14, multiple tabs blocks injected on the same page +Before version 0.14, multiple tab blocks injected on the same page would result in broken links: clicking on a tab would bring the user to the wrong one. Please upgrade to version 0.14 or higher. See also: -- [Issue #193](https://github.com/pawamoy/mkdocstrings/issues/193) +- [Issue #193](https://github.com/mkdocstrings/mkdocstrings/issues/193) - [Footnotes are duplicated or overridden](#footnotes-are-duplicated-or-overridden). -**JavaScript workaround:** - -If you are stuck on version lower than 0.14, -and want to use multiple tabs blocks in one page, +If you are stuck on a version before 0.14, +and want to use multiple tab blocks in one page, use this workaround. -Put the following code in a .js file, -and list it in MkDocs' `extra_javascript`: - -```javascript -// Credits to Nikolaos Zioulis (@zuru on GitHub) -function setID(){ - var tabs = document.getElementsByClassName("tabbed-set"); - for (var i = 0; i < tabs.length; i++) { - children = tabs[i].children; - var counter = 0; - var iscontent = 0; - for(var j = 0; j < children.length;j++){ - if(typeof children[j].htmlFor === 'undefined'){ - if((iscontent + 1) % 2 == 0){ - // check if it is content - if(iscontent == 1){ - btn = children[j].childNodes[1].getElementsByTagName("button"); +??? example "JavaScript workaround" + + Put the following code in a .js file, + and list it in MkDocs' `extra_javascript`: + + ```javascript + // Credits to Nikolaos Zioulis (@zuru on GitHub) + function setID(){ + var tabs = document.getElementsByClassName("tabbed-set"); + for (var i = 0; i < tabs.length; i++) { + children = tabs[i].children; + var counter = 0; + var iscontent = 0; + for(var j = 0; j < children.length;j++){ + if(typeof children[j].htmlFor === 'undefined'){ + if((iscontent + 1) % 2 == 0){ + // check if it is content + if(iscontent == 1){ + btn = children[j].childNodes[1].getElementsByTagName("button"); + } } + else{ + // if not change the id + children[j].id = "__tabbed_" + String(i + 1) + "_" + String(counter + 1); + children[j].name = "__tabbed_" + String(i + 1); + // make default tab open + if(j == 0) + children[j].click(); + } + iscontent++; } else{ - // if not change the id - children[j].id = "__tabbed_" + String(i + 1) + "_" + String(counter + 1); - children[j].name = "__tabbed_" + String(i + 1); - // make default tab open - if(j == 0) - children[j].click(); + // link to the correct tab + children[j].htmlFor = "__tabbed_" + String(i+1) + "_" + String(counter + 1); + counter ++; } - iscontent++; - } - else{ - // link to the correct tab - children[j].htmlFor = "__tabbed_" + String(i+1) + "_" + String(counter + 1); - counter ++; } } } -} -setID(); -``` + setID(); + ``` -This code will correctly reset the IDs for tabs on a same page. + This code will correctly reset the IDs for tabs on a same page. ## The generated documentation does not look good @@ -130,12 +122,16 @@ Are you using the Material theme? asking to support your theme. If you find one, vote with a thumbs up. If not, you can open a ticket. - "Yes": Please open an ticket on the [bugtracker][bugtracker] with a detailed explanation and screenshots of the bad-looking parts. - + +Note that you can always [customize the look](theming.md) of *mkdocstrings* blocks -- through both HTML and CSS. + ## Warning: could not find cross-reference target -- Make sure you have defined `site_url` in `mkdocs.yml`, as it is required for cross-references when building the site - (the error does not happen when serving because then `site_url` is auto-populated by `mkdocs`). -- Make sure the referenced object was both collected and rendered: verify your selection and rendering options. +!!! important "New in version 0.15" + Cross-linking used to include any Markdown heading, but now it's only for *mkdocstrings* identifiers by default. + See [Cross-references to any Markdown heading](usage.md#cross-references-to-any-markdown-heading) to opt back in. + +Make sure the referenced object was both collected and rendered: verify your selection and rendering options. For false-positives, you can wrap the text in backticks (\`) to prevent `mkdocstrings` from trying to process it. @@ -166,6 +162,12 @@ Version 2.11.1 seems to be working fine. ## Python specifics +### Nothing is rendered at all + +Is your package available in the Python path? + +See [Python handler: Finding modules](handlers/python.md#finding-modules). + ### LaTeX in docstrings is not rendered correctly If you are using a Markdown extension like @@ -260,7 +262,7 @@ def my_function(*args, **kwargs): print(*args, **kwargs) ``` -[bugtracker]: https://github.com/pawamoy/mkdocstrings +[bugtracker]: https://github.com/mkdocstrings/mkdocstrings [pytkdocs]: https://github.com/pawamoy/pytkdocs [inspect]: https://docs.python.org/3/library/inspect.html [ast]: https://docs.python.org/3/library/ast.html diff --git a/docs/usage.md b/docs/usage.md index ef634df9..ffdefc50 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,14 +2,14 @@ ## Autodoc syntax -MkDocstrings works by processing special expressions in your Markdown files. +*mkdocstrings* works by processing special expressions in your Markdown files. The syntax is as follows: ```md ::: identifier YAML block -``` +``` The `identifier` is a string identifying the object you want to document. The format of an identifier can vary from one handler to another. @@ -30,13 +30,13 @@ The YAML block is optional, and contains some configuration options: Every handler accepts at least these two keys, `selection` and `rendering`, and some handlers accept additional keys. -Check the documentation for your handler of interest in [Handlers](../handlers/overview). +Check the documentation for your handler of interest in [Handlers](handlers/overview.md). !!! example "Example with the Python handler" === "docs/my_page.md" ```md # Documentation for `MyClass` - + ::: my_package.my_module.MyClass handler: python selection: @@ -47,31 +47,31 @@ Check the documentation for your handler of interest in [Handlers](../handlers/o show_root_heading: false show_source: false ``` - + === "mkdocs.yml" ```yaml nav: - "My page": my_page.md ``` - + === "src/my_package/my_module.py" ```python class MyClass: """Print print print!""" - + def method_a(self): """Print A!""" print("A!") - + def method_b(self): """Print B!""" print("B!") - + def method_c(self): """Print C!""" print("C!") ``` - + === "Result" <h3 id="documentation-for-myclass" style="margin: 0;">Documentation for <code>MyClass</code></h3> <div><div><p>Print print print!</p><div><div> @@ -101,34 +101,33 @@ The above is equivalent to: heading_level: 2 ``` - - ## Global options -MkDocstrings accept a few top-level configuration options in `mkdocs.yml`: +*mkdocstrings* accepts a few top-level configuration options in `mkdocs.yml`: - `watch`: a list of directories to watch while serving the documentation. See [Watch directories](#watch-directories). - `default_handler`: the handler that is used by default when no handler is specified. - `custom_templates`: the path to a directory containing custom templates. The path is relative to the docs directory. - See [Customization](#customization). + See [Theming](theming.md). - `handlers`: the handlers global configuration. Example: -```yaml -plugins: -- mkdocstrings: - default_handler: python - handlers: - python: - rendering: - show_source: false - custom_templates: templates - watch: - - src/my_package -``` +!!! example "mkdocs.yml" + ```yaml + plugins: + - mkdocstrings: + default_handler: python + handlers: + python: + rendering: + show_source: false + custom_templates: templates + watch: + - src/my_package + ``` The handlers global configuration can then be overridden by local configurations: @@ -146,11 +145,11 @@ Cross-references are written as Markdown *reference-style* links: ```md With a custom title: [`Object 1`][full.path.object1] - + With the identifier as title: [full.path.object2][] ``` - + === "HTML Result" ```html <p>With a custom title: @@ -169,149 +168,62 @@ Web browser will show in the URL bar when clicking an item's entry in the table If the URL is `https://example.com/some/page.html#full.path.object1` then you know that this item is possible to link to with `[example][full.path.object1]`, regardless of the current page. -## Themes - -MkDocstrings can support multiple MkDocs themes. -It currently supports supports the -*[Material for MkDocs](https://squidfunk.github.io/mkdocs-material/)* -theme and, partially, the built-in ReadTheDocs theme. - -Each renderer can fallback to a particular theme when the user selected theme is not supported. -For example, the Python renderer will fallback to the *Material for MkDocs* templates. - -## Customization +### Cross-references to any Markdown heading -There is some degree of customization possible in MkDocstrings. -First, you can write custom templates to override the theme templates. -Second, the provided templates make use of CSS classes, -so you can tweak the look and feel with extra CSS rules. +If you want to link to *any* Markdown heading, not just *mkdocstrings*-inserted items, please +enable the [*autorefs* plugin for *MkDocs*](https://github.com/mkdocstrings/autorefs) by adding +`autorefs` to `plugins`: -### Templates +!!! example "mkdocs.yml" + ```yaml hl_lines="4" + plugins: + - admonition + - search + - autorefs + - mkdocstrings: + [...] + ``` -To use custom templates and override the theme ones, -specify the relative path to your templates directory -with the `custom_templates` global configuration option: +Prior to *mkdocstrings* version 0.15 this was the default, but now opt-in is required. -```yaml -# mkdocs.yml -plugins: - - mkdocstrings: - custom_templates: templates -``` +Note that you don't need to (`pip`) install anything more; this plugin is guaranteed to be pulled in with *mkdocstrings*. -You directory structure must be identical to the provided templates one: - -``` -templates -├── <HANDLER 1> -│ ├── <THEME 1> -│ └── <THEME 2> -└── <HANDLER 2> - ├── <THEME 1> - └── <THEME 2> -``` -(*[Check out the template tree on GitHub](https://github.com/pawamoy/mkdocstrings/tree/master/src/mkdocstrings/templates/)*) +!!! example + === "doc1.md" + ```md + ## Hello, world! -You don't have to replicate the whole tree, -only the handlers, themes or templates you want to override. -For example, to override some templates of the *Material* theme for Python: + Testing + ``` -``` -templates -└── python - └── material - ├── parameters.html - └── exceptions.html -``` + === "doc2.md" + ```md + ## Something else -In the HTML files, replace the original contents with your modified version. -In the future, the templates will use Jinja blocks, so it will be easier -to modify a small part of the template without copy-pasting the whole file. - -The *Material* theme provides the following template structure: - -- `children.html`: where the recursion happen, to render all children of an object - - `attribute.html`: to render attributes (class-attributes, etc.) - - `class.html`: to render classes - - `function.html`: to render functions - - `method.html`: to render methods - - `module.html`: to render modules -- `docstring.html`: to render docstrings - - `attributes.html`: to render attributes sections of docstrings - - `examples.html`: to render examples sections of docstrings - - `exceptions.html`: to render exceptions/"raises" sections of docstrings - - `parameters.html`: to render parameters/arguments sections of docstrings - - `return.html`: to render "return" sections of docstrings -- `properties.html`: to render the properties of an object (`staticmethod`, `read-only`, etc.) -- `signature.html`: to render functions and methods signatures - -#### Debugging - -Every template has access to a `log` function, allowing to log messages as usual: - -```jinja -{{ log.debug("A DEBUG message.") }} -{{ log.info("An INFO message.") }} -{{ log.warning("A WARNING message.") }} -{{ log.error("An ERROR message.") }} -{{ log.critical("A CRITICAL message.") }} -``` + Please see the [Hello, World!][hello-world] section. + ``` -### CSS classes - -The *Material* theme uses the following CSS classes in the HTML: - -- `doc`: on all the following elements -- `doc-children`: on `div`s containing the children of an object -- `doc-object`: on `div`s containing an object - - `doc-attribute`: on `div`s containing an attribute - - `doc-class`: on `div`s containing a class - - `doc-function`: on `div`s containing a function - - `doc-method`: on `div`s containing a method - - `doc-module`: on `div`s containing a module -- `doc-heading`: on objects headings -- `doc-contents`: on `div`s wrapping the docstring then the children (if any) - - `first`: same, but only on the root object's contents `div` -- `doc-properties`: on `span`s wrapping the object's properties - - `doc-property`: on `small` elements containing a property - - `doc-property-PROPERTY`: same, where `PROPERTY` is replaced by the actual property - -!!! example "Example with colorful properties" - === "CSS" - ```css - .doc-property { border-radius: 15px; padding: 0 5px; } - .doc-property-special { background-color: blue; color: white; } - .doc-property-private { background-color: red; color: white; } - .doc-property-property { background-color: green; color: white; } - .doc-property-read-only { background-color: yellow; color: black; } + === "Result HTML for doc2" + ```html + <p>Please see the <a href="doc1.html#hello-world">Hello, World!</a> section.</p> ``` - - === "Result" - <style> - .prop { border-radius: 15px; padding: 0 5px; } - </style> - <h3 style="margin: 0;"><span> - <small class="prop" style="background-color: blue; color: white !important;">special</small> - <small class="prop" style="background-color: red; color: white !important;">private</small> - <small class="prop" style="background-color: green; color: white !important;">property</small> - <small class="prop" style="background-color: yellow; color: black !important;">read-only</small> - </span></h3> - - As you can see, CSS is not my field of predilection... + ## Watch directories You can add directories to watch with the `watch` key. It accepts a list of paths. -```yaml -plugins: - - mkdocstrings: - watch: - - src/my_package_1 - - src/my_package_2 -``` +!!! example "mkdocs.yml" + ```yaml + plugins: + - mkdocstrings: + watch: + - src/my_package_1 + - src/my_package_2 + ``` + When serving your documentation and a change occur in one of the listed path, MkDocs will rebuild the site and reload the current page. @@ -321,4 +233,4 @@ MkDocs will rebuild the site and reload the current page. For example, it will not tell the Python handler to look for packages in these paths (the paths are not added to the `PYTHONPATH` variable). If you want to tell Python where to look for packages and modules, - see [Python Handler: Finding modules](../handlers/python/#finding-modules). + see [Python Handler: Finding modules](handlers/python.md#finding-modules). diff --git a/mkdocs.yml b/mkdocs.yml index edd2d02f..603d7aae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,19 +2,18 @@ site_name: "mkdocstrings" site_description: "Automatic documentation from sources, for MkDocs." site_url: "https://mkdocstrings.github.io/" repo_url: "https://github.com/mkdocstrings/mkdocstrings" +edit_uri: "blob/master/docs/" repo_name: "mkdocstrings/mkdocstrings" nav: -- Home: - - Overview: index.md - - Changelog: changelog.md - - Credits: credits.md - - License: license.md +- Overview: index.md - Usage: - - Usage: usage.md + - usage.md + - Theming: theming.md - Handlers: - - Overview: handlers/overview.md + - handlers/overview.md - Python: handlers/python.md + - Crystal: https://mkdocstrings.github.io/crystal/ - Troubleshooting: troubleshooting.md - Code Reference: - mkdocstrings: @@ -27,23 +26,23 @@ nav: - mkdocs_autorefs: - references.py: reference/autorefs/references.md - plugin.py: reference/autorefs/plugin.md -- Development: - - Contributing: contributing.md +- Contributing: + - contributing.md - Code of Conduct: code_of_conduct.md - Coverage report: coverage.md +- Changelog: changelog.md +- Credits: credits.md +- License: license.md theme: name: material - features: - - navigation.tabs - - navigation.expand palette: scheme: slate primary: teal accent: purple extra_css: -- css/mkdocstrings.css +- css/style.css markdown_extensions: - admonition @@ -59,6 +58,7 @@ markdown_extensions: plugins: - search +- section-index - coverage: html_report_dir: build/coverage - macros: diff --git a/pyproject.toml b/pyproject.toml index 491597e3..c293922a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,8 @@ description = "Automatic documentation from sources, for MkDocs." authors = ["Timothée Mazzucotelli <pawamoy@pm.me>"] license = "ISC License" readme = "README.md" -repository = "https://github.com/pawamoy/mkdocstrings" -homepage = "https://github.com/pawamoy/mkdocstrings" +repository = "https://github.com/mkdocstrings/mkdocstrings" +homepage = "https://github.com/mkdocstrings/mkdocstrings" keywords = ["mkdocs", "mkdocs-plugin", "docstrings", "autodoc", "documentation"] packages = [ { include = "mkdocstrings", from = "src" }, @@ -42,12 +42,12 @@ flake8-variables-names = "^0.0.4" flake8-pytest-style = "^1.3.0" git-changelog = "^0.4.2" httpx = "^0.16.1" - isort = {version = "^5.7.0", extras = ["pyproject"]} jinja2-cli = "^0.7.0" mkdocs-coverage = "^0.2.1" mkdocs-macros-plugin = "^0.5.0" mkdocs-material = "^6.2.7" +mkdocs-section-index = "^0.2.3" mypy = "^0.782" pytest = "^6.2.2" pytest-cov = "^2.11.1" From 2de0371f87dbe397092158cc67f4b11d16a6cdef Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Sun, 14 Feb 2021 21:47:24 +0100 Subject: [PATCH 28/40] chore: Migrate from flakehell to flake8 Flakehell is abandoned, I personally ran into problems with its faulty lingering cache, and now it's preventing me from configuring darglint. --- config/flake8.ini | 50 +++++++++++++++++++++++++++++++++++++++++++++++ duties.py | 2 +- pyproject.toml | 34 -------------------------------- 3 files changed, 51 insertions(+), 35 deletions(-) create mode 100644 config/flake8.ini diff --git a/config/flake8.ini b/config/flake8.ini new file mode 100644 index 00000000..e15c9801 --- /dev/null +++ b/config/flake8.ini @@ -0,0 +1,50 @@ +[flake8] +exclude = fixtures,docs,site +max-line-length = 132 +ignore = + # we write docstrings in markdown, not rst + RST*, + # redundant with W0622 (builtin override), which is more precise about line number + A001, + # missing docstring in magic method + D105, + # multi-line docstring summary should start at the first line + D212, + # whitespace before ‘:’ (incompatible with Black) + E203, + # redundant with E0602 (undefined variable) + F821, + # black already deals with quoting + Q000, + # use of assert + S101, + # we are not parsing XML + S405, + # line break before binary operator (incompatible with Black) + W503, + # two-lowercase-letters variable DO conform to snake_case naming style + C0103, + # redunant with D102 (missing docstring) + C0116, + # line too long + C0301, + # too many instance attributes + R0902, + # too few public methods + R0903, + # too many public methods + R0904, + # too many branches + R0912, + # too many methods + R0913, + # too many local variables + R0914, + # too many statements + R0915, + # redundant with F401 (unused import) + W0611, + # lazy formatting for logging calls + W1203, + # short name + VNE001 diff --git a/duties.py b/duties.py index 69e91b5b..814c67ee 100644 --- a/duties.py +++ b/duties.py @@ -159,7 +159,7 @@ def check_code_quality(ctx, files=PY_SRC): ctx: The context instance (passed automatically). files: The files to check. """ - ctx.run(f"flakehell lint {files}", title="Checking code quality", pty=PTY) + ctx.run(f"flake8 --config=config/flake8.ini {files}", title="Checking code quality", pty=PTY) @duty diff --git a/pyproject.toml b/pyproject.toml index c293922a..a7ffa6a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,37 +80,3 @@ balanced_wrapping = true default_section = "THIRDPARTY" known_first_party = "mkdocstrings" include_trailing_comma = true - -[tool.flakehell] -format = "colored" -max_line_length = 132 -show_source = false -exclude = ["tests/fixtures"] - -[tool.flakehell.plugins] -"*" = [ - "+*", - "-RST*", # we write docstrings in markdown, not rst - "-A001", # redundant with W0622 (builtin override), which is more precise about line number - "-D105", # missing docstring in magic method - "-D212", # multi-line docstring summary should start at the first line - "-E203", # whitespace before ‘:’ (incompatible with Black) - "-F821", # redundant with E0602 (undefined variable) - "-Q000", # black already deals with quoting - "-S101", # use of assert - "-S405", # we are not parsing XML - "-W503", # line break before binary operator (incompatible with Black) - "-C0103", # two-lowercase-letters variable DO conform to snake_case naming style - "-C0116", # redunant with D102 (missing docstring) - "-C0301", # line too long - "-R0902", # too many instance attributes - "-R0903", # too few public methods - "-R0904", # too many public methods - "-R0912", # too many branches - "-R0913", # too many methods - "-R0914", # too many local variables - "-R0915", # too many statements - "-W0611", # redundant with F401 (unused import) - "-W1203", # lazy formatting for logging calls - "-VNE001", # short name -] From 432142388a3c427b8e1003ebe11b29e61d873484 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Sun, 14 Feb 2021 22:44:04 +0100 Subject: [PATCH 29/40] tests: Build test sites into a temp directory This was causing random test failures because multiple tests concurrently try to delete and re-create the same site dir. Refactor to use pytest fixtures instead while on it. Bonus: allow docstrings to not specify Arguments sections etc. But if they are specified, they will still be checked. Here because it's ridiculous to document fixtures in every test function. Generally because sometimes all args are obvious if it's just an override. --- config/flake8.ini | 1 + tests/test_extension.py | 114 +++++++++++++++++----------------------- tests/test_plugin.py | 6 ++- 3 files changed, 53 insertions(+), 68 deletions(-) diff --git a/config/flake8.ini b/config/flake8.ini index e15c9801..8ca67f33 100644 --- a/config/flake8.ini +++ b/config/flake8.ini @@ -1,6 +1,7 @@ [flake8] exclude = fixtures,docs,site max-line-length = 132 +strictness = long ignore = # we write docstrings in markdown, not rst RST*, diff --git a/tests/test_extension.py b/tests/test_extension.py index a9dc9fca..c1eca778 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -1,6 +1,5 @@ """Tests for the extension module.""" from collections import ChainMap -from contextlib import contextmanager from textwrap import dedent import pytest @@ -8,22 +7,16 @@ from mkdocs import config -@contextmanager -def ext_markdown(**kwargs): - """Yield a Markdown instance with MkdocstringsExtension, with config adjustments from **kwargs. - - Arguments: - **kwargs: Changes to apply to the config, on top of the default config. - - Yields: - A `markdown.Markdown` instance. - """ +@pytest.fixture(name="ext_markdown") +def fixture_ext_markdown(request, tmp_path): + """Yield a Markdown instance with MkdocstringsExtension, with config adjustments.""" conf = config.Config(schema=config.DEFAULT_SCHEMA) conf_dict = { "site_name": "foo", + "site_dir": str(tmp_path), "plugins": [{"mkdocstrings": {"default_handler": "python"}}], - **kwargs, + **getattr(request, "param", {}), } # Re-create it manually as a workaround for https://github.com/mkdocs/mkdocs/issues/2289 mdx_configs = dict(ChainMap(*conf_dict.get("markdown_extensions", []))) @@ -41,102 +34,91 @@ def ext_markdown(**kwargs): conf["plugins"]["mkdocstrings"].on_post_build(conf) -def test_render_html_escaped_sequences(): +def test_render_html_escaped_sequences(ext_markdown): """Assert HTML-escaped sequences are correctly parsed as XML.""" - with ext_markdown() as md: - md.convert("::: tests.fixtures.html_escaped_sequences") + ext_markdown.convert("::: tests.fixtures.html_escaped_sequences") -def test_multiple_footnotes(): +@pytest.mark.parametrize("ext_markdown", [{"markdown_extensions": [{"footnotes": {}}]}], indirect=["ext_markdown"]) +def test_multiple_footnotes(ext_markdown): """Assert footnotes don't get added to subsequent docstrings.""" - with ext_markdown(markdown_extensions=[{"footnotes": {}}]) as md: - output = md.convert( - dedent( - """ - Top.[^aaa] + output = ext_markdown.convert( + dedent( + """ + Top.[^aaa] - ::: tests.fixtures.footnotes.func_a + ::: tests.fixtures.footnotes.func_a - ::: tests.fixtures.footnotes.func_b + ::: tests.fixtures.footnotes.func_b - ::: tests.fixtures.footnotes.func_c + ::: tests.fixtures.footnotes.func_c - [^aaa]: Top footnote - """, - ), - ) + [^aaa]: Top footnote + """, + ), + ) assert output.count("Footnote A") == 1 assert output.count("Footnote B") == 1 assert output.count("Top footnote") == 1 -def test_markdown_heading_level(): +def test_markdown_heading_level(ext_markdown): """Assert that Markdown headings' level doesn't exceed heading_level.""" - with ext_markdown() as md: - output = md.convert("::: tests.fixtures.headings\n rendering:\n show_root_heading: true") + output = ext_markdown.convert("::: tests.fixtures.headings\n rendering:\n show_root_heading: true") assert ">Foo</h3>" in output assert ">Bar</h5>" in output assert ">Baz</h6>" in output -def test_keeps_preceding_text(): +def test_keeps_preceding_text(ext_markdown): """Assert that autodoc is recognized in the middle of a block and preceding text is kept.""" - with ext_markdown() as md: - output = md.convert("**preceding**\n::: tests.fixtures.headings") + output = ext_markdown.convert("**preceding**\n::: tests.fixtures.headings") assert "<strong>preceding</strong>" in output assert ">Foo</h2>" in output assert ":::" not in output -def test_reference_inside_autodoc(): +def test_reference_inside_autodoc(ext_markdown): """Assert cross-reference Markdown extension works correctly.""" - with ext_markdown() as md: - output = md.convert("::: tests.fixtures.cross_reference") + output = ext_markdown.convert("::: tests.fixtures.cross_reference") snippet = 'Link to <span data-mkdocstrings-identifier="something.Else">something.Else</span>.' assert snippet in output -def test_html_inside_heading(): +def test_html_inside_heading(ext_markdown): """Assert that headings don't double-escape HTML.""" - with ext_markdown() as md: - output = md.convert("::: tests.fixtures.builtin") + output = ext_markdown.convert("::: tests.fixtures.builtin") assert "=<" in output assert "&" not in output @pytest.mark.parametrize( - ("permalink_setting", "expect_permalink"), + ("ext_markdown", "expect_permalink"), [ - ("@@@", "@@@"), - ("TeSt", "TeSt"), - (True, "¶"), + ({"markdown_extensions": [{"toc": {"permalink": "@@@"}}]}, "@@@"), + ({"markdown_extensions": [{"toc": {"permalink": "TeSt"}}]}, "TeSt"), + ({"markdown_extensions": [{"toc": {"permalink": True}}]}, "¶"), ], + indirect=["ext_markdown"], ) -def test_no_double_toc(permalink_setting, expect_permalink): - """ - Assert that the 'toc' extension doesn't apply its modification twice. - - Arguments: - permalink_setting: The 'permalink' setting of 'toc' extension. - expect_permalink: Text of the permalink to search for in the output. - """ - with ext_markdown(markdown_extensions=[{"toc": {"permalink": permalink_setting}}]) as md: - output = md.convert( - dedent( - """ - # aa - - ::: tests.fixtures.headings - rendering: - show_root_toc_entry: false - - # bb - """ - ) +def test_no_double_toc(ext_markdown, expect_permalink): + """Assert that the 'toc' extension doesn't apply its modification twice.""" + output = ext_markdown.convert( + dedent( + """ + # aa + + ::: tests.fixtures.headings + rendering: + show_root_toc_entry: false + + # bb + """ ) + ) assert output.count(expect_permalink) == 5 assert 'id="tests.fixtures.headings--foo"' in output - assert md.toc_tokens == [ # noqa: E1101 (the member gets populated only with 'toc' extension) + assert ext_markdown.toc_tokens == [ # noqa: E1101 (the member gets populated only with 'toc' extension) { "level": 1, "id": "aa", diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 61080301..14078fc2 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -8,6 +8,8 @@ @pytest.mark.xfail(sys.version.startswith("3.9"), reason="pytkdocs is failing on Python 3.9") -def test_plugin(): +def test_plugin(tmp_path): """Build our own documentation.""" - build(load_config()) + config = load_config() + config["site_dir"] = tmp_path + build(config) From e2d74efb0d59f9a1aa45e42525ceb1d4b7638426 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 17 Feb 2021 21:35:50 +0100 Subject: [PATCH 30/40] refactor: Use the autorefs plugin from its new external location PR #235: https://github.com/mkdocstrings/mkdocstrings/pull/235 --- config/coverage.ini | 3 - duties.py | 5 +- pyproject.toml | 7 +- src/mkdocs_autorefs/plugin.py | 186 ---------------------------- src/mkdocs_autorefs/references.py | 199 ------------------------------ src/mkdocstrings/extension.py | 2 +- src/mkdocstrings/plugin.py | 2 +- tests/test_references.py | 146 ---------------------- 8 files changed, 6 insertions(+), 544 deletions(-) delete mode 100644 src/mkdocs_autorefs/plugin.py delete mode 100644 src/mkdocs_autorefs/references.py delete mode 100644 tests/test_references.py diff --git a/config/coverage.ini b/config/coverage.ini index 0e7d7d51..27b21edf 100644 --- a/config/coverage.ini +++ b/config/coverage.ini @@ -1,15 +1,12 @@ [coverage:paths] source = src/mkdocstrings - src/mkdocs_autorefs */site-packages/mkdocstrings - */site-packages/mkdocs_autorefs [coverage:run] branch = true source = src/mkdocstrings - src/mkdocs_autorefs tests parallel = true diff --git a/duties.py b/duties.py index 814c67ee..5b572a3e 100644 --- a/duties.py +++ b/duties.py @@ -11,7 +11,7 @@ from git_changelog.build import Changelog, Version from jinja2.sandbox import SandboxedEnvironment -PY_SRC_LIST = ("src/mkdocstrings", "src/mkdocs_autorefs", "tests", "duties.py", "docs/macros.py") +PY_SRC_LIST = ("src/mkdocstrings", "tests", "duties.py", "docs/macros.py") PY_SRC = " ".join(PY_SRC_LIST) TESTING = os.environ.get("TESTING", "0") in {"1", "true"} CI = os.environ.get("CI", "0") in {"1", "true", "yes", ""} @@ -214,8 +214,7 @@ def check_types(ctx): Arguments: ctx: The context instance (passed automatically). """ - for f in PY_SRC_LIST: - ctx.run(f"mypy --config-file config/mypy.ini {f}", title="Type-checking", pty=PTY, progress=True) + ctx.run(f"mypy --config-file config/mypy.ini {PY_SRC}", title="Type-checking", pty=PTY, progress=True) @duty(silent=True) diff --git a/pyproject.toml b/pyproject.toml index a7ffa6a8..eedaf2f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,7 @@ readme = "README.md" repository = "https://github.com/mkdocstrings/mkdocstrings" homepage = "https://github.com/mkdocstrings/mkdocstrings" keywords = ["mkdocs", "mkdocs-plugin", "docstrings", "autodoc", "documentation"] -packages = [ - { include = "mkdocstrings", from = "src" }, - { include = "mkdocs_autorefs", from = "src" } -] +packages = [ { include = "mkdocstrings", from = "src" } ] include = [ "README.md", "pyproject.toml" @@ -27,6 +24,7 @@ Jinja2 = "^2.11" Markdown = "^3.3" MarkupSafe = "^1.1" mkdocs = "^1.1" +mkdocs-autorefs = "^0.1" pymdown-extensions = ">=6.3, <9.0" pytkdocs = ">=0.2.0, <0.11.0" @@ -65,7 +63,6 @@ pep8-naming = "^0.11.1" [tool.poetry.plugins."mkdocs.plugins"] mkdocstrings = "mkdocstrings.plugin:MkdocstringsPlugin" -autorefs = "mkdocs_autorefs.plugin:AutorefsPlugin" [tool.black] line-length = 120 diff --git a/src/mkdocs_autorefs/plugin.py b/src/mkdocs_autorefs/plugin.py deleted file mode 100644 index 5ef8a08d..00000000 --- a/src/mkdocs_autorefs/plugin.py +++ /dev/null @@ -1,186 +0,0 @@ -""" -This module contains the "mkdocs-autorefs" plugin. - -After each page is processed by the Markdown converter, this plugin stores absolute URLs of every HTML anchors -it finds to later be able to fix unresolved references. -It stores them during the [`on_page_content` event hook](https://www.mkdocs.org/user-guide/plugins/#on_page_content). - -Just before writing the final HTML to the disc, during the -[`on_post_page` event hook](https://www.mkdocs.org/user-guide/plugins/#on_post_page), -this plugin searches for references of the form `[identifier][]` or `[title][identifier]` that were not resolved, -and fixes them using the previously stored identifier-URL mapping. -""" - -import logging -from typing import Callable, Dict, Optional - -from mkdocs.config import Config -from mkdocs.plugins import BasePlugin -from mkdocs.structure.pages import Page -from mkdocs.structure.toc import AnchorLink -from mkdocs.utils import warning_filter - -from mkdocs_autorefs.references import AutorefsExtension, fix_refs - -log = logging.getLogger(f"mkdocs.plugins.{__name__}") -log.addFilter(warning_filter) - - -class AutorefsPlugin(BasePlugin): - """ - An `mkdocs` plugin. - - This plugin defines the following event hooks: - - - `on_config` - - `on_page_content` - - `on_post_page` - - Check the [Developing Plugins](https://www.mkdocs.org/user-guide/plugins/#developing-plugins) page of `mkdocs` - for more information about its plugin system. - """ - - scan_toc: bool = True - current_page: Optional[str] = None - - def __init__(self) -> None: - """Initialize the object.""" - super().__init__() - self._url_map: Dict[str, str] = {} - self.get_fallback_anchor: Callable[[str], Optional[str]] = lambda identifier: None - - def register_anchor(self, page: str, anchor: str): - """ - Register that an anchor corresponding to an identifier was encountered when rendering the page. - - Arguments: - page: The relative URL of the current page. Examples: `'foo/bar/'`, `'foo/index.html'` - anchor: The HTML anchor (without '#') as a string. - """ - self._url_map[anchor] = f"{page}#{anchor}" - - def get_item_url(self, anchor: str) -> str: - """ - Return a site-relative URL with anchor to the identifier, if it's present anywhere. - - Arguments: - anchor: The identifier (one that [collect][mkdocstrings.handlers.base.BaseCollector.collect] can accept). - - Returns: - A site-relative URL. - - Raises: - KeyError: If there isn't an item by this identifier anywhere on the site. - """ - try: - return self._url_map[anchor] - except KeyError: - new_anchor = self.get_fallback_anchor(anchor) - if new_anchor and new_anchor in self._url_map: - return self._url_map[new_anchor] - raise - - def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613,R0201 (unused arguments, cannot be static) - """ - Instantiate our Markdown extension. - - Hook for the [`on_config` event](https://www.mkdocs.org/user-guide/plugins/#on_config). - In this hook, we instantiate our [`AutorefsExtension`][mkdocs_autorefs.references.AutorefsExtension] - and add it to the list of Markdown extensions used by `mkdocs`. - - Arguments: - config: The MkDocs config object. - kwargs: Additional arguments passed by MkDocs. - - Returns: - The modified config. - """ - log.debug(f"{__name__}: Adding AutorefsExtension to the list") - config["markdown_extensions"].append(AutorefsExtension()) - return config - - def on_page_markdown(self, markdown: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) - """ - Remember which page is the current one. - - Arguments: - markdown: Input Markdown. - page: The related MkDocs page instance. - kwargs: Additional arguments passed by MkDocs. - - Returns: - The same Markdown. We only use this hook to map anchors to URLs. - """ - self.current_page = page.url - return markdown - - def on_page_content(self, html: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) - """ - Map anchors to URLs. - - Hook for the [`on_page_content` event](https://www.mkdocs.org/user-guide/plugins/#on_page_content). - In this hook, we map the IDs of every anchor found in the table of contents to the anchors absolute URLs. - This mapping will be used later to fix unresolved reference of the form `[title][identifier]` or - `[identifier][]`. - - Arguments: - html: HTML converted from Markdown. - page: The related MkDocs page instance. - kwargs: Additional arguments passed by MkDocs. - - Returns: - The same HTML. We only use this hook to map anchors to URLs. - """ - if self.scan_toc: - log.debug(f"{__name__}: Mapping identifiers to URLs for page {page.file.src_path}") - for item in page.toc.items: - self.map_urls(page.url, item) - return html - - def map_urls(self, base_url: str, anchor: AnchorLink) -> None: - """ - Recurse on every anchor to map its ID to its absolute URL. - - This method populates `self.url_map` by side-effect. - - Arguments: - base_url: The base URL to use as a prefix for each anchor's relative URL. - anchor: The anchor to process and to recurse on. - """ - self.register_anchor(base_url, anchor.id) - for child in anchor.children: - self.map_urls(base_url, child) - - def on_post_page(self, output: str, page: Page, **kwargs) -> str: # noqa: W0613 (unused arguments) - """ - Fix cross-references. - - Hook for the [`on_post_page` event](https://www.mkdocs.org/user-guide/plugins/#on_post_page). - In this hook, we try to fix unresolved references of the form `[title][identifier]` or `[identifier][]`. - Doing that allows the user of `mkdocstrings` to cross-reference objects in their documentation strings. - It uses the native Markdown syntax so it's easy to remember and use. - - We log a warning for each reference that we couldn't map to an URL, but try to be smart and ignore identifiers - that do not look legitimate (sometimes documentation can contain strings matching - our [`AUTO_REF_RE`][mkdocs_autorefs.references.AUTO_REF_RE] regular expression that did not intend to reference anything). - We currently ignore references when their identifier contains a space or a slash. - - Arguments: - output: HTML converted from Markdown. - page: The related MkDocs page instance. - kwargs: Additional arguments passed by MkDocs. - - Returns: - Modified HTML. - """ - log.debug(f"{__name__}: Fixing references in page {page.file.src_path}") - - fixed_output, unmapped = fix_refs(output, page.url, self.get_item_url) - - if unmapped and log.isEnabledFor(logging.WARNING): - for ref in unmapped: - log.warning( - f"{__name__}: {page.file.src_path}: Could not find cross-reference target '[{ref}]'", - ) - - return fixed_output diff --git a/src/mkdocs_autorefs/references.py b/src/mkdocs_autorefs/references.py deleted file mode 100644 index 53f5205d..00000000 --- a/src/mkdocs_autorefs/references.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Cross-references module.""" - -import re -from html import escape, unescape -from typing import Any, Callable, List, Match, Tuple, Union -from xml.etree.ElementTree import Element - -from markdown import Markdown -from markdown.extensions import Extension -from markdown.inlinepatterns import REFERENCE_RE, ReferenceInlineProcessor - -AUTO_REF_RE = re.compile(r'<span data-mkdocstrings-identifier=("?)(?P<identifier>[^"<>]*)\1>(?P<title>.*?)</span>') -""" -A regular expression to match mkdocstrings' special reference markers -in the [`on_post_page` hook][mkdocs_autorefs.plugin.AutorefsPlugin.on_post_page]. -""" - -EvalIDType = Tuple[Any, Any, Any] - - -class AutoRefInlineProcessor(ReferenceInlineProcessor): - """A Markdown extension.""" - - def __init__(self, *args, **kwargs): # noqa: D107 - super().__init__(REFERENCE_RE, *args, **kwargs) - - # Code based on - # https://github.com/Python-Markdown/markdown/blob/8e7528fa5c98bf4652deb13206d6e6241d61630b/markdown/inlinepatterns.py#L780 - - def handleMatch(self, m, data) -> Union[Element, EvalIDType]: # noqa: N802 (parent's casing) - """ - Handle an element that matched. - - Arguments: - m: The match object. - data: The matched data. - - Returns: - A new element or a tuple. - """ - text, index, handled = self.getText(data, m.end(0)) - if not handled: - return None, None, None - - identifier, end, handled = self.evalId(data, index, text) - if not handled: - return None, None, None - - if re.search(r"[/ \x00-\x1f]", identifier): # type: ignore - # Do nothing if the matched reference contains: - # - a space, slash or control character (considered unintended); - # - specifically \x01 is used by Python-Markdown HTML stash when there's inline formatting, - # but references with Markdown formatting are not possible anyway. - return None, m.start(0), end - - return self.makeTag(identifier, text), m.start(0), end - - def evalId(self, data: str, index: int, text: str) -> EvalIDType: # noqa: N802 (parent's casing) - """ - Evaluate the id portion of `[ref][id]`. - - If `[ref][]` use `[ref]`. - - Arguments: - data: The data to evaluate. - index: The starting position. - text: The text to use when no identifier. - - Returns: - A tuple containing the identifier, its end position, and whether it matched. - """ - m = self.RE_LINK.match(data, pos=index) - if not m: - return None, index, False - identifier = m.group(1) or text - end = m.end(0) - return identifier, end, True - - def makeTag(self, identifier: str, text: str) -> Element: # noqa: N802,W0221 (parent's casing, different params) - """ - Create a tag that can be matched by `AUTO_REF_RE`. - - Arguments: - identifier: The identifier to use in the HTML property. - text: The text to use in the HTML tag. - - Returns: - A new element. - """ - el = Element("span") - el.set("data-mkdocstrings-identifier", identifier) - el.text = text - return el - - -def relative_url(url_a: str, url_b: str) -> str: - """ - Compute the relative path from URL A to URL B. - - Arguments: - url_a: URL A. - url_b: URL B. - - Returns: - The relative URL to go from A to B. - """ - parts_a = url_a.split("/") - url_b, anchor = url_b.split("#", 1) - parts_b = url_b.split("/") - - # remove common left parts - while parts_a and parts_b and parts_a[0] == parts_b[0]: - parts_a.pop(0) - parts_b.pop(0) - - # go up as many times as remaining a parts' depth - levels = len(parts_a) - 1 - parts_relative = [".."] * levels + parts_b - relative = "/".join(parts_relative) - return f"{relative}#{anchor}" - - -def fix_ref(url_mapper: Callable[[str], str], from_url: str, unmapped: List[str]) -> Callable: - """ - Return a `repl` function for [`re.sub`](https://docs.python.org/3/library/re.html#re.sub). - - In our context, we match Markdown references and replace them with HTML links. - - When the matched reference's identifier was not mapped to an URL, we append the identifier to the outer - `unmapped` list. It generally means the user is trying to cross-reference an object that was not collected - and rendered, making it impossible to link to it. We catch this exception in the caller to issue a warning. - - Arguments: - url_mapper: A callable that gets an object's site URL by its identifier, - such as [mkdocs_autorefs.plugin.AutorefsPlugin.get_item_url][]. - from_url: The URL of the base page, from which we link towards the targeted pages. - unmapped: A list to store unmapped identifiers. - - Returns: - The actual function accepting a [`Match` object](https://docs.python.org/3/library/re.html#match-objects) - and returning the replacement strings. - """ - - def inner(match: Match): - identifier = match["identifier"] - title = match["title"] - - try: - url = relative_url(from_url, url_mapper(unescape(identifier))) - except KeyError: - unmapped.append(identifier) - if title == identifier: - return f"[{identifier}][]" - return f"[{title}][{identifier}]" - - return f'<a href="{escape(url)}">{title}</a>' - - return inner - - -def fix_refs( - html: str, - from_url: str, - url_mapper: Callable[[str], str], -) -> Tuple[str, List[str]]: - """ - Fix all references in the given HTML text. - - Arguments: - html: The text to fix. - from_url: The URL at which this HTML is served. - url_mapper: A callable that gets an object's site URL by its identifier, - such as [mkdocs_autorefs.plugin.AutorefsPlugin.get_item_url][]. - - Returns: - The fixed HTML. - """ - unmapped = [] # type: ignore - html = AUTO_REF_RE.sub(fix_ref(url_mapper, from_url, unmapped), html) - return html, unmapped - - -class AutorefsExtension(Extension): - """Extension that inserts auto-references in Markdown.""" - - def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) - """ - Register the extension. - - Add an instance of our [`AutoRefInlineProcessor`][mkdocs_autorefs.references.AutoRefInlineProcessor] to the Markdown parser. - - Arguments: - md: A `markdown.Markdown` instance. - """ - md.inlinePatterns.register( - AutoRefInlineProcessor(md), - "mkdocstrings", - priority=168, # Right after markdown.inlinepatterns.ReferenceInlineProcessor - ) diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 135ffc51..6bb4cfd3 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -34,8 +34,8 @@ from markdown.blockprocessors import BlockProcessor from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor - from mkdocs_autorefs.plugin import AutorefsPlugin + from mkdocstrings.handlers.base import CollectionError, CollectorItem, Handlers from mkdocstrings.loggers import get_logger diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 0eff9ed6..921bbc84 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -21,8 +21,8 @@ from mkdocs.config.config_options import Type as MkType from mkdocs.plugins import BasePlugin from mkdocs.utils import write_file - from mkdocs_autorefs.plugin import AutorefsPlugin + from mkdocstrings.extension import MkdocstringsExtension from mkdocstrings.handlers.base import BaseHandler, Handlers from mkdocstrings.loggers import get_logger diff --git a/tests/test_references.py b/tests/test_references.py deleted file mode 100644 index fd587910..00000000 --- a/tests/test_references.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Tests for the references module.""" -import markdown -import pytest - -from mkdocs_autorefs.references import AutorefsExtension, fix_refs, relative_url - - -@pytest.mark.parametrize( - ("current_url", "to_url", "href_url"), - [ - ("a/", "a#b", "#b"), - ("a/", "a/b#c", "b#c"), - ("a/b/", "a/b#c", "#c"), - ("a/b/", "a/c#d", "../c#d"), - ("a/b/", "a#c", "..#c"), - ("a/b/c/", "d#e", "../../../d#e"), - ("a/b/", "c/d/#e", "../../c/d/#e"), - ("a/index.html", "a/index.html#b", "#b"), - ("a/index.html", "a/b.html#c", "b.html#c"), - ("a/b.html", "a/b.html#c", "#c"), - ("a/b.html", "a/c.html#d", "c.html#d"), - ("a/b.html", "a/index.html#c", "index.html#c"), - ("a/b/c.html", "d.html#e", "../../d.html#e"), - ("a/b.html", "c/d.html#e", "../c/d.html#e"), - ("a/b/index.html", "a/b/c/d.html#e", "c/d.html#e"), - ("", "#x", "#x"), - ("a/", "#x", "../#x"), - ("a/b.html", "#x", "../#x"), - ("", "a/#x", "a/#x"), - ("", "a/b.html#x", "a/b.html#x"), - ], -) -def test_relative_url(current_url, to_url, href_url): - """ - Compute relative URLs correctly. - - Arguments: - current_url: The URL of the source page. - to_url: The URL of the target page. - href_url: The relative URL to put in the `href` HTML field. - """ - assert relative_url(current_url, to_url) == href_url - - -def run_references_test(url_map, source, output, unmapped=None, from_url="page.html"): - """ - Help running tests about references. - - Arguments: - url_map: The URL mapping. - source: The source text. - output: The expected output. - unmapped: The expected unmapped list. - from_url: The source page URL. - """ - md = markdown.Markdown(extensions=[AutorefsExtension()]) - content = md.convert(source) - - actual_output, actual_unmapped = fix_refs(content, from_url, url_map.__getitem__) - assert actual_output == output - assert actual_unmapped == (unmapped or []) - - -def test_reference_implicit(): - """Check implicit references (identifier only).""" - run_references_test( - url_map={"Foo": "foo.html#Foo"}, - source="This [Foo][].", - output='<p>This <a href="foo.html#Foo">Foo</a>.</p>', - ) - - -def test_reference_explicit_with_markdown_text(): - """Check explicit references with Markdown formatting.""" - run_references_test( - url_map={"Foo": "foo.html#Foo"}, - source="This [`Foo`][Foo].", - output='<p>This <a href="foo.html#Foo"><code>Foo</code></a>.</p>', - ) - - -def test_reference_with_punctuation(): - """Check references with punctuation.""" - run_references_test( - url_map={'Foo&"bar': 'foo.html#Foo&"bar'}, - source='This [Foo&"bar][].', - output='<p>This <a href="foo.html#Foo&"bar">Foo&"bar</a>.</p>', - ) - - -def test_no_reference_with_space(): - """Check that references with spaces are not fixed.""" - run_references_test( - url_map={"Foo bar": "foo.html#Foo bar"}, - source="This [Foo bar][].", - output="<p>This [Foo bar][].</p>", - ) - - -def test_no_reference_inside_markdown(): - """Check that references inside code are not fixed.""" - run_references_test( - url_map={"Foo": "foo.html#Foo"}, - source="This `[Foo][]`.", - output="<p>This <code>[Foo][]</code>.</p>", - ) - - -def test_missing_reference(): - """Check that implicit references are correctly seen as unmapped.""" - run_references_test( - url_map={"NotFoo": "foo.html#NotFoo"}, - source="[Foo][]", - output="<p>[Foo][]</p>", - unmapped=["Foo"], - ) - - -def test_missing_reference_with_markdown_text(): - """Check unmapped explicit references.""" - run_references_test( - url_map={"NotFoo": "foo.html#NotFoo"}, - source="[`Foo`][Foo]", - output="<p>[<code>Foo</code>][Foo]</p>", - unmapped=["Foo"], - ) - - -def test_missing_reference_with_markdown_id(): - """Check unmapped explicit references with Markdown in the identifier.""" - run_references_test( - url_map={"NotFoo": "foo.html#NotFoo"}, - source="[Foo][*oh*]", - output="<p>[Foo][*oh*]</p>", - unmapped=["*oh*"], - ) - - -def test_missing_reference_with_markdown_implicit(): - """Check that implicit references are not fixed when the identifier is not the exact one.""" - run_references_test( - url_map={"Foo": "foo.html#Foo"}, - source="[`Foo`][]", - output="<p>[<code>Foo</code>][]</p>", - unmapped=[], - ) From 6357144b100be6a2e7e6140e035c289c225cec22 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Tue, 23 Feb 2021 19:10:27 +0100 Subject: [PATCH 31/40] fix: Don't double-escape characters in highlighted headings --- src/mkdocstrings/handlers/rendering.py | 2 ++ tests/fixtures/string_annotation.py | 8 ++++++++ tests/test_extension.py | 9 +++++++++ 3 files changed, 19 insertions(+) create mode 100644 tests/fixtures/string_annotation.py diff --git a/src/mkdocstrings/handlers/rendering.py b/src/mkdocstrings/handlers/rendering.py index 970b2413..4cfcee9a 100644 --- a/src/mkdocstrings/handlers/rendering.py +++ b/src/mkdocstrings/handlers/rendering.py @@ -62,6 +62,8 @@ def highlight( # noqa: W0221 (intentionally different params, we're extending t Returns: The highlighted code as HTML text, marked safe (not escaped for HTML). """ + if isinstance(src, Markup): + src = src.unescape() if dedent: src = textwrap.dedent(src) diff --git a/tests/fixtures/string_annotation.py b/tests/fixtures/string_annotation.py new file mode 100644 index 00000000..cc0f09f3 --- /dev/null +++ b/tests/fixtures/string_annotation.py @@ -0,0 +1,8 @@ +from typing import Literal + + +class Foo: + @property + def foo() -> Literal["hi"]: + "hi" + return "hi" diff --git a/tests/test_extension.py b/tests/test_extension.py index c1eca778..e2e92903 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -1,4 +1,5 @@ """Tests for the extension module.""" +import sys from collections import ChainMap from textwrap import dedent @@ -85,6 +86,14 @@ def test_reference_inside_autodoc(ext_markdown): assert snippet in output +@pytest.mark.skipif(sys.version_info < (3, 8), reason="typing.Literal requires Python 3.8") +def test_quote_inside_annotation(ext_markdown): + """Assert that inline highlighting doesn't double-escape HTML.""" + output = ext_markdown.convert("::: tests.fixtures.string_annotation.Foo") + assert ";hi&" in output + assert "&" not in output + + def test_html_inside_heading(ext_markdown): """Assert that headings don't double-escape HTML.""" output = ext_markdown.convert("::: tests.fixtures.builtin") From 2724a83087b004bb224fc255ded1ad3d1b971072 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 24 Feb 2021 20:49:29 +0100 Subject: [PATCH 32/40] chore: Automatically generate API reference stub files PR 242: https://github.com/mkdocstrings/mkdocstrings/pull/242 --- docs/gen_doc_stubs.py | 14 ++++++++++++++ docs/reference/extension.md | 1 - docs/reference/handlers/base.md | 3 --- docs/reference/handlers/python.md | 1 - docs/reference/handlers/rendering.md | 1 - docs/reference/plugin.md | 1 - mkdocs.yml | 4 ++++ pyproject.toml | 1 + src/mkdocstrings/loggers.py | 6 +++--- tests/test_plugin.py | 3 ++- 10 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 docs/gen_doc_stubs.py delete mode 100644 docs/reference/extension.md delete mode 100644 docs/reference/handlers/base.md delete mode 100644 docs/reference/handlers/python.md delete mode 100644 docs/reference/handlers/rendering.md delete mode 100644 docs/reference/plugin.md diff --git a/docs/gen_doc_stubs.py b/docs/gen_doc_stubs.py new file mode 100644 index 00000000..f37247c3 --- /dev/null +++ b/docs/gen_doc_stubs.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from pathlib import Path + +import mkdocs_gen_files + +for path in Path("src", "mkdocstrings").glob("**/*.py"): + doc_path = Path("reference", path.relative_to("src", "mkdocstrings")).with_suffix(".md") + + with mkdocs_gen_files.open(doc_path, "w") as f: + ident = ".".join(path.relative_to("src").with_suffix("").parts) + print("::: " + ident, file=f) + + mkdocs_gen_files.set_edit_path(doc_path, Path('..', path)) diff --git a/docs/reference/extension.md b/docs/reference/extension.md deleted file mode 100644 index 93d3efb8..00000000 --- a/docs/reference/extension.md +++ /dev/null @@ -1 +0,0 @@ -::: mkdocstrings.extension diff --git a/docs/reference/handlers/base.md b/docs/reference/handlers/base.md deleted file mode 100644 index 375d1396..00000000 --- a/docs/reference/handlers/base.md +++ /dev/null @@ -1,3 +0,0 @@ -::: mkdocstrings.handlers.base - rendering: - show_root_heading: false diff --git a/docs/reference/handlers/python.md b/docs/reference/handlers/python.md deleted file mode 100644 index b4935a99..00000000 --- a/docs/reference/handlers/python.md +++ /dev/null @@ -1 +0,0 @@ -::: mkdocstrings.handlers.python diff --git a/docs/reference/handlers/rendering.md b/docs/reference/handlers/rendering.md deleted file mode 100644 index d434aaa1..00000000 --- a/docs/reference/handlers/rendering.md +++ /dev/null @@ -1 +0,0 @@ -::: mkdocstrings.handlers.rendering diff --git a/docs/reference/plugin.md b/docs/reference/plugin.md deleted file mode 100644 index be612e11..00000000 --- a/docs/reference/plugin.md +++ /dev/null @@ -1 +0,0 @@ -::: mkdocstrings.plugin diff --git a/mkdocs.yml b/mkdocs.yml index 603d7aae..5a0c7ff8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,6 +23,7 @@ nav: - python.py: reference/handlers/python.md - extension.py: reference/extension.md - plugin.py: reference/plugin.md + - loggers.py: reference/loggers.md - mkdocs_autorefs: - references.py: reference/autorefs/references.md - plugin.py: reference/autorefs/plugin.md @@ -58,6 +59,9 @@ markdown_extensions: plugins: - search +- gen-files: + scripts: + - docs/gen_doc_stubs.py - section-index - coverage: html_report_dir: build/coverage diff --git a/pyproject.toml b/pyproject.toml index eedaf2f5..16a5d890 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ isort = {version = "^5.7.0", extras = ["pyproject"]} jinja2-cli = "^0.7.0" mkdocs-coverage = "^0.2.1" mkdocs-macros-plugin = "^0.5.0" +mkdocs-gen-files = {version = "^0.3.0", markers = "python_version>='3.7'"} mkdocs-material = "^6.2.7" mkdocs-section-index = "^0.2.3" mypy = "^0.782" diff --git a/src/mkdocstrings/loggers.py b/src/mkdocstrings/loggers.py index 83b7b74e..26c7c92f 100644 --- a/src/mkdocstrings/loggers.py +++ b/src/mkdocstrings/loggers.py @@ -2,7 +2,7 @@ import logging from pathlib import Path -from typing import Callable, Optional +from typing import Any, Callable, Optional, Tuple from jinja2 import contextfunction from jinja2.runtime import Context @@ -14,7 +14,7 @@ class LoggerAdapter(logging.LoggerAdapter): """A logger adapter to prefix messages.""" - def __init__(self, prefix, logger): + def __init__(self, prefix: str, logger): """ Initialize the object. @@ -25,7 +25,7 @@ def __init__(self, prefix, logger): super().__init__(logger, {}) self.prefix = prefix - def process(self, msg, kwargs): + def process(self, msg: str, kwargs) -> Tuple[str, Any]: """ Process the message. diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 14078fc2..3bdad73c 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -7,7 +7,8 @@ from mkdocs.config.base import load_config -@pytest.mark.xfail(sys.version.startswith("3.9"), reason="pytkdocs is failing on Python 3.9") +@pytest.mark.skipif(sys.version_info < (3, 7), reason="using plugins that require Python 3.7") +@pytest.mark.xfail(sys.version_info >= (3, 9), reason="pytkdocs is failing on Python 3.9") def test_plugin(tmp_path): """Build our own documentation.""" config = load_config() From 085227c7a89ee15d0a018088cdc0673038ba3471 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 24 Feb 2021 20:50:29 +0100 Subject: [PATCH 33/40] chore: Fold the credits generation in docs into 'gen-files' plugin --- docs/credits.md | 1 - docs/{macros.py => gen_credits.py} | 20 ++++---------------- docs/gen_doc_stubs.py | 2 +- duties.py | 2 +- mkdocs.yml | 5 +---- pyproject.toml | 1 - 6 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 docs/credits.md rename docs/{macros.py => gen_credits.py} (82%) diff --git a/docs/credits.md b/docs/credits.md deleted file mode 100644 index 76c466aa..00000000 --- a/docs/credits.md +++ /dev/null @@ -1 +0,0 @@ -[[ credits() ]] diff --git a/docs/macros.py b/docs/gen_credits.py similarity index 82% rename from docs/macros.py rename to docs/gen_credits.py index cf7f0707..8bf2130f 100644 --- a/docs/macros.py +++ b/docs/gen_credits.py @@ -1,10 +1,9 @@ -"""Macros and filters made available in Markdown pages.""" - import functools from itertools import chain from pathlib import Path import httpx +import mkdocs_gen_files import toml from jinja2 import StrictUndefined from jinja2.sandbox import SandboxedEnvironment @@ -65,17 +64,6 @@ def get_credits(): return jinja_env.from_string(template_text).render(**template_data) -def define_env(env): - """ - Add macros and filters into the Jinja2 environment. - - This hook is called by `mkdocs-macros-plugin` - when building the documentation. - - Arguments: - env: An object used to add macros and filters to the environment. - """ - - @env.macro # noqa: WPS430 (nested function) - def credits(): # noqa: W0612,W0622,WPS430 (unused, shadows credits) - return get_credits() +with mkdocs_gen_files.open("credits.md", "w") as f: + f.write(get_credits()) +mkdocs_gen_files.set_edit_path("credits.md", "gen_credits.py") diff --git a/docs/gen_doc_stubs.py b/docs/gen_doc_stubs.py index f37247c3..ccbbdc86 100644 --- a/docs/gen_doc_stubs.py +++ b/docs/gen_doc_stubs.py @@ -11,4 +11,4 @@ ident = ".".join(path.relative_to("src").with_suffix("").parts) print("::: " + ident, file=f) - mkdocs_gen_files.set_edit_path(doc_path, Path('..', path)) + mkdocs_gen_files.set_edit_path(doc_path, Path("..", path)) diff --git a/duties.py b/duties.py index 5b572a3e..e93231be 100644 --- a/duties.py +++ b/duties.py @@ -11,7 +11,7 @@ from git_changelog.build import Changelog, Version from jinja2.sandbox import SandboxedEnvironment -PY_SRC_LIST = ("src/mkdocstrings", "tests", "duties.py", "docs/macros.py") +PY_SRC_LIST = ("src/mkdocstrings", "tests", "duties.py", "docs") PY_SRC = " ".join(PY_SRC_LIST) TESTING = os.environ.get("TESTING", "0") in {"1", "true"} CI = os.environ.get("CI", "0") in {"1", "true", "yes", ""} diff --git a/mkdocs.yml b/mkdocs.yml index 5a0c7ff8..80dbc607 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,14 +61,11 @@ plugins: - search - gen-files: scripts: + - docs/gen_credits.py - docs/gen_doc_stubs.py - section-index - coverage: html_report_dir: build/coverage -- macros: - module_name: docs/macros - j2_variable_start_string: "[[" - j2_variable_end_string: "]]" - mkdocstrings: handlers: python: diff --git a/pyproject.toml b/pyproject.toml index 16a5d890..524f491d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ httpx = "^0.16.1" isort = {version = "^5.7.0", extras = ["pyproject"]} jinja2-cli = "^0.7.0" mkdocs-coverage = "^0.2.1" -mkdocs-macros-plugin = "^0.5.0" mkdocs-gen-files = {version = "^0.3.0", markers = "python_version>='3.7'"} mkdocs-material = "^6.2.7" mkdocs-section-index = "^0.2.3" From ffcca0250a503cc8c71f0b8944f6a31ccb0f0678 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 24 Feb 2021 21:44:43 +0100 Subject: [PATCH 34/40] Docs: document xrefs to a sub-heading & CSS classes for highlighting Fixes #223 --- docs/theming.md | 11 +++++++ docs/usage.md | 45 ++++++++++++++++++++++++-- src/mkdocstrings/handlers/rendering.py | 20 ++++++++++-- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/docs/theming.md b/docs/theming.md index 39159713..8a1d969c 100644 --- a/docs/theming.md +++ b/docs/theming.md @@ -128,3 +128,14 @@ The *Material* theme uses the following CSS classes in the HTML: </span></h3> As you can see, CSS is not my field of predilection... + +### Syntax highlighting + +Code blocks that occur in the docstring of an item inserted with *mkdocstrings*, as well as code blocks (such as *Source code*) that *mkdocstrings* inserts itself, are syntax-highlighted according to the same rules as other normal code blocks in your document. See more details in [mkdocstrings.handlers.rendering.Highlighter][]. + +As for the CSS class used for code blocks -- it will also match the "normal" config, so the default (`.codehilite` or `.highlight`) will match your chosen Markdown extension for highlighting. + +!!! important "Changed in version 0.15" + The CSS class used to always be `.highlight`, but now it depends on the configuration. + +Long story short, you probably should add `pymdownx.highlight` to your `markdown_extensions`, and then use `.doc-contents .highlight` as the CSS selector in case you want to change something about *mkdocstrings'* code blocks specifically. diff --git a/docs/usage.md b/docs/usage.md index ffdefc50..ce8c0bef 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -163,6 +163,8 @@ Any item that was inserted using the [autodoc syntax](#autodoc-syntax) cross-reference syntax (`[example][full.path.object1]`). But the cross-references are also applicable to the items' children that get pulled in. +#### Finding out the anchor + If you're not sure which exact identifier a doc item uses, you can look at its "anchor", which your Web browser will show in the URL bar when clicking an item's entry in the table of contents. If the URL is `https://example.com/some/page.html#full.path.object1` then you know that this item @@ -170,6 +172,9 @@ is possible to link to with `[example][full.path.object1]`, regardless of the cu ### Cross-references to any Markdown heading +!!! important "Changed in version 0.15" + Linking to any Markdown heading used to be the default, but now opt-in is required. + If you want to link to *any* Markdown heading, not just *mkdocstrings*-inserted items, please enable the [*autorefs* plugin for *MkDocs*](https://github.com/mkdocstrings/autorefs) by adding `autorefs` to `plugins`: @@ -184,8 +189,6 @@ enable the [*autorefs* plugin for *MkDocs*](https://github.com/mkdocstrings/auto [...] ``` -Prior to *mkdocstrings* version 0.15 this was the default, but now opt-in is required. - Note that you don't need to (`pip`) install anything more; this plugin is guaranteed to be pulled in with *mkdocstrings*. @@ -209,6 +212,44 @@ Note that you don't need to (`pip`) install anything more; this plugin is guaran <p>Please see the <a href="doc1.html#hello-world">Hello, World!</a> section.</p> ``` +### Cross-references to a sub-heading in a docstring + +!!! important "New in version 0.14" + +If you have a Markdown heading *inside* your docstring, you can also link directly to it. +In the example below you see the identifier to be linked is `foo.bar--tips`, because it's the "Tips" heading that's part of the `foo.bar` object, joined with "`--`". + +!!! example + === "foo.py" + ```python + def bar(): + """Hello, world! + + # Tips + + - Stay hydrated. + """ + ``` + + === "doc1.md" + ```md + ::: foo.bar + ``` + + === "doc2.md" + ```md + Check out the [tips][foo.bar--tips] + ``` + + === "Result HTML for doc2" + ```html + <p>Check out the <a href="doc1.html#foo.bar--tips">tips</a></p> + ``` + +The above tip about [Finding out the anchor](#finding-out-the-anchor) also applies the same way here. + +You may also notice that such a heading does not get rendered as a `<h1>` element directly, but rather the level gets shifted to fit the encompassing document structure. If you're curious about the implementation, check out [mkdocstrings.handlers.rendering.HeadingShiftingTreeprocessor][] and others. + ## Watch directories diff --git a/src/mkdocstrings/handlers/rendering.py b/src/mkdocstrings/handlers/rendering.py index 4cfcee9a..73f249a5 100644 --- a/src/mkdocstrings/handlers/rendering.py +++ b/src/mkdocstrings/handlers/rendering.py @@ -15,7 +15,23 @@ class Highlighter(Highlight): - """Code highlighter that tries to match the Markdown configuration.""" + """Code highlighter that tries to match the Markdown configuration. + + Picking up the global config and defaults works only if you use the `codehilite` or + `pymdownx.highlight` (recommended) Markdown extension. + + - If you use `pymdownx.highlight`, highlighting settings are picked up from it, and the + default CSS class is `.highlight`. This also means the default of `guess_lang: false`. + + - Otherwise, if you use the `codehilite` extension, settings are picked up from it, and the + default CSS class is `.codehilite`. Also consider setting `guess_lang: false`. + + - If neither are added to `markdown_extensions`, highlighting is enabled anyway. This is for + backwards compatibility. If you really want to disable highlighting even in *mkdocstrings*, + add one of these extensions anyway and set `use_pygments: false`. + + The underlying implementation is `pymdownx.highlight` regardless. + """ _highlight_config_keys = frozenset( "use_pygments guess_lang css_class pygments_style noclasses linenums language_prefix".split(), @@ -41,7 +57,7 @@ def __init__(self, md: Markdown): def highlight( # noqa: W0221 (intentionally different params, we're extending the functionality) self, src: str, - language: str = None, + language: Optional[str] = None, *, inline: bool = False, dedent: bool = True, From c7d80e63a042913b7511c38a788967796dd10997 Mon Sep 17 00:00:00 2001 From: Oleh Prypin <oleh@pryp.in> Date: Wed, 24 Feb 2021 21:56:41 +0100 Subject: [PATCH 35/40] fix: Propagate the CSS class to inline highlighting as well --- src/mkdocstrings/handlers/rendering.py | 2 +- tests/test_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mkdocstrings/handlers/rendering.py b/src/mkdocstrings/handlers/rendering.py index 4cfcee9a..0334e3a5 100644 --- a/src/mkdocstrings/handlers/rendering.py +++ b/src/mkdocstrings/handlers/rendering.py @@ -77,7 +77,7 @@ def highlight( # noqa: W0221 (intentionally different params, we're extending t self.linenums = old_linenums if inline: - return Markup(f'<code class="highlight language-{language}">{result.text}</code>') + return Markup(f'<code class="{kwargs["css_class"]} language-{language}">{result.text}</code>') return Markup(result) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 0d0c1b40..3e225cff 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -23,7 +23,7 @@ def test_highlighter_without_pygments(extension_name): ) assert ( hl.highlight("import foo", language="python", inline=True) - == '<code class="highlight language-python">import foo</code>' + == '<code class="hiiii language-python">import foo</code>' ) From 83c87d0b568d7ec2293d76022c858e543a23d2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Sat, 27 Feb 2021 23:32:01 +0100 Subject: [PATCH 36/40] refactor: Remove circular import --- src/mkdocstrings/loggers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mkdocstrings/loggers.py b/src/mkdocstrings/loggers.py index 26c7c92f..53d287f3 100644 --- a/src/mkdocstrings/loggers.py +++ b/src/mkdocstrings/loggers.py @@ -8,7 +8,7 @@ from jinja2.runtime import Context from mkdocs.utils import warning_filter -from mkdocstrings.handlers import base +TEMPLATES_DIR = Path(__file__).parent / "templates" class LoggerAdapter(logging.LoggerAdapter): @@ -108,7 +108,7 @@ def get_template_path(context: Context) -> str: filename = context.environment.get_template(context.name).filename if filename: try: - return str(Path(filename).relative_to(base.TEMPLATES_DIR)) + return str(Path(filename).relative_to(TEMPLATES_DIR)) except ValueError: return filename return context.name From c29ba6b6b9babce7f17372e7e5b3a1299814e6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Sun, 28 Feb 2021 10:31:19 +0100 Subject: [PATCH 37/40] style: Normalize and enforce docstring style The overall convention checking was lost in translation in #236 and this linting decision is new so we also drop that ignore line. Co-authored-by: Oleh Prypin <oleh@pryp.in> --- config/flake8.ini | 3 +- docs/gen_credits.py | 6 +- docs/snippets/function_annotations_google.py | 3 +- docs/snippets/function_annotations_rst.py | 3 +- docs/troubleshooting.md | 3 +- duties.py | 57 ++++++----------- src/mkdocstrings/extension.py | 30 +++------ src/mkdocstrings/handlers/base.py | 66 +++++++------------- src/mkdocstrings/handlers/python.py | 27 +++----- src/mkdocstrings/handlers/rendering.py | 3 +- src/mkdocstrings/loggers.py | 27 +++----- src/mkdocstrings/plugin.py | 21 +++---- tests/test_handlers.py | 6 +- 13 files changed, 85 insertions(+), 170 deletions(-) diff --git a/config/flake8.ini b/config/flake8.ini index 8ca67f33..3e559fd2 100644 --- a/config/flake8.ini +++ b/config/flake8.ini @@ -2,6 +2,7 @@ exclude = fixtures,docs,site max-line-length = 132 strictness = long +docstring-convention = google ignore = # we write docstrings in markdown, not rst RST*, @@ -9,8 +10,6 @@ ignore = A001, # missing docstring in magic method D105, - # multi-line docstring summary should start at the first line - D212, # whitespace before ‘:’ (incompatible with Black) E203, # redundant with E0602 (undefined variable) diff --git a/docs/gen_credits.py b/docs/gen_credits.py index 8bf2130f..d626e220 100644 --- a/docs/gen_credits.py +++ b/docs/gen_credits.py @@ -11,8 +11,7 @@ def get_credits_data() -> dict: - """ - Return data used to generate the credits file. + """Return data used to generate the credits file. Returns: Data required to render the credits template. @@ -50,8 +49,7 @@ def get_credits_data() -> dict: @functools.lru_cache(maxsize=None) def get_credits(): - """ - Return credits as Markdown. + """Return credits as Markdown. Returns: The credits page Markdown. diff --git a/docs/snippets/function_annotations_google.py b/docs/snippets/function_annotations_google.py index 73b00219..f3b20897 100644 --- a/docs/snippets/function_annotations_google.py +++ b/docs/snippets/function_annotations_google.py @@ -2,8 +2,7 @@ def my_function(param1: int, param2: Optional[str] = None) -> str: - """ - A short description of this function. + """A short description of this function. Arguments: param1: An integer? diff --git a/docs/snippets/function_annotations_rst.py b/docs/snippets/function_annotations_rst.py index 8fd79d62..6a802f74 100644 --- a/docs/snippets/function_annotations_rst.py +++ b/docs/snippets/function_annotations_rst.py @@ -2,8 +2,7 @@ def my_function(param1: int, param2: Optional[str] = None) -> str: - """ - A short description of this function. + """A short description of this function. Complex markup is supported in the main description section. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 25ed85dd..20e88195 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -214,8 +214,7 @@ You can use: import enum class MyEnum(enum.Enum): - """ - My enum. + """My enum. Attributes: v1: The first choice. diff --git a/duties.py b/duties.py index e93231be..16aa7bcb 100644 --- a/duties.py +++ b/duties.py @@ -20,8 +20,7 @@ def latest(lines: List[str], regex: Pattern) -> Optional[str]: - """ - Return the last released version. + """Return the last released version. Arguments: lines: Lines of the changelog file. @@ -38,8 +37,7 @@ def latest(lines: List[str], regex: Pattern) -> Optional[str]: def unreleased(versions: List[Version], last_release: str) -> List[Version]: - """ - Return the most recent versions down to latest release. + """Return the most recent versions down to latest release. Arguments: versions: All the versions (released and unreleased). @@ -55,8 +53,7 @@ def unreleased(versions: List[Version], last_release: str) -> List[Version]: def read_changelog(filepath: str) -> List[str]: - """ - Read the changelog file. + """Read the changelog file. Arguments: filepath: The path to the changelog file. @@ -69,8 +66,7 @@ def read_changelog(filepath: str) -> List[str]: def write_changelog(filepath: str, lines: List[str]) -> None: - """ - Write the changelog file. + """Write the changelog file. Arguments: filepath: The path to the changelog file. @@ -87,8 +83,7 @@ def update_changelog( template_url: str, commit_style: str, ) -> None: - """ - Update the given changelog file in place. + """Update the given changelog file in place. Arguments: inplace_file: The file to update in-place. @@ -120,8 +115,7 @@ def update_changelog( @duty def changelog(ctx): - """ - Update the changelog in-place with latest commits. + """Update the changelog in-place with latest commits. Arguments: ctx: The context instance (passed automatically). @@ -142,8 +136,7 @@ def changelog(ctx): @duty(pre=["check_code_quality", "check_types", "check_docs", "check_dependencies"]) def check(ctx): # noqa: W0613 (no use for the context argument) - """ - Check it all! + """Check it all! Arguments: ctx: The context instance (passed automatically). @@ -152,8 +145,7 @@ def check(ctx): # noqa: W0613 (no use for the context argument) @duty def check_code_quality(ctx, files=PY_SRC): - """ - Check the code quality. + """Check the code quality. Arguments: ctx: The context instance (passed automatically). @@ -164,8 +156,7 @@ def check_code_quality(ctx, files=PY_SRC): @duty def check_dependencies(ctx): - """ - Check for vulnerabilities in dependencies. + """Check for vulnerabilities in dependencies. Arguments: ctx: The context instance (passed automatically). @@ -195,8 +186,7 @@ def check_dependencies(ctx): @duty def check_docs(ctx): - """ - Check if the documentation builds correctly. + """Check if the documentation builds correctly. Arguments: ctx: The context instance (passed automatically). @@ -208,8 +198,7 @@ def check_docs(ctx): @duty def check_types(ctx): - """ - Check that the code is correctly typed. + """Check that the code is correctly typed. Arguments: ctx: The context instance (passed automatically). @@ -219,8 +208,7 @@ def check_types(ctx): @duty(silent=True) def clean(ctx): - """ - Delete temporary files. + """Delete temporary files. Arguments: ctx: The context instance (passed automatically). @@ -238,8 +226,7 @@ def clean(ctx): @duty def docs(ctx): - """ - Build the documentation locally. + """Build the documentation locally. Arguments: ctx: The context instance (passed automatically). @@ -249,8 +236,7 @@ def docs(ctx): @duty def docs_serve(ctx, host="127.0.0.1", port=8000): - """ - Serve the documentation (localhost:8000). + """Serve the documentation (localhost:8000). Arguments: ctx: The context instance (passed automatically). @@ -262,8 +248,7 @@ def docs_serve(ctx, host="127.0.0.1", port=8000): @duty def docs_deploy(ctx): - """ - Deploy the documentation on GitHub pages. + """Deploy the documentation on GitHub pages. Arguments: ctx: The context instance (passed automatically). @@ -274,8 +259,7 @@ def docs_deploy(ctx): @duty def format(ctx): # noqa: W0622 (we don't mind shadowing the format builtin) - """ - Run formatting tools on the code. + """Run formatting tools on the code. Arguments: ctx: The context instance (passed automatically). @@ -291,8 +275,7 @@ def format(ctx): # noqa: W0622 (we don't mind shadowing the format builtin) @duty def release(ctx, version): - """ - Release a new Python package. + """Release a new Python package. Arguments: ctx: The context instance (passed automatically). @@ -312,8 +295,7 @@ def release(ctx, version): @duty(silent=True) def coverage(ctx): - """ - Report coverage as text and HTML. + """Report coverage as text and HTML. Arguments: ctx: The context instance (passed automatically). @@ -324,8 +306,7 @@ def coverage(ctx): @duty def test(ctx, cleancov: bool = True, match: str = ""): - """ - Run the test suite. + """Run the test suite. Arguments: ctx: The context instance (passed automatically). diff --git a/src/mkdocstrings/extension.py b/src/mkdocstrings/extension.py index 6bb4cfd3..c9f42e6d 100644 --- a/src/mkdocstrings/extension.py +++ b/src/mkdocstrings/extension.py @@ -1,5 +1,4 @@ -""" -This module holds the code of the Markdown extension responsible for matching "autodoc" instructions. +"""This module holds the code of the Markdown extension responsible for matching "autodoc" instructions. The extension is composed of a Markdown [block processor](https://python-markdown.github.io/extensions/api/#blockparser) that matches indented blocks starting with a line like '::: identifier'. @@ -49,8 +48,7 @@ class AutoDocProcessor(BlockProcessor): - """ - Our "autodoc" Markdown block processor. + """Our "autodoc" Markdown block processor. It has a [`test` method][mkdocstrings.extension.AutoDocProcessor.test] that tells if a block matches a criterion, and a [`run` method][mkdocstrings.extension.AutoDocProcessor.run] that processes it. @@ -64,8 +62,7 @@ class AutoDocProcessor(BlockProcessor): def __init__( self, parser: BlockParser, md: Markdown, config: dict, handlers: Handlers, autorefs: AutorefsPlugin ) -> None: - """ - Initialize the object. + """Initialize the object. Arguments: parser: A `markdown.blockparser.BlockParser` instance. @@ -83,8 +80,7 @@ def __init__( self._updated_env = False def test(self, parent: Element, block: str) -> bool: - """ - Match our autodoc instructions. + """Match our autodoc instructions. Arguments: parent: The parent element in the XML tree. @@ -96,8 +92,7 @@ def test(self, parent: Element, block: str) -> bool: return bool(self.regex.search(block)) def run(self, parent: Element, blocks: MutableSequence[str]) -> None: - """ - Run code on the matched blocks. + """Run code on the matched blocks. The identifier and configuration lines are retrieved from a matched block and used to collect and render an object. @@ -141,8 +136,7 @@ def run(self, parent: Element, blocks: MutableSequence[str]) -> None: blocks.insert(0, the_rest) def _process_block(self, identifier: str, yaml_block: str, heading_level: int = 0) -> Tuple[str, Sequence[Element]]: - """ - Process an autodoc block. + """Process an autodoc block. Arguments: identifier: The identifier of the object to collect and render. @@ -195,8 +189,7 @@ def _process_block(self, identifier: str, yaml_block: str, heading_level: int = def get_item_configs(handler_config: dict, config: dict) -> Tuple[Mapping, Mapping]: - """ - Get the selection and rendering configuration merged into the global configuration of the given handler. + """Get the selection and rendering configuration merged into the global configuration of the given handler. Arguments: handler_config: The global configuration of a handler. It can be an empty dictionary. @@ -226,15 +219,13 @@ def run(self, root: Element): class MkdocstringsExtension(Extension): - """ - Our Markdown extension. + """Our Markdown extension. It cannot work outside of `mkdocstrings`. """ def __init__(self, config: dict, handlers: Handlers, autorefs: AutorefsPlugin, **kwargs) -> None: - """ - Initialize the object. + """Initialize the object. Arguments: config: The configuration items from `mkdocs` and `mkdocstrings` that must be passed to the block processor @@ -249,8 +240,7 @@ def __init__(self, config: dict, handlers: Handlers, autorefs: AutorefsPlugin, * self._autorefs = autorefs def extendMarkdown(self, md: Markdown) -> None: # noqa: N802 (casing: parent method's name) - """ - Register the extension. + """Register the extension. Add an instance of our [`AutoDocProcessor`][mkdocstrings.extension.AutoDocProcessor] to the Markdown parser. diff --git a/src/mkdocstrings/handlers/base.py b/src/mkdocstrings/handlers/base.py index 6523bbb7..2a95e775 100644 --- a/src/mkdocstrings/handlers/base.py +++ b/src/mkdocstrings/handlers/base.py @@ -1,5 +1,4 @@ -""" -Base module for handlers. +"""Base module for handlers. This module contains the base classes for implementing collectors, renderers, and the combination of the two: handlers. @@ -41,8 +40,7 @@ class ThemeNotSupported(Exception): def do_any(seq: Sequence, attribute: str = None) -> bool: - """ - Check if at least one of the item in the sequence evaluates to true. + """Check if at least one of the item in the sequence evaluates to true. The `any` builtin as a filter for Jinja templates. @@ -59,8 +57,7 @@ def do_any(seq: Sequence, attribute: str = None) -> bool: class BaseRenderer(ABC): - """ - The base renderer class. + """The base renderer class. Inherit from this class to implement a renderer. @@ -76,8 +73,7 @@ class BaseRenderer(ABC): extra_css = "" def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = None) -> None: - """ - Initialize the object. + """Initialize the object. If the given theme is not supported (it does not exist), it will look for a `fallback_theme` attribute in `self` to use as a fallback theme. @@ -118,8 +114,7 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = @abstractmethod def render(self, data: CollectorItem, config: dict) -> str: - """ - Render a template using provided data and configuration options. + """Render a template using provided data and configuration options. Arguments: data: The collected data to render. @@ -130,8 +125,7 @@ def render(self, data: CollectorItem, config: dict) -> str: """ # noqa: DAR202 (excess return section) def get_anchor(self, data: CollectorItem) -> Optional[str]: - """ - Return the canonical identifier (HTML anchor) for a collected item. + """Return the canonical identifier (HTML anchor) for a collected item. This must match what the renderer would've actually rendered, e.g. if rendering the item contains `<h2 id="foo">...` then the return value should be "foo". @@ -144,8 +138,7 @@ def get_anchor(self, data: CollectorItem) -> Optional[str]: """ # noqa: DAR202 (excess return section) def do_convert_markdown(self, text: str, heading_level: int, html_id: str = "") -> Markup: - """ - Render Markdown text; for use inside templates. + """Render Markdown text; for use inside templates. Arguments: text: The text to convert. @@ -174,8 +167,7 @@ def do_heading( toc_label: Optional[str] = None, **attributes: str, ) -> Markup: - """ - Render an HTML heading and register it for the table of contents. For use inside templates. + """Render an HTML heading and register it for the table of contents. For use inside templates. Arguments: content: The HTML within the heading. @@ -219,8 +211,7 @@ def do_heading( return Markup(html) def get_headings(self) -> Sequence[Element]: - """ - Return and clear the headings gathered so far. + """Return and clear the headings gathered so far. Returns: A list of HTML elements. @@ -230,8 +221,7 @@ def get_headings(self) -> Sequence[Element]: return result def update_env(self, md: Markdown, config: dict) -> None: # noqa: W0613 (unused argument 'config') - """ - Update the Jinja environment. + """Update the Jinja environment. Arguments: md: The Markdown instance. Useful to add functions able to convert Markdown into the environment filters. @@ -255,8 +245,7 @@ def _update_env(self, md: Markdown, config: dict): class BaseCollector(ABC): - """ - The base collector class. + """The base collector class. Inherit from this class to implement a collector. @@ -266,8 +255,7 @@ class BaseCollector(ABC): @abstractmethod def collect(self, identifier: str, config: dict) -> CollectorItem: - """ - Collect data given an identifier and selection configuration. + """Collect data given an identifier and selection configuration. In the implementation, you typically call a subprocess that returns JSON, and load that JSON again into a Python dictionary for example, though the implementation is completely free. @@ -284,8 +272,7 @@ def collect(self, identifier: str, config: dict) -> CollectorItem: """ # noqa: DAR202 (excess return section) def teardown(self) -> None: - """ - Teardown the collector. + """Teardown the collector. This method should be implemented to, for example, terminate a subprocess that was started when creating the collector instance. @@ -293,8 +280,7 @@ def teardown(self) -> None: class BaseHandler: - """ - The base handler class. + """The base handler class. Inherit from this class to implement a handler. @@ -302,8 +288,7 @@ class BaseHandler: """ def __init__(self, collector: BaseCollector, renderer: BaseRenderer) -> None: - """ - Initialize the object. + """Initialize the object. Arguments: collector: A collector instance. @@ -314,16 +299,14 @@ def __init__(self, collector: BaseCollector, renderer: BaseRenderer) -> None: class Handlers: - """ - A collection of handlers. + """A collection of handlers. Do not instantiate this directly. [The plugin][mkdocstrings.plugin.MkdocstringsPlugin] will keep one instance of this for the purpose of caching. Use [mkdocstrings.plugin.MkdocstringsPlugin.get_handler][] for convenient access. """ def __init__(self, config: dict) -> None: - """ - Initialize the object. + """Initialize the object. Arguments: config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code @@ -333,8 +316,7 @@ def __init__(self, config: dict) -> None: self._handlers: Dict[str, BaseHandler] = {} def get_anchor(self, identifier: str) -> Optional[str]: - """ - Return the canonical HTML anchor for the identifier, if any of the seen handlers can collect it. + """Return the canonical HTML anchor for the identifier, if any of the seen handlers can collect it. Arguments: identifier: The identifier (one that [collect][mkdocstrings.handlers.base.BaseCollector.collect] can accept). @@ -353,8 +335,7 @@ def get_anchor(self, identifier: str) -> Optional[str]: return None def get_handler_name(self, config: dict) -> str: - """ - Return the handler name defined in an "autodoc" instruction YAML configuration, or the global default handler. + """Return the handler name defined in an "autodoc" instruction YAML configuration, or the global default handler. Arguments: config: A configuration dictionary, obtained from YAML below the "autodoc" instruction. @@ -368,8 +349,7 @@ def get_handler_name(self, config: dict) -> str: return config["default_handler"] def get_handler_config(self, name: str) -> dict: - """ - Return the global configuration of the given handler. + """Return the global configuration of the given handler. Arguments: name: The name of the handler to get the global configuration of. @@ -383,8 +363,7 @@ def get_handler_config(self, name: str) -> dict: return {} def get_handler(self, name: str, handler_config: Optional[dict] = None) -> BaseHandler: - """ - Get a handler thanks to its name. + """Get a handler thanks to its name. This function dynamically imports a module named "mkdocstrings.handlers.NAME", calls its `get_handler` method to get an instance of a handler, and caches it in dictionary. @@ -412,8 +391,7 @@ def get_handler(self, name: str, handler_config: Optional[dict] = None) -> BaseH @property def seen_handlers(self) -> Iterable[BaseHandler]: - """ - Get the handlers that were encountered so far throughout the build. + """Get the handlers that were encountered so far throughout the build. Returns: An iterable of instances of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler] diff --git a/src/mkdocstrings/handlers/python.py b/src/mkdocstrings/handlers/python.py index 75b82db1..a9e63786 100644 --- a/src/mkdocstrings/handlers/python.py +++ b/src/mkdocstrings/handlers/python.py @@ -1,5 +1,4 @@ -""" -This module implements a handler for the Python language. +"""This module implements a handler for the Python language. The handler collects data with [`pytkdocs`](https://github.com/pawamoy/pytkdocs). """ @@ -21,8 +20,7 @@ class PythonRenderer(BaseRenderer): - """ - The class responsible for loading Jinja templates and rendering them. + """The class responsible for loading Jinja templates and rendering them. It defines some configuration options, implements the `render` method, and overrides the `update_env` method of the [`BaseRenderer` class][mkdocstrings.handlers.base.BaseRenderer]. @@ -48,8 +46,7 @@ class PythonRenderer(BaseRenderer): "group_by_category": True, "heading_level": 2, } - """ - The default rendering options. + """The default rendering options. Option | Type | Description | Default ------ | ---- | ----------- | ------- @@ -91,16 +88,14 @@ def update_env(self, md: Markdown, config: dict) -> None: # noqa: D102 (ignore class PythonCollector(BaseCollector): - """ - The class responsible for loading Jinja templates and rendering them. + """The class responsible for loading Jinja templates and rendering them. It defines some configuration options, implements the `render` method, and overrides the `update_env` method of the [`BaseRenderer` class][mkdocstrings.handlers.base.BaseRenderer]. """ default_config: dict = {"filters": ["!^_[^_]"]} - """ - The default selection options. + """The default selection options. Option | Type | Description | Default ------ | ---- | ----------- | ------- @@ -123,8 +118,7 @@ class PythonCollector(BaseCollector): """ def __init__(self, setup_commands: Optional[List[str]] = None) -> None: - """ - Initialize the object. + """Initialize the object. When instantiating a Python collector, we open a subprocess in the background with `subprocess.Popen`. It will allow us to feed input to and read output from this subprocess, keeping it alive during @@ -166,8 +160,7 @@ def __init__(self, setup_commands: Optional[List[str]] = None) -> None: ) def collect(self, identifier: str, config: dict) -> CollectorItem: - """ - Collect the documentation tree given an identifier and selection options. + """Collect the documentation tree given an identifier and selection options. In this method, we feed one line of JSON to the standard input of the subprocess that was opened during instantiation of the collector. Then we read one line of JSON on its standard output. @@ -252,8 +245,7 @@ def get_handler( setup_commands: Optional[List[str]] = None, **config: Any, ) -> PythonHandler: - """ - Simply return an instance of `PythonHandler`. + """Simply return an instance of `PythonHandler`. Arguments: theme: The theme to use when rendering contents. @@ -271,8 +263,7 @@ def get_handler( def rebuild_category_lists(obj: dict) -> None: - """ - Recursively rebuild the category lists of a collected object. + """Recursively rebuild the category lists of a collected object. Since `pytkdocs` dumps JSON on standard output, it must serialize the object-tree and flatten it to reduce data duplication and avoid cycle-references. Indeed, each node of the object-tree has a `children` list, containing diff --git a/src/mkdocstrings/handlers/rendering.py b/src/mkdocstrings/handlers/rendering.py index 40cf4349..ed9049ba 100644 --- a/src/mkdocstrings/handlers/rendering.py +++ b/src/mkdocstrings/handlers/rendering.py @@ -64,8 +64,7 @@ def highlight( # noqa: W0221 (intentionally different params, we're extending t linenums: Optional[bool] = None, **kwargs, ) -> str: - """ - Highlight a code-snippet. + """Highlight a code-snippet. Arguments: src: The code to highlight. diff --git a/src/mkdocstrings/loggers.py b/src/mkdocstrings/loggers.py index 53d287f3..1b1a5dff 100644 --- a/src/mkdocstrings/loggers.py +++ b/src/mkdocstrings/loggers.py @@ -15,8 +15,7 @@ class LoggerAdapter(logging.LoggerAdapter): """A logger adapter to prefix messages.""" def __init__(self, prefix: str, logger): - """ - Initialize the object. + """Initialize the object. Arguments: prefix: The string to insert in front of every message. @@ -26,8 +25,7 @@ def __init__(self, prefix: str, logger): self.prefix = prefix def process(self, msg: str, kwargs) -> Tuple[str, Any]: - """ - Process the message. + """Process the message. Arguments: msg: The message: @@ -40,8 +38,7 @@ def process(self, msg: str, kwargs) -> Tuple[str, Any]: class TemplateLogger: - """ - A wrapper class to allow logging in templates. + """A wrapper class to allow logging in templates. Attributes: debug: Function to log a DEBUG message. @@ -52,8 +49,7 @@ class TemplateLogger: """ def __init__(self, logger: LoggerAdapter): - """ - Initialize the object. + """Initialize the object. Arguments: logger: A logger adapter. @@ -66,8 +62,7 @@ def __init__(self, logger: LoggerAdapter): def get_template_logger_function(logger_func: Callable) -> Callable: - """ - Create a wrapper function that automatically receives the Jinja template context. + """Create a wrapper function that automatically receives the Jinja template context. Arguments: logger_func: The logger function to use within the wrapper. @@ -78,8 +73,7 @@ def get_template_logger_function(logger_func: Callable) -> Callable: @contextfunction def wrapper(context: Context, msg: Optional[str] = None) -> str: - """ - Log a message. + """Log a message. Arguments: context: The template context, automatically provided by Jinja. @@ -96,8 +90,7 @@ def wrapper(context: Context, msg: Optional[str] = None) -> str: def get_template_path(context: Context) -> str: - """ - Return the path to the template currently using the given context. + """Return the path to the template currently using the given context. Arguments: context: The template context. @@ -115,8 +108,7 @@ def get_template_path(context: Context) -> str: def get_logger(name: str) -> LoggerAdapter: - """ - Return a pre-configured logger. + """Return a pre-configured logger. Arguments: name: The name to use with `logging.getLogger`. @@ -130,8 +122,7 @@ def get_logger(name: str) -> LoggerAdapter: def get_template_logger() -> TemplateLogger: - """ - Return a logger usable in templates. + """Return a logger usable in templates. Returns: A template logger. diff --git a/src/mkdocstrings/plugin.py b/src/mkdocstrings/plugin.py index 921bbc84..054cd2e0 100644 --- a/src/mkdocstrings/plugin.py +++ b/src/mkdocstrings/plugin.py @@ -1,5 +1,4 @@ -""" -This module contains the "mkdocstrings" plugin for MkDocs. +"""This module contains the "mkdocstrings" plugin for MkDocs. The plugin instantiates a Markdown extension ([`MkdocstringsExtension`][mkdocstrings.extension.MkdocstringsExtension]), and adds it to the list of Markdown extensions used by `mkdocs` @@ -36,8 +35,7 @@ class MkdocstringsPlugin(BasePlugin): - """ - An `mkdocs` plugin. + """An `mkdocs` plugin. This plugin defines the following event hooks: @@ -95,8 +93,7 @@ def __init__(self) -> None: @property def handlers(self) -> Handlers: - """ - Get the instance of [mkdocstrings.handlers.base.Handlers][] for this plugin/build. + """Get the instance of [mkdocstrings.handlers.base.Handlers][] for this plugin/build. Raises: RuntimeError: If the plugin hasn't been initialized with a config. @@ -109,8 +106,7 @@ def handlers(self) -> Handlers: return self._handlers def on_serve(self, server: Server, builder: Callable = None, **kwargs) -> Server: # noqa: W0613 (unused arguments) - """ - Watch directories. + """Watch directories. Hook for the [`on_serve` event](https://www.mkdocs.org/user-guide/plugins/#on_serve). In this hook, we add the directories specified in the plugin's configuration to the list of directories @@ -135,8 +131,7 @@ def on_serve(self, server: Server, builder: Callable = None, **kwargs) -> Server return server def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused arguments) - """ - Instantiate our Markdown extension. + """Instantiate our Markdown extension. Hook for the [`on_config` event](https://www.mkdocs.org/user-guide/plugins/#on_config). In this hook, we instantiate our [`MkdocstringsExtension`][mkdocstrings.extension.MkdocstringsExtension] @@ -189,8 +184,7 @@ def on_config(self, config: Config, **kwargs) -> Config: # noqa: W0613 (unused return config def on_post_build(self, config: Config, **kwargs) -> None: # noqa: W0613,R0201 (unused arguments, cannot be static) - """ - Teardown the handlers. + """Teardown the handlers. Hook for the [`on_post_build` event](https://www.mkdocs.org/user-guide/plugins/#on_post_build). This hook is used to teardown all the handlers that were instantiated and cached during documentation buildup. @@ -213,8 +207,7 @@ def on_post_build(self, config: Config, **kwargs) -> None: # noqa: W0613,R0201 self._handlers.teardown() def get_handler(self, handler_name: str) -> BaseHandler: - """ - Get a handler by its name. See [mkdocstrings.handlers.base.Handlers.get_handler][]. + """Get a handler by its name. See [mkdocstrings.handlers.base.Handlers.get_handler][]. Arguments: handler_name: The name of the handler. diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 3e225cff..cfe04cd8 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -8,8 +8,7 @@ @pytest.mark.parametrize("extension_name", ["codehilite", "pymdownx.highlight"]) def test_highlighter_without_pygments(extension_name): - """ - Assert that it's possible to disable Pygments highlighting. + """Assert that it's possible to disable Pygments highlighting. Arguments: extension_name: The "user-chosen" Markdown extension for syntax highlighting. @@ -30,8 +29,7 @@ def test_highlighter_without_pygments(extension_name): @pytest.mark.parametrize("extension_name", [None, "codehilite", "pymdownx.highlight"]) @pytest.mark.parametrize("inline", [False, True]) def test_highlighter_basic(extension_name, inline): - """ - Assert that Pygments syntax highlighting works. + """Assert that Pygments syntax highlighting works. Arguments: extension_name: The "user-chosen" Markdown extension for syntax highlighting. From d5a1e390db3037fbb62f65a27a339592daead767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Sun, 28 Feb 2021 10:33:10 +0100 Subject: [PATCH 38/40] ci: Don't fail when checking docs on Python 3.6 Co-authored-by: Oleh Prypin <oleh@pryp.in> PR 248: https://github.com/mkdocstrings/mkdocstrings/pull/248 --- duties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/duties.py b/duties.py index 16aa7bcb..be0f8d90 100644 --- a/duties.py +++ b/duties.py @@ -191,8 +191,8 @@ def check_docs(ctx): Arguments: ctx: The context instance (passed automatically). """ - # pytkdocs fails on Python 3.9 for now - nofail = sys.version.startswith("3.9") + # mkdocs-gen-files works on 3.7+ only + nofail = sys.version_info < (3, 7) ctx.run("mkdocs build -s", title="Building documentation", pty=PTY, nofail=nofail, quiet=nofail) From fc9fe340a4274db86e9290541d52d39a23d78213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Sun, 28 Feb 2021 13:04:09 +0100 Subject: [PATCH 39/40] pkg: Accept pytkdocs v0.11 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 524f491d..d5ac1f30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ MarkupSafe = "^1.1" mkdocs = "^1.1" mkdocs-autorefs = "^0.1" pymdown-extensions = ">=6.3, <9.0" -pytkdocs = ">=0.2.0, <0.11.0" +pytkdocs = ">=0.2.0, <0.12.0" [tool.poetry.dev-dependencies] autoflake = "^1.4" From 65732a2a6fee368a3d82ab54212c8deb6b7f2aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= <pawamoy@pm.me> Date: Sun, 28 Feb 2021 13:07:58 +0100 Subject: [PATCH 40/40] chore: Prepare release 0.15.0 --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 258115b2..b60b3fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,41 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). <!-- insertion marker --> +## [0.15.0](https://github.com/mkdocstrings/mkdocstrings/releases/tag/0.15.0) - 2021-02-28 + +<small>[Compare with 0.14.0](https://github.com/mkdocstrings/mkdocstrings/compare/0.14.0...0.15.0)</small> + +### Breaking Changes + +The following two items are *possible* breaking changes: + +- Cross-linking to arbitrary headings now requires to opt-in to the *autorefs* plugin, + which is installed as a dependency of *mkdocstrings*. + See [Cross-references to any Markdown heading](https://mkdocstrings.github.io/usage/#cross-references-to-any-markdown-heading). +- *mkdocstrings* now respects your code highlighting configured method, + so if you are using CodeHilite, the `highlight` CSS classes in the rendered HTML will be replaced by `codehilite`. + In that case make sure to replace `.highlight` by `.codehilite` in any extra CSS rule of yours. + See [Syntax highlighting](https://mkdocstrings.github.io/theming/#syntax-highlighting). + +### Features +- Nicer-looking error outputs - no tracebacks from mkdocstrings ([6baf720](https://github.com/mkdocstrings/mkdocstrings/commit/6baf720850d359ddb55713553a757fe7b2283e10) by Oleh Prypin). [PR #230](https://github.com/mkdocstrings/mkdocstrings/pull/230) +- Let handlers add CSS to the pages, do so for Python handler ([05c7a3f](https://github.com/mkdocstrings/mkdocstrings/commit/05c7a3fc83b67d3244ea3bfe97dab19aa53f2d38) by Oleh Prypin). [Issue #189](https://github.com/mkdocstrings/mkdocstrings/issues/189), [PR #218](https://github.com/mkdocstrings/mkdocstrings/pull/218) +- Allow linking to an object heading not only by its canonical identifier, but also by its possible aliases ([4789950](https://github.com/mkdocstrings/mkdocstrings/commit/4789950ff43c354d47afbed5c89d5abb917ffee6) by Oleh Prypin). [PR #217](https://github.com/mkdocstrings/mkdocstrings/pull/217) + +### Bug Fixes +- Propagate the CSS class to inline highlighting as well ([c7d80e6](https://github.com/mkdocstrings/mkdocstrings/commit/c7d80e63a042913b7511c38a788967796dd10997) by Oleh Prypin). [PR #245](https://github.com/mkdocstrings/mkdocstrings/pull/245) +- Don't double-escape characters in highlighted headings ([6357144](https://github.com/mkdocstrings/mkdocstrings/commit/6357144b100be6a2e7e6140e035c289c225cec22) by Oleh Prypin). [Issue #228](https://github.com/mkdocstrings/mkdocstrings/issues/228), [PR #241](https://github.com/mkdocstrings/mkdocstrings/pull/241) + +### Code Refactoring +- Use the autorefs plugin from its new external location ([e2d74ef](https://github.com/mkdocstrings/mkdocstrings/commit/e2d74efb0d59f9a1aa45e42525ceb1d4b7638426) by Oleh Prypin). [PR #235](https://github.com/mkdocstrings/mkdocstrings/pull/235) +- Split out Markdown extensions from `handlers` to `handlers.rendering` ([7533852](https://github.com/mkdocstrings/mkdocstrings/commit/7533852e3ac0a378b70a380cef1100421b7d5763) by Oleh Prypin). [PR #233](https://github.com/mkdocstrings/mkdocstrings/pull/233) +- Theme-agnostic code highlighting, respecting configs ([f9ea009](https://github.com/mkdocstrings/mkdocstrings/commit/f9ea00979545e39983ba377f1930d73ae94165ea) by Oleh Prypin). [PR #202](https://github.com/mkdocstrings/mkdocstrings/pull/202) +- Split out autorefs plugin, make it optional ([fc67656](https://github.com/mkdocstrings/mkdocstrings/commit/fc676564f9b11269b3e0b0482703ac924069a3fa) by Oleh Prypin). [PR #220](https://github.com/mkdocstrings/mkdocstrings/pull/220) +- Remove the extra wrapper div from the final doc ([7fe438c](https://github.com/mkdocstrings/mkdocstrings/commit/7fe438c4040a2124b00c39e582ef4c38be7c55c9) by Oleh Prypin). [PR #209](https://github.com/mkdocstrings/mkdocstrings/pull/209) +- Don't re-parse the whole subdoc, expose only headings ([15f84f9](https://github.com/mkdocstrings/mkdocstrings/commit/15f84f981982c8e2b15498f5c869ac207f3ce5d7) by Oleh Prypin). [PR #209](https://github.com/mkdocstrings/mkdocstrings/pull/209) +- Actually exclude hidden headings from the doc ([0fdb082](https://github.com/mkdocstrings/mkdocstrings/commit/0fdb0821867eb0e14a972a603c22301aafecf4f4) by Oleh Prypin). [PR #209](https://github.com/mkdocstrings/mkdocstrings/pull/209) + + ## [0.14.0](https://github.com/pawamoy/mkdocstrings/releases/tag/0.14.0) - 2021-01-06 <small>[Compare with 0.13.6](https://github.com/pawamoy/mkdocstrings/compare/0.13.6...0.14.0)</small> diff --git a/pyproject.toml b/pyproject.toml index d5ac1f30..cf5c7286 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "mkdocstrings" -version = "0.14.0" +version = "0.15.0" description = "Automatic documentation from sources, for MkDocs." authors = ["Timothée Mazzucotelli <pawamoy@pm.me>"] license = "ISC License"