Support luatex as alternative usetex engine.

anntzer · anntzer · commit 3866b0ebc13e · 2025-03-26T00:01:14.000+01:00
Currently, this PR is mostly a proof of concept; it only implements the
dvi generation and parsing parts, but does not implement rendering in
any of the builtin backends.  However, there is a companion branch on
the mplcairo repository, also named "luadvi", which implements support.

Example (requiring both this PR, and mplcairo installed from its luadvi
branch):
```
import matplotlib as mpl; mpl.use("module://mplcairo.qt")
from matplotlib import pyplot as plt

plt.rcParams["text.latex.engine"] = "lualatex";
plt.rcParams["text.latex.preamble"] = (
    # luatex can use any font installed on the system, spec'd using its
    # "normal" name.
    r"\usepackage{fontspec}\setmainfont{TeX Gyre Pagella}")
plt.figtext(.5, .5, r"\textrm{gff\textwon}", usetex=True)
plt.show()
```

TODO:

- Fix many likely remaining bugs.
- Rework font selection in texmanager, which is currently very ad-hoc
  due to the limited number of fonts supported by latex.
- Implement rendering support in the builtin backends.  In particular
  the Agg (and, if we care, cairo) backend will require significant
  reworking because dvipng, currently used to rasterize dvi to png,
  doesn't support luatex-generated dvi; instead we will need to proceed
  as with the other backends, reading the glyphs one at a time from the
  dvi file and rasterizing them one at a time to the output buffer.
  Working on the other backends is not very high on my priority list (as
  I already have mplcairo as playground...) so it would be nice if
  others showed some interest for it :-)

NOTES:

- xetex could possibly be supported as well, but uses its own (more
  divergent) dvi variant which would require a bit more work...
diff --git a/lib/matplotlib/backends/backend_pdf.py b/lib/matplotlib/backends/backend_pdf.py
@@ -991,7 +991,7 @@ def _embedTeXFont(self, fontinfo):
 
         # Widths
         widthsObject = self.reserveObject('font widths')
-        self.writeObject(widthsObject, fontinfo.dvifont.widths)
+        self.writeObject(widthsObject, fontinfo.dvifont._metrics.widths)
 
         # Font dictionary
         fontdictObject = self.reserveObject('font dictionary')
diff --git a/lib/matplotlib/dviread.py b/lib/matplotlib/dviread.py
@@ -30,7 +30,8 @@
 
 import numpy as np
 
-from matplotlib import _api, cbook
+from matplotlib import _api, cbook, textpath
+from matplotlib.ft2font import FT2Font, LoadFlags
 
 _log = logging.getLogger(__name__)
 
@@ -106,18 +107,29 @@ def font_effects(self):
     @property
     def glyph_name_or_index(self):
         """
-        Either the glyph name or the native charmap glyph index.
-
-        If :file:`pdftex.map` specifies an encoding for this glyph's font, that
-        is a mapping of glyph indices to Adobe glyph names; use it to convert
-        dvi indices to glyph names.  Callers can then convert glyph names to
-        glyph indices (with FT_Get_Name_Index/get_name_index), and load the
-        glyph using FT_Load_Glyph/load_glyph.
-
-        If :file:`pdftex.map` specifies no encoding, the indices directly map
-        to the font's "native" charmap; glyphs should directly load using
-        FT_Load_Char/load_char after selecting the native charmap.
+        The glyph name, the native charmap glyph index, or the raw glyph index.
+
+        If the font is a TrueType file (which can currently only happen for
+        DVI files generated by luatex), then this number is the raw index of
+        the glyph, which can be passed to FT_Load_Glyph/load_glyph.  Note that
+        xetex is currently unsupported and the behavior on xdv files (xetex's
+        version of dvi) is undefined.
+
+        Otherwise, the font is a PostScript font.  For such fonts, if
+        :file:`pdftex.map` specifies an encoding for this glyph's font,
+        that is a mapping of glyph indices to Adobe glyph names; which
+        is used by this property to convert dvi numbers to glyph names.
+        Callers can then convert glyph names to glyph indices (with
+        FT_Get_Name_Index/get_name_index), and load the glyph using
+        FT_Load_Glyph/load_glyph.
+
+        If :file:`pdftex.map` specifies no encoding for a PostScript font,
+        this number is an index to the font's "native" charmap; glyphs should
+        directly load using FT_Load_Char/load_char after selecting the native
+        charmap.
         """
+        # TODO: The last section is only true since luaotfload 3.15; add a
+        # version check in the tex file generated by texmanager.
         entry = self._get_pdftexmap_entry()
         return (_parse_enc(entry.encoding)[self.glyph]
                 if entry.encoding is not None else self.glyph)
@@ -399,7 +411,7 @@ def _put_char_real(self, char):
             scale = font._scale
             for x, y, f, g, w in font._vf[char].text:
                 newf = DviFont(scale=_mul2012(scale, f._scale),
-                               tfm=f._tfm, texname=f.texname, vf=f._vf)
+                               metrics=f._metrics, texname=f.texname, vf=f._vf)
                 self.text.append(Text(self.h + _mul2012(x, scale),
                                       self.v + _mul2012(y, scale),
                                       newf, g, newf._width_of(g)))
@@ -496,6 +508,12 @@ def _fnt_def(self, k, c, s, d, a, l):
     def _fnt_def_real(self, k, c, s, d, a, l):
         n = self.file.read(a + l)
         fontname = n[-l:].decode('ascii')
+        # TODO: Implement full spec, https://tug.org/pipermail/dvipdfmx/2021-January/000168.html
+        # Note that checksum seems wrong?
+        if fontname.startswith('[') and fontname.endswith(']'):
+            metrics = TtfMetrics(fontname[1:-1])
+            self.fonts[k] = DviFont(scale=s, metrics=metrics, texname=n, vf=None)
+            return
         try:
             tfm = _tfmfile(fontname)
         except FileNotFoundError as exc:
@@ -512,7 +530,7 @@ def _fnt_def_real(self, k, c, s, d, a, l):
             vf = _vffile(fontname)
         except FileNotFoundError:
             vf = None
-        self.fonts[k] = DviFont(scale=s, tfm=tfm, texname=n, vf=vf)
+        self.fonts[k] = DviFont(scale=s, metrics=tfm, texname=n, vf=vf)
 
     @_dispatch(247, state=_dvistate.pre, args=('u1', 'u4', 'u4', 'u4', 'u1'))
     def _pre(self, i, num, den, mag, k):
@@ -562,7 +580,7 @@ class DviFont:
     ----------
     scale : float
         Factor by which the font is scaled from its natural size.
-    tfm : Tfm
+    tfm : Tfm | TtfMetrics
         TeX font metrics for this font
     texname : bytes
        Name of the font as used internally by TeX and friends, as an ASCII
@@ -582,21 +600,17 @@ class DviFont:
        the point size.
 
     """
-    __slots__ = ('texname', 'size', 'widths', '_scale', '_vf', '_tfm')
+    __slots__ = ('texname', 'size', '_scale', '_vf', '_metrics')
 
-    def __init__(self, scale, tfm, texname, vf):
+    def __init__(self, scale, metrics, texname, vf):
         _api.check_isinstance(bytes, texname=texname)
         self._scale = scale
-        self._tfm = tfm
+        self._metrics = metrics
         self.texname = texname
         self._vf = vf
         self.size = scale * (72.0 / (72.27 * 2**16))
-        try:
-            nchars = max(tfm.width) + 1
-        except ValueError:
-            nchars = 0
-        self.widths = [(1000*tfm.width.get(char, 0)) >> 20
-                       for char in range(nchars)]
+
+    widths = _api.deprecated("3.11")(property(lambda self: ...))
 
     def __eq__(self, other):
         return (type(self) is type(other)
@@ -610,32 +624,30 @@ def __repr__(self):
 
     def _width_of(self, char):
         """Width of char in dvi units."""
-        width = self._tfm.width.get(char, None)
-        if width is not None:
-            return _mul2012(width, self._scale)
-        _log.debug('No width for char %d in font %s.', char, self.texname)
-        return 0
+        metrics = self._metrics.get_metrics(char)
+        if metrics is None:
+            _log.debug('No width for char %d in font %s.', char, self.texname)
+            return 0
+        return _mul2012(metrics.width, self._scale)
 
     def _height_depth_of(self, char):
         """Height and depth of char in dvi units."""
-        result = []
-        for metric, name in ((self._tfm.height, "height"),
-                             (self._tfm.depth, "depth")):
-            value = metric.get(char, None)
-            if value is None:
-                _log.debug('No %s for char %d in font %s',
-                           name, char, self.texname)
-                result.append(0)
-            else:
-                result.append(_mul2012(value, self._scale))
+        metrics = self._metrics.get_metrics(char)
+        if metrics is None:
+            _log.debug('No metrics for char %d in font %s', char, self.texname)
+            return [0, 0]
+        metrics = [
+            _mul2012(metrics.height, self._scale),
+            _mul2012(metrics.depth, self._scale),
+        ]
         # cmsyXX (symbols font) glyph 0 ("minus") has a nonzero descent
         # so that TeX aligns equations properly
         # (https://tex.stackexchange.com/q/526103/)
         # but we actually care about the rasterization depth to align
         # the dvipng-generated images.
         if re.match(br'^cmsy\d+$', self.texname) and char == 0:
-            result[-1] = 0
-        return result
+            metrics[-1] = 0
+        return metrics
 
 
 class Vf(Dvi):
@@ -767,6 +779,9 @@ def _mul2012(num1, num2):
     return (num1*num2) >> 20
 
 
+WHD = namedtuple('WHD', 'width height depth')
+
+
 class Tfm:
     """
     A TeX Font Metric file.
@@ -782,13 +797,13 @@ class Tfm:
     checksum : int
        Used for verifying against the dvi file.
     design_size : int
-       Design size of the font (unknown units)
+       Design size of the font (unknown units).
     width, height, depth : dict
        Dimensions of each character, need to be scaled by the factor
        specified in the dvi file. These are dicts because indexing may
        not start from 0.
     """
-    __slots__ = ('checksum', 'design_size', 'width', 'height', 'depth')
+    __slots__ = ('checksum', 'design_size', '_whds', 'widths')
 
     def __init__(self, filename):
         _log.debug('opening tfm file %s', filename)
@@ -804,15 +819,37 @@ def __init__(self, filename):
             widths = struct.unpack(f'!{nw}i', file.read(4*nw))
             heights = struct.unpack(f'!{nh}i', file.read(4*nh))
             depths = struct.unpack(f'!{nd}i', file.read(4*nd))
-        self.width = {}
-        self.height = {}
-        self.depth = {}
+        self._whds = {}
         for idx, char in enumerate(range(bc, ec+1)):
             byte0 = char_info[4*idx]
             byte1 = char_info[4*idx+1]
-            self.width[char] = widths[byte0]
-            self.height[char] = heights[byte1 >> 4]
-            self.depth[char] = depths[byte1 & 0xf]
+            self._whds[char] = WHD(
+                widths[byte0], heights[byte1 >> 4], depths[byte1 & 0xf])
+        self.widths = [(1000 * self._whds[c].width if c in self._whds else 0) >> 20
+                       for c in range(max(self._whds))] if self._whds else []
+
+    def get_metrics(self, char):
+        return self._whds[char]
+
+    width = _api.deprecated("3.11")(
+        property(lambda self: {c: m.width for c, m in self._whds}))
+    height = _api.deprecated("3.11")(
+        property(lambda self: {c: m.height for c, m in self._whds}))
+    depth = _api.deprecated("3.11")(
+        property(lambda self: {c: m.depth for c, m in self._whds}))
+
+
+class TtfMetrics:
+    def __init__(self, filename):
+        self._face = FT2Font(filename, hinting_factor=1)  # Manage closing?
+        textpath.TextToPath._select_native_charmap(self._face)
+
+    def get_metrics(self, char):
+        mul = self._face.units_per_EM
+        g = self._face.load_glyph(char, LoadFlags.NO_SCALE)
+        return WHD(g.horiAdvance * mul,
+                   g.height * mul,
+                   (g.height - g.horiBearingY) * mul)
 
 
 PsFont = namedtuple('PsFont', 'texname psname effects encoding filename')
@@ -1007,8 +1044,7 @@ def _parse_enc(path):
     Returns
     -------
     list
-        The nth entry of the list is the PostScript glyph name of the nth
-        glyph.
+        The nth list item is the PostScript glyph name of the nth glyph.
     """
     no_comments = re.sub("%.*", "", Path(path).read_text(encoding="ascii"))
     array = re.search(r"(?s)\[(.*)\]", no_comments).group(1)
@@ -1113,26 +1149,45 @@ def _fontfile(cls, suffix, texname):
     from argparse import ArgumentParser
     import itertools
 
+    import fontTools.agl
+
     parser = ArgumentParser()
     parser.add_argument("filename")
     parser.add_argument("dpi", nargs="?", type=float, default=None)
     args = parser.parse_args()
     with Dvi(args.filename, args.dpi) as dvi:
         fontmap = PsfontsMap(find_tex_file('pdftex.map'))
         for page in dvi:
-            print(f"=== new page === "
+            print(f"=== NEW PAGE === "
                   f"(w: {page.width}, h: {page.height}, d: {page.descent})")
-            for font, group in itertools.groupby(
-                    page.text, lambda text: text.font):
-                print(f"font: {font.texname.decode('latin-1')!r}\t"
-                      f"scale: {font._scale / 2 ** 20}")
-                print("x", "y", "glyph", "chr", "w", "(glyphs)", sep="\t")
+            print("--- GLYPHS ---")
+            for font, group in itertools.groupby(page.text, lambda text: text.font):
+                font_name = font.texname.decode("latin-1")
+                filename = (font_name[1:-1] if font_name.startswith("[")
+                            else fontmap[font.texname].filename)
+                if font_name.startswith("["):
+                    print(f"font: {font_name}")
+                else:
+                    print(f"font: {font_name} at {filename}")
+                print(f"scale: {font._scale / 2 ** 20}")
+                print(" ".join(map("{:>11}".format, ["x", "y", "glyph", "chr", "w"])))
+                face = FT2Font(filename)
+                textpath.TextToPath._select_native_charmap(face)
                 for text in group:
-                    print(text.x, text.y, text.glyph,
-                          chr(text.glyph) if chr(text.glyph).isprintable()
-                          else ".",
-                          text.width, sep="\t")
+                    if font_name.startswith("["):
+                        glyph_name = face.get_glyph_name(text.glyph)
+                    else:
+                        if isinstance(text.glyph_name_or_index, str):
+                            glyph_name = text.glyph_name_or_index
+                        else:
+                            glyph_name = face.get_glyph_name(
+                                face.get_char_index(text.glyph))
+                    glyph_str = fontTools.agl.toUnicode(glyph_name)
+                    print(" ".join(map("{:>11}".format, [
+                        text.x, text.y, text.glyph, glyph_str, text.width])))
             if page.boxes:
-                print("x", "y", "h", "w", "", "(boxes)", sep="\t")
+                print("--- BOXES ---")
+                print(" ".join(map("{:>11}".format, ["x", "y", "h", "w"])))
                 for box in page.boxes:
-                    print(box.x, box.y, box.height, box.width, sep="\t")
+                    print(" ".join(map("{:>11}".format, [
+                        box.x, box.y, box.height, box.width])))
diff --git a/lib/matplotlib/mpl-data/matplotlibrc b/lib/matplotlib/mpl-data/matplotlibrc
@@ -322,6 +322,7 @@
                      # zapf chancery, charter, serif, sans-serif, helvetica,
                      # avant garde, courier, monospace, computer modern roman,
                      # computer modern sans serif, computer modern typewriter
+#text.latex.engine: latex
 #text.latex.preamble:   # IMPROPER USE OF THIS FEATURE WILL LEAD TO LATEX FAILURES
                         # AND IS THEREFORE UNSUPPORTED. PLEASE DO NOT ASK FOR HELP
                         # IF THIS FEATURE DOES NOT DO WHAT YOU EXPECT IT TO.
diff --git a/lib/matplotlib/rcsetup.py b/lib/matplotlib/rcsetup.py
@@ -1031,6 +1031,7 @@ def _convert_validator_spec(key, conv):
     # text props
     "text.color":          validate_color,
     "text.usetex":         validate_bool,
+    "text.latex.engine":   ["latex", "lualatex"],
     "text.latex.preamble": validate_string,
     "text.hinting":        ["default", "no_autohint", "force_autohint",
                             "no_hinting", "auto", "native", "either", "none"],
diff --git a/lib/matplotlib/texmanager.py b/lib/matplotlib/texmanager.py
