diff --git a/.dockerignore b/.dockerignore index e08d9e1df5..792dbe9046 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ doc tests -Pygments.egg-info TAGS build dist @@ -8,8 +7,6 @@ htmlcov venv **/__pycache__ .* -*.rst -*.egg *.pyo .*.sw[op] diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dd4e63b77b..0e8808559e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,63 +1,86 @@ name: Pygments -on: [push, pull_request] +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +env: + FORCE_COLOR: 1 + +permissions: + contents: read # to fetch code (actions/checkout) jobs: build: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] - max-parallel: 4 + os: [windows-latest, ubuntu-latest, macos-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + # Run macOS CI only when pushing tags, not on every push/PR. + # There seems not to be a prettier solution than this trick ... + exclude: + - os: ${{ !(github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')) && 'macos-latest' || 'dummy' }} steps: - - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install package - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install . + allow-prereleases: true + cache: 'pip' + - name: Install tox + run: pip install -r requirements.txt - name: Test package - run: pytest -W error + run: tox -- -W error check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - name: Run make check - run: make check - - name: Fail if the basic checks failed - run: make check + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install tox + run: pip install -r requirements.txt + - name: Perform basic checks + run: tox -e check if: runner.os == 'Linux' check-mapfiles: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install tox + run: pip install -r requirements.txt - name: Regenerate mapfiles - run: make mapfiles + run: tox -e mapfiles - name: Fail if mapfiles changed run: | if git ls-files -m | grep mapping; then - echo 'Please run "make mapfiles" and add the changes to a commit.' + echo 'Please run "tox -e mapfiles" and add the changes to a commit.' exit 1 fi lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.8 - - name: Check out regexlint - run: git clone https://github.com/pygments/regexlint + python-version: "3.x" + - name: Install tox + run: pip install -r requirements.txt - name: Run regexlint - run: make regexlint REGEXLINT=`pwd`/regexlint + run: tox -e regexlint diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 66a57b8dd3..0fdf2825c4 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -5,30 +5,35 @@ on: branches: - master + +env: + FORCE_COLOR: 1 + +permissions: {} jobs: build: + permissions: + contents: write # to push pages branch (peaceiris/actions-gh-pages) + runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.x" - name: Checkout Pygments - uses: actions/checkout@v2 - - name: Install Sphinx & WCAG contrast ratio - run: pip install Sphinx wcag-contrast-ratio - - name: Create Pyodide WASM package - run: cd doc && make pyodide + uses: actions/checkout@v4 + - name: Install tox + run: pip install -r requirements.txt - name: Sphinx build run: | - cd doc - WEBSITE_BUILD=1 make dirhtml - touch _build/dirhtml/.nojekyll - echo -e 'pygments.org\nwww.pygments.org' > _build/dirhtml/CNAME - echo 'Automated deployment of docs for GitHub pages.' > _build/dirhtml/README + tox -e web-doc -- dirhtml + touch doc/_build/dirhtml/.nojekyll + echo -e 'pygments.org\nwww.pygments.org' > doc/_build/dirhtml/CNAME + echo 'Automated deployment of docs for GitHub pages.' > doc/_build/dirhtml/README - name: Deploy to repo if: github.repository_owner == 'pygments' - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} external_repository: pygments/pygments.github.io diff --git a/.gitignore b/.gitignore index cbfddbe49c..f904026203 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -*.egg +*.egg-info/ *.pyc *.pyo .*.sw[op] @@ -8,7 +8,6 @@ /.tags /.tox/ /.cache/ -/Pygments.egg-info/* /TAGS /build/* /dist/* diff --git a/AUTHORS b/AUTHORS index 9f417bf909..811c66ae17 100644 --- a/AUTHORS +++ b/AUTHORS @@ -11,7 +11,9 @@ Other contributors, listed alphabetically, are: * Ali Afshar -- image formatter * Thomas Aglassinger -- Easytrieve, JCL, Rexx, Transact-SQL and VBScript lexers +* Maxence Ahlouche -- PostgreSQL Explain lexer * Muthiah Annamalai -- Ezhil lexer +* Nikolay Antipov -- OpenSCAD lexer * Kumar Appaiah -- Debian control lexer * Andreas Amann -- AppleScript lexer * Timothy Armstrong -- Dart lexer fixes @@ -29,6 +31,7 @@ Other contributors, listed alphabetically, are: * Michael Bayer -- Myghty lexers * Thomas Beale -- Archetype lexers * John Benediktsson -- Factor lexer +* David Benjamin, Google LLC -- TLS lexer * Trevor Bergeron -- mIRC formatter * Vincent Bernat -- LessCSS lexer * Christopher Bertels -- Fancy lexer @@ -39,13 +42,18 @@ Other contributors, listed alphabetically, are: * Frits van Bommel -- assembler lexers * Pierre Bourdon -- bugfixes * Martijn Braam -- Kernel log lexer, BARE lexer +* JD Browne, Google LLC -- GoogleSQL lexer * Matthias Bussonnier -- ANSI style handling for terminal-256 formatter * chebee7i -- Python traceback lexer improvements * Hiram Chirino -- Scaml and Jade lexers * Mauricio Caceres -- SAS and Stata lexers. +* Michael Camilleri, John Gabriele, sogaiu -- Janet lexer +* Daren Chandisingh -- Gleam lexer * Ian Cooper -- VGL lexer * David Corbett -- Inform, Jasmin, JSGF, Snowball, and TADS 3 lexers * Leaf Corcoran -- MoonScript lexer +* Fraser Cormack -- TableGen lexer +* Gabriel Corona -- ASN.1 lexer * Christopher Creutzig -- MuPAD lexer * Daniël W. Crompton -- Pike lexer * Pete Curry -- bugfixes @@ -64,6 +72,7 @@ Other contributors, listed alphabetically, are: * Paweł Fertyk -- GDScript lexer, HTML formatter improvements * Michael Ficarra -- CPSA lexer * James H. Fisher -- PostScript lexer +* Amanda Fitch, Google LLC -- GoogleSQL lexer * William S. Fulton -- SWIG lexer * Carlos Galdino -- Elixir and Elixir Console lexers * Michael Galloy -- IDL lexer @@ -111,6 +120,8 @@ Other contributors, listed alphabetically, are: MSDOS session, BC, WDiff * Brian R. Jackson -- Tea lexer * Christian Jann -- ShellSession lexer +* Jonas Camillus Jeppesen -- Line numbers and line highlighting for + RTF-formatter * Dennis Kaarsemaker -- sources.list lexer * Dmitri Kabak -- Inferno Limbo lexer * Igor Kalnitsky -- vhdl lexer @@ -132,16 +143,20 @@ Other contributors, listed alphabetically, are: * Jon Larimer, Google Inc. -- Smali lexer * Olov Lassus -- Dart lexer * Matt Layman -- TAP lexer +* Dan Lazin, Google LLC -- GoogleSQL lexer * Kristian Lyngstøl -- Varnish lexers * Sylvestre Ledru -- Scilab lexer * Chee Sing Lee -- Flatline lexer * Mark Lee -- Vala lexer +* Thomas Linder Puls -- Visual Prolog lexer +* Pete Lomax -- Phix lexer * Valentin Lorentz -- C++ lexer improvements * Ben Mabey -- Gherkin lexer * Angus MacArthur -- QML lexer * Louis Mandel -- X10 lexer * Louis Marchand -- Eiffel lexer * Simone Margaritelli -- Hybris lexer +* Tim Martin - World of Warcraft TOC lexer * Kirk McDonald -- D lexer * Gordon McGregor -- SystemVerilog lexer * Stephen McKamey -- Duel/JBST lexer @@ -160,7 +175,9 @@ Other contributors, listed alphabetically, are: * Paulo Moura -- Logtalk lexer * Mher Movsisyan -- DTD lexer * Dejan Muhamedagic -- Crmsh lexer +* Adrien Nayrat -- PostgreSQL Explain lexer * Ana Nelson -- Ragel, ANTLR, R console lexers +* David Neto, Google LLC -- WebGPU Shading Language lexer * Kurt Neufeld -- Markdown lexer * Nam T. Nguyen -- Monokai style * Jesper Noehr -- HTML formatter "anchorlinenos" @@ -186,9 +203,10 @@ Other contributors, listed alphabetically, are: * Kashif Rasul -- CUDA lexer * Nathan Reed -- HLSL lexer * Justin Reidy -- MXML lexer +* Jonathon Reinhart, Google LLC -- Soong lexer * Norman Richards -- JSON lexer * Corey Richardson -- Rust lexer updates -* Fabrizio Riguzzi -- cplint leder +* Fabrizio Riguzzi -- cplint leder * Lubomir Rintel -- GoodData MAQL and CL lexers * Andre Roberge -- Tango style * Georg Rollinger -- HSAIL lexer @@ -200,6 +218,7 @@ Other contributors, listed alphabetically, are: * Matteo Sasso -- Common Lisp lexer * Joe Schafer -- Ada lexer * Max Schillinger -- TiddlyWiki5 lexer +* Andrew Schmidt -- X++ lexer * Ken Schutte -- Matlab lexers * René Schwaiger -- Rainbow Dash style * Sebastian Schweizer -- Whiley lexer @@ -231,12 +250,14 @@ Other contributors, listed alphabetically, are: * Erick Tryzelaar -- Felix lexer * Alexander Udalov -- Kotlin lexer improvements * Thomas Van Doren -- Chapel lexer +* Dave Van Ee -- Uxntal lexer updates * Daniele Varrazzo -- PostgreSQL lexers * Abe Voelker -- OpenEdge ABL lexer * Pepijn de Vos -- HTML formatter CTags support * Matthias Vallentin -- Bro lexer * Benoît Vinot -- AMPL lexer * Linh Vu Hong -- RSL lexer +* Taavi Väänänen -- Debian control lexer * Immanuel Washington -- Smithy lexer * Nathan Weizenbaum -- Haml and Sass lexers * Nathan Whetsell -- Csound lexers @@ -248,6 +269,7 @@ Other contributors, listed alphabetically, are: * Enrique Zamudio -- Ceylon lexer * Alex Zimin -- Nemerle lexer * Rob Zimmerman -- Kal lexer +* Evgenii Zheltonozhskii -- Maple lexer * Vincent Zurczak -- Roboconf lexer * Hubert Gruniaux -- C and C++ lexer improvements * Thomas Symalla -- AMDGPU Lexer @@ -258,5 +280,12 @@ Other contributors, listed alphabetically, are: * Thomas Voss -- Sed lexer * Martin Fischer -- WCAG contrast testing * Marc Auberer -- Spice lexer +* Amr Hesham -- Carbon lexer +* diskdance -- Wikitext lexer +* vanillajonathan -- PRQL lexer +* Nikolay Antipov -- OpenSCAD lexer +* Markus Meyer, Nextron Systems -- YARA lexer +* Hannes Römer -- Mojo lexer +* Jan Frederik Schaefer -- PDDL lexer Many thanks for all contributions! diff --git a/CHANGES b/CHANGES index cd9eb9a2ff..175d193c2e 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,421 @@ Pygments changelog Pull request numbers before 2.4.2 are not linked as they refer to the now defunct Bitbucket project. +Version 2.20.0 +-------------- +(unreleased) + +Version 2.19.2 +-------------- +(released June 21st, 2025) + +* Lua: Fix regression introduced in 2.19.0 (#2882, #2839) + +Version 2.19.1 +-------------- +(released January 6th, 2025) + +- Updated lexers: + + * Ini: Fix quoted string regression introduced in 2.19.0 + * Lua: Fix a regression introduced in 2.19.0 + +Version 2.19.0 +-------------- +(released January 5th, 2025) + +- New lexers: + + * CodeQL (#2819) + * Debian Sources (#2788, #2747) + * Gleam (#2662) + * GoogleSQL (#2820, #2814) + * JSON5 (#2734, #1880) + * Maple (#2763, #2548) + * NumbaIR (#2433) + * PDDL (#2799, #2616) + * Rego (#2794) + * TableGen (#2751) + * Vue.js (#2832) + +- Updated lexers: + + * BQN: Various improvements (#2789) + * C#: Fix number highlighting (#986, #2727), add ``file`` keyword (#2726, #2805, #2806), add various other keywords (#2745, #2770) + * CSS: Add ``revert`` (#2766, #2775) + * Debian control: Add ``Change-By`` field (#2757) + * Elip: Improve punctuation handling (#2651) + * Igor: Add ``int`` (#2801) + * Ini: Fix quoted strings with embedded comment characters (#2767, #2720) + * Java: Support functions returning types containing a question mark (#2737) + * JavaScript: Support private identiiers (#2729, #2671) + * LLVM: Add ``splat``, improve floating-point number parsing (#2755) + * Lua: Improve variable detection, add built-in functions (#2829) + * Macaulay2: Update to 1.24.11 (#2800) + * PostgreSQL: Add more ``EXPLAIN`` keywords (#2785), handle ``/`` (#2774) + * S-Lexer: Fix keywords (#2082, #2750) + * TransactSQL: Fix single-line comments (#2717) + * Turtle: Fix triple quoted strings (#2744, #2758) + * Typst: Various improvements (#2724) + * Various: Add ``^`` as an operator to Matlab, Octave and Scilab (#2798) + * Vyper: Add ``staticcall`` and ``extcall`` (#2719) + +* Mark file extensions for ``HTML/XML+Evoque`` as aliases (#2743) +* Add a color for ``Operator.Word`` to the ``rrt`` style (#2709) +* Fix broken link in the documentation (#2803, #2804) +* Drop executable bit where not needed (#2781) +* Reduce Mojo priority relative to Python in ``analyze_text´` (#2771, #2772) +* Fix documentation builds (#2712) +* Match example file names to the lexer's name (#2713, #2715) +* Ensure lexer metadata is present (#2714) +* Search more directories on macOS for fonts (#2809) +* Improve test robustness (#2812) + +Version 2.18.0 +-------------- +(released May 4th, 2024) + +- New lexers: + + * Janet (#2557) + * Lean 4 (#2618, #2626) + * Luau (#2605) + * Mojo (#2691, #2515) + * org-mode (#2628, #2636) + * Promela (#2620) + * Soong / ``Android.bp`` (#2659) + * Tact (#2571) + * TSX + * Typst (#2596) + +- Updated lexers: + + * Awk: recognize ternary operator (#2687) + * Bash: add ``openrc`` alias (#2599, #2371) + * Coq: add keywords, lex more vernacular command arguments, produce + fewer tokens on heading comments (#2678) + * DNS zone files: Fix comment parsing (#2595) + * Hy: Support unicode literals (#1126) + * Inform6: Update to Inform 6.42 (#2644) + * lean: Fix name handling (#2614) + * Logtalk: add ``uninstantiation`` keyword and recognize + escape sequences (#2619) + * Macaulay2: Update to 1.23 (#2655) + * Python: fix highlighting of soft keywords before ``None``/``True``/``False`` + * reStructuredText: use ``Token.Comment`` for comments instead of + ``Comment.Preproc`` (#2598) + * Rust: highlight ``:``, ``::`` and ``->`` as ``Punctuation`` + and whitespace as ``Whitespace``, instead of ``Text`` + in both cases (#2631) + * Spice: Add keywords (#2621) + * SQL Explain: allow negative numbers (#2610) + * Swift: Support multiline strings (#2681) + * ThingsDB: add constants and new functions; support template + strings (#2624) + * UL4: support nested ```` and ```` tags (#2597) + * VHDL: support multi-line comments of VHDL-2008 (#2622) + * Wikitext: Remove ``kk-*`` in ``variant_langs`` (#2647) + * Xtend: Add ``val`` and ``var`` (#2602) + +- New styles: + + * Coffee (#2609) + +- Make background colors in the image formatter work with Pillow 10.0 (#2623) +- Require Python 3.8. As a result, the ``importlib-metadata`` package + is no longer needed for fast plugin discovery on Python 3.7. + The ``plugins`` extra (used as, e.g., ``pip install pygments[plugins]``) + is kept for backwards compatibility but now has no effect. (#2601) +- Require the ``url`` attribute for lexers inside Pygments, add + it to many lexers (#2588) +- Replace Pyflakes linter with Ruff (#2592) +- Add macOS CI (#2594) +- Built-in lexers now declare the version of Pygments in which they were + added in a required ``version_added`` lexer attribute, instead of a + ``.. versionadded::`` directive in the docstring (#2589, #2634) +- The ``url`` attribute is now required for built-in lexers and + has been added to all existing lexers (#2588) +- The RTF formatter supports line number and line highlighting now (#1217, #2654) +- Add ``\sa0`` keyword in the RTF formatter (#1111, #2607) +- Register ``pycon`` as an alias for the Python console lexer (#2697) +- Add MIME-Type for ``DesktopLexer`` (#2613) +- Fix native style to meet WCAG AA guidelines (#2600) +- Fix typo in documentation (#2672) +- Use format strings consistently (#2661) +- Add ``__class_getitem__`` to ``Formatter`` to improve typing support (#2665) + +Version 2.17.2 +-------------- +(released November 21, 2023) + +- Fix a packaging issue on macOS (#2593) + +Version 2.17.1 +-------------- +(released November 19, 2023) + +- Updated lexers: + + * TOML: Fix bug making lexing of single-quoted strings too eager + +Version 2.17.0 +-------------- +(released November 18, 2023) + +- New lexers: + + * JSX (#2524, #709) + * Kusto (#2552) + * ldaprc (#2532) + * LDIF (#2489) + * PRQL (#2507, #2523, #2559) + * Visual Prolog and Visual Prolog Grammar (#2480) + * Vyper (#2531, #2579) + +- Updated lexers: + + * Cypher: fix comment matching, add missing keywords (#2504) + * Fortran: add ``elseif`` keyword (#2528) + * Lean: make it available as ``lean3``, in preparation for + a possible switch to ``lean`` highlighting as Lean 4 (#2546) + * JSON: add MIME types and file extensions for several line-delimited + JSON formats (#2490) + * Nix: many improvements (#2551, #1800) + * OCaml: Add ``and`` keyword, remove ``value`` from keywords (#2521) + * Python: add ``starlark`` and ``bazel`` aliases (#2517, #2516) + * Snowball: Treat ``len`` like ``size`` (#2508) + * Spice: add ``panic`` keyword and ``->`` operator (#2510) + * squid.conf: fix catastrophic backtracking (#2583) + * TOML: rewritten, with many fixes (#2576) + * Turtle: support blank nodes (#2581) + * Wikitext: fix erroneous highlighting of LanguageConverter markup + (#2493), add missing variant languages (#2494) + * CMake: support ``[=[ bracketed arguments ]=]`` (#2549) + +- Fix ctags support and tests (#2487) +- Include ``Lexer.add_filter`` in the documentation (#2519) +- Add a ``Lean3Lexer`` alias (#2546) +- The ``pygments.styles`` module contains a new ``STYLES`` variable + with a dictionary of built-in styles. The old ``STYLE_MAP`` variable, + which uses a different format, is kept for backwards compatibility. +- On Windows, add a new installation extra (``windows-terminal``) which pulls in + dependencies for colored console output. See :doc:`cmdline` for more details. + (#2505) +- Support more file types in ``autopygmentize`` script (#2513) +- Change color of numbers in ``rrt`` style (#2526) +- Fix error when trying to look up plugin formatters by file extension + of the output format (#2563) +- Use Hatchling as a build backend instead of setuptools. + This change is transparent to most users. Distribution packagers + who build without build isolation need to add hatchling as a build + dependency and remove setuptools. People downloading source distributions + and wheels from PyPI directly should note that they now have ``pygments`` + in their file names instead of ``Pygments``. (#2573) +- Improve the test framework to also check for lost tokens when processing the + snippets and example files (#2582.) +- Improve the *Dracula* style definition to make it easier to maintain (#2575) + +Version 2.16.1 +-------------- +(released August 6th, 2023) + +- Fix ``native`` style missing from style list (#2484) + +Version 2.16.0 +-------------- +(released August 6th, 2023) + +- New lexers: + + * ASN.1 (#2462) + * Blueprint (#2434) + * BQN (#2472) + * DNS zone files (#2464) + * GraphQL (#2428) + * Linux desktop files (following the specification of the + Freedesktop group, formerly known as XDG) (#2470) + * NVIDIA PTX (#2432) + * OpenSCAD (#2449) + * systemd (#2470) + * TLS presentation language (#2455) + * Verifpal (#2430) + * YARA (#2453) + +- Updated lexers: + + * ASC: Add ``application/pem-certificate-chain`` mimetype (#2471) + * C/C++: Refine keyword lists (#2421, #2422) + * Carbon: Fix long processing times on invalid input, fix number + lexing (#2454, #2456) + * Elpi: Handle quotations (#2419) + * Go: Support additional built-ins (#2481) + * HTTP: Support empty headers (#2461), support more general methods (#2460), + also recognize responses in ``analyse_text`` implementation (#2460), and + highlight URL encoded data (#2465, #1620) + * Igor Pro: Update to Igor Pro 9 (#2482) + * lean: Recognize expressions nested within attributes (#1817) + * Macaulay2: Update builtins (#2457) + * Markdown: Allow extra characters after language name + in code blocks (#2437) + * NestedText: Update to version 3 (#2459) + * scdoc: Improve language guessing implementation (#2402) + * Spice: Update to latest version (#2476) + * Transact SQL: Add Pre-sorted Group keyword (#2417) + * Uxntal: Update for current runes (#2424) + * Wikitext: Fix templates in wiki links; fix a language converter false + positive; add bold italic markup (#2447) + +- Add ``Generic.EmphStrong`` token for bold italic markup (#2444) +- Add Lightbulb style (#2474) +- Improve contrast in Monokai style (#2448) +- Add documentation how to create terminal code highlighting commands (#2131, #2425) +- Add support for loading TrueType fonts to the ``ImageFormatter`` (#1960) + +Version 2.15.1 +-------------- +(released April 18th, 2023) + +- Updated lexers: + + * Java properties: Fix catastrophic backtracking (#2356, #2404) + +- Fix Python console traceback lexing being too strict + and sometimes reordering output (#2407, #2410, #2412) + + +Version 2.15.0 +-------------- +(released April 10th, 2023) + +- Added lexers: + + * Carbon (#2362, #2365, #2366, #2367, #2368, #2369, #2370) + * Dax (#2335, #2345) + * MediaWiki Wikitext (#2373, #827) + * PostgreSQL Explain (#2398) + * WGSL (WebGPU Shading Language) (#2386) + * X++ (#2339) + +- Updated lexers: + + * AMDGPU: Add support for ``scratch_`` instructions, the ``attr*.*`` argument, + as well as the ``off`` modifier (#2327). + * APDL: Miscellaneous improvements (#2314) + * bash/tcsh: + + - Move ``break`` to keywords (#2377) + - Improve bash math expansion lexing (#2255, #2353) + + * Chapel: Support attributes (#2376) + * CMake: Implement bracket style comments (#2338, #2354) + * CSS: Improve lexing of numbers inside function calls (#2382, #2383) + * diff: Support normal diff syntax, as opposed to unified diff syntax (#2321) + * GLSL, HLSL: + + - Support line continuations in preprocessor code (#2350) + - Improve preprocessor directive handling (#2357) + + * LilyPond: minor update of builtins + * PHP: support attributes (#2055, #2347, #2360), fix anonymous classes without + parameters (#2359), improve lexing of variable variable syntax (#2358) + * Python: + + - Add missing builtins (#2334) + - Fix inconsistent lexing of ``None`` (#2406) + + * Rebol/Red: Don't require script headers (#2348, #2349) + * Spice: Update keywords (#2336) + * SQL+Jinja (``analyse_text`` method): Fix catastrophic backtracking (#2355) + * Terraform: Add ``hcl`` alias (#2375) + +- Declare support for Python 3.11 and drop support for Python 3.6 (#2324). +- Update ``native`` style to improve contrast (#2325). +- Update `github-dark`` style to match latest Primer style (#2401) +- Revert a change that made guessing lexers based on file names slower + on Python 3.10 and older (#2328). +- Fix some places where a locale-dependent encoding could unintentionally + be used instead of UTF-8 (#2326). +- Fix Python traceback handling (#2226, #2329). +- Groff formatter: sort color definitions for reproducibility (#2343) +- Move project metadata to ``pyproject.toml``, remove ``setup.py`` + and ``setup.cfg`` (#2342) + +- The top-level ``Makefile`` has been removed. Instead, all shortcuts + for developing are now defined and run through tox. The ``doc`` folder + still contains a ``Makefile`` as an alternative to ``tox -e doc``. + + +Version 2.14.0 +-------------- +(released January 1st, 2023) + +- Added lexers: + + * Arturo (#2259) + * GAP session (#2211) + * Fift (#2249) + * func (#2232) + * Jsonnet (#2239) + * Minecraft schema (#2276) + * MIPS (#2228) + * Phix (#2222) + * Portugol (#2300) + * TL-b (#2247) + * World of Warcraft TOC format (#2244, #2245) + * Wren (#2271) + +- Updated lexers: + + * Abap: Update keywords (#2281) + * Alloy: Update for Alloy 6 (#1963) + * C family (C, C++ and many others): + + - Fix an issue where a chunk would be wrongly recognized as a function + definition due to braces in comments (#2210) + - Improve parantheses handling for function definitions (#2207, #2208) + + * C#: Fix number and operator recognition (#2256, #2257) + * CSound: Updated builtins (#2268) + * F#: Add ``.fsx`` file extension (#2282) + * gas (GNU assembler): recognize braces as punctuation (#2230) + * HTTP: Add `CONNECT` keyword (#2242) + * Inform 6: Fix lexing of properties and doubles (#2214) + * INI: Allow comments that are not their own line (#2217, #2161) + * Java properties: Fix issue with whitespace-delimited keys, support + comments starting with `!` and escapes, no longer support undocumented + `;` and `//` comments (#2241) + * LilyPond: Improve heuristics, add ``\maxima`` duration (#2283) + * LLVM: Add opaque pointer type (#2269) + * Macaulay2: Update keywords (#2305) + * Minecraft-related lexers (SNB and Minecraft function) moved to + ``pygments.lexers.minecraft`` (#2276) + * Nim: General improvements (#1970) + * Nix: Fix single quotes inside indented strings (#2289) + * Objective J: Fix catastrophic backtracking (#2225) + * NASM: Add support for SSE/AVX/AVX-512 registers as well as 'rel' + and 'abs' address operators (#2212) + * Powershell: + + - Add ``local:`` keyword (#2254) + - Allow continuations without markers (#2262, #2263) + + * Solidity: Add boolean operators (#2292) + * Spice: Add ``enum`` keyword and fix a bug regarding binary, + hexadecimal and octal number tokens (#2227) + * YAML: Accept colons in key names (#2277) + +- Fix `make mapfiles` when Pygments is not installed in editable mode + (#2223) + +- Support more filetypes and compression types in `autopygmentize` (#2219) +- Merge consecutive tokens in Autohotkey, Clay (#2248) +- Add ``.nasm`` as a recognized file type for NASM (#2280) +- Add ``*Spec.hs`` as a recognized file type for ``HSpec`` (#2308) +- Add ``*.pyi`` (for typing stub files) as a recognized file type for + Python (#2231) +- The HTML lexer no longer emits empty spans for whitespace (#2304) +- Fix ``IRCFormatter`` inserting linenumbers incorrectly (#2270) + Version 2.13.0 -------------- (released August 15th, 2022) @@ -22,29 +437,29 @@ Version 2.13.0 elements of function headers, e.g. between the arguments and the opening brace for the body (#1891) * C++: Resolve several cases of ``Error`` tokens (#2207, #2208) - * Coq: add some common keywords, improve recognition of ``Set`` + * Coq: Add some common keywords, improve recognition of ``Set`` and qualified identifiers (#2158) * F*: Allow C-style comments anywhere in a line - * Fortran: fix catastrophic backtracking with backslashes in strings + * Fortran: Fix catastrophic backtracking with backslashes in strings (#2194) * Go: add support for generics (#2167) - * Inform: update for version 6.40 (#2190) + * Inform: Update for version 6.40 (#2190) * Isabelle: recognize cartouches (#2089) * Java: support multiline strings aka. text blocks (#2132) - * Kotlin: add ``value`` modifier (#2142) - * LilyPond: add some missing builtins - * Macaulay2: update builtins (#2139) + * Kotlin: Add ``value`` modifier (#2142) + * LilyPond: Add some missing builtins + * Macaulay2: Update builtins (#2139) * Matlab session: fix traceback when a line continuation ellipsis appears in the output (#2166) - * .NET: add aliases for LibreOffice Basic, OpenOfficeBasic and + * .NET: Add aliases for LibreOffice Basic, OpenOfficeBasic and StarOffice Basic (#2170) - * Nim: use ``Name.Builtin`` instead of ``Keyword.Type`` (#2136) + * Nim: Use ``Name.Builtin`` instead of ``Keyword.Type`` (#2136) * PHP: fix `\"$var\"` inside strings (#2105) * Python: only recognize ``\N``, ``\u`` and ``\U`` escape sequences in string literals, but not in bytes literals where they are not supported (#2204) * Tcl: support ``${name}`` variables (#2145) - * Terraform: accept leading whitespace for `<<` heredoc + * Terraform: Accept leading whitespace for `<<` heredoc delimiters (#2162) * Teraterm: Various improvements (#2165) * Spice: add support for the recently added features including more diff --git a/Contributing.md b/Contributing.md deleted file mode 100644 index 0d831f0b1f..0000000000 --- a/Contributing.md +++ /dev/null @@ -1,167 +0,0 @@ -Licensing -========= - -The code is distributed under the BSD 2-clause license. Contributors making pull -requests must agree that they are able and willing to put their contributions -under that license. - -Goals & non-goals of Pygments -============================= - -Python support --------------- - -Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/#status-of-python-branches). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments. - -Validation ----------- - -Pygments does not attempt to validate the input. Accepting code that is not legal for a given language is acceptable if it simplifies the codebase and does not result in surprising behavior. For instance, in C89, accepting `//` based comments would be fine because de-facto all compilers supported it, and having a separate lexer for it would not be worth it. - -Contribution checklist -====================== - -* Check the documentation for how to write - [a new lexer](https://pygments.org/docs/lexerdevelopment/), - [a new formatter](https://pygments.org/docs/formatterdevelopment/) or - [a new filter](https://pygments.org/docs/filterdevelopment/) - -* Make sure to add a test for your new functionality, and where applicable, - write documentation. - -* When writing rules, try to merge simple rules. For instance, combine: - - ```python - _PUNCTUATION = [ - (r"\(", token.Punctuation), - (r"\)", token.Punctuation), - (r"\[", token.Punctuation), - (r"\]", token.Punctuation), - ("{", token.Punctuation), - ("}", token.Punctuation), - ] - ``` - - into: - - ```python - (r"[\(\)\[\]{}]", token.Punctuation) - ``` - -* Be careful with ``.*``. This matches greedily as much as it can. For instance, - a rule like ``@.*@`` will match the whole string ``@first@ second @third@``, - instead of matching ``@first@`` and ``@second@``. You can use ``@.*?@`` in - this case to stop early. The ``?`` tries to match _as few times_ as possible. - -* Beware of so-called "catastrophic backtracking". As a first example, consider - the regular expression ``(A+)*C``. This is equivalent to ``A*B`` regarding - what it matches, but *non*-matches will take very long. This is because - of the way the regular expression engine works. Suppose you feed it 50 - 'A's, and a 'C' at the end. It first matches the 'A's greedily in ``A+``, - but finds that it cannot match the end since 'B' is not the same as 'C'. - Then it backtracks, removing one 'A' from the first ``A+`` and trying to - match the rest as another ``(A+)*``. This fails again, so it backtracks - further left in the input string, etc. In effect, it tries all combinations - - ``` - (AAAAAAAAAAAAAAAAA) - (AAAAAAAAAAAAAAAA)(A) - (AAAAAAAAAAAAAAA)(AA) - (AAAAAAAAAAAAAAA)(A)(A) - (AAAAAAAAAAAAAA)(AAA) - (AAAAAAAAAAAAAA)(AA)(A) - ... - ``` - - Thus, the matching has exponential complexity. In a lexer, the - effect is that Pygments will seemingly hang when parsing invalid - input. - - ```python - >>> import re - >>> re.match('(A+)*B', 'A'*50 + 'C') # hangs - ``` - - As a more subtle and real-life example, here is a badly written - regular expression to match strings: - - ```python - r'"(\\?.)*?"' - ``` - - If the ending quote is missing, the regular expression engine will - find that it cannot match at the end, and try to backtrack with less - matches in the ``*?``. When it finds a backslash, as it has already - tried the possibility ``\\.``, it tries ``.`` (recognizing it as a - simple character without meaning), which leads to the same - exponential backtracking problem if there are lots of backslashes in - the (invalid) input string. A good way to write this would be - ``r'"([^\\]|\\.)*?"'``, where the inner group can only match in one - way. Better yet is to use a dedicated state, which not only - sidesteps the issue without headaches, but allows you to highlight - string escapes. - - ```python - 'root': [ - ..., - (r'"', String, 'string'), - ... - ], - 'string': [ - (r'\\.', String.Escape), - (r'"', String, '#pop'), - (r'[^\\"]+', String), - ] - ``` - -* When writing rules for patterns such as comments or strings, match as many - characters as possible in each token. This is an example of what not to - do: - - ```python - 'comment': [ - (r'\*/', Comment.Multiline, '#pop'), - (r'.', Comment.Multiline), - ] - ``` - - This generates one token per character in the comment, which slows - down the lexing process, and also makes the raw token output (and in - particular the test output) hard to read. Do this instead: - - ```python - 'comment': [ - (r'\*/', Comment.Multiline, '#pop'), - (r'[^*]+', Comment.Multiline), - (r'\*', Comment.Multiline), - ] - ``` - -* Don't add imports of your lexer anywhere in the codebase. (In case you're - curious about ``compiled.py`` -- this file exists for backwards compatibility - reasons.) - -* Use the standard importing convention: ``from token import Punctuation`` - -* For test cases that assert on the tokens produced by a lexer, use tools: - - * You can use the ``testcase`` formatter to produce a piece of code that - can be pasted into a unittest file: - ``python -m pygments -l lua -f testcase <<< "local a = 5"`` - - * Most snippets should instead be put as a sample file under - ``tests/snippets//*.txt``. These files are automatically - picked up as individual tests, asserting that the input produces the - expected tokens. - - To add a new test, create a file with just your code snippet under a - subdirectory based on your lexer's main alias. Then run - ``pytest --update-goldens `` to auto-populate the currently - expected tokens. Check that they look good and check in the file. - - Also run the same command whenever you need to update the test if the - actual produced tokens change (assuming the change is expected). - - * Large test files should go in ``tests/examplefiles``. This works - similar to ``snippets``, but the token output is stored in a separate - file. Output can also be regenerated with ``--update-goldens``. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c6a8567d25..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include Makefile CHANGES LICENSE AUTHORS -include external/* -recursive-include tests * -recursive-include doc * -recursive-include scripts * diff --git a/Makefile b/Makefile deleted file mode 100644 index 427e5a47b0..0000000000 --- a/Makefile +++ /dev/null @@ -1,68 +0,0 @@ -# -# Makefile for Pygments -# ~~~~~~~~~~~~~~~~~~~~~ -# -# Combines scripts for common tasks. -# -# :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. -# :license: BSD, see LICENSE for details. -# - -PYTHON ?= python3 - -export PYTHONPATH = $(shell echo "$$PYTHONPATH"):$(shell python -c 'import os; print ":".join(os.path.abspath(line.strip()) for line in file("PYTHONPATH"))' 2>/dev/null) - -.PHONY: all check clean clean-pyc docs mapfiles \ - pylint reindent test test-coverage \ - tox-test tox-test-coverage regexlint - -all: clean-pyc check test - -check: - @$(PYTHON) scripts/check_crlf.py pygments build external - @$(PYTHON) scripts/detect_missing_analyse_text.py --skip-no-aliases - @pyflakes pygments | grep -v 'but unused' || true - @$(PYTHON) scripts/check_sources.py -i build -i dist -i pygments/lexers/_mapping.py \ - -i docs/build -i pygments/formatters/_mapping.py -i pygments/unistring.py \ - -i tests/support/empty.py - @$(PYTHON) scripts/count_token_references.py --minfiles=1 --maxfiles=1 \ - --minlines=1 --maxlines=3 --subtoken - -clean: clean-pyc - -rm -rf doc/_build build Pygments.egg-info - -rm -f codetags.html - -clean-pyc: - find . -name '__pycache__' -exec rm -rf {} + - -docs: - make -C doc html - -mapfiles: - $(PYTHON) scripts/gen_mapfiles.py - -pylint: - @pylint --rcfile scripts/pylintrc pygments - -reindent: - @$(PYTHON) scripts/reindent.py -r -B . - -TEST = tests - -test: - @$(PYTHON) `which pytest` $(TEST) - -test-coverage: - @$(PYTHON) `which pytest` --cov --cov-report=html --cov-report=term $(TEST) - -tox-test: - @tox -- $(TEST) - -tox-test-coverage: - @tox -- --with-coverage --cover-package=pygments --cover-erase $(TEST) - -RLMODULES = pygments.lexers - -regexlint: - @if [ -z "$(REGEXLINT)" ]; then echo "Please set REGEXLINT=checkout path"; exit 1; fi - PYTHONPATH=`pwd`:$(REGEXLINT) $(PYTHON) $(REGEXLINT)/regexlint/cmdline.py $(RLMODULES) diff --git a/README.rst b/README.rst index e9633e058d..a58149184e 100644 --- a/README.rst +++ b/README.rst @@ -9,20 +9,20 @@ Installing ---------- ... works as usual, use ``pip install Pygments`` to get published versions, -or ``python setup.py install`` to install from a checkout. +or ``pip install -e .`` to install from a checkout in editable mode. Documentation ------------- ... can be found online at https://pygments.org/ or created with Sphinx by :: - make docs + tox -e doc By default, the documentation does not include the demo page, as it requires having Docker installed for building Pyodide. To build the documentation with the demo page, use :: - WEBSITE_BUILD=1 make docs + tox -e web-doc The initial build might take some time, but subsequent ones should be instant because of Docker caching. @@ -44,15 +44,44 @@ Continuous testing runs on GitHub workflows: .. image:: https://github.com/pygments/pygments/workflows/Pygments/badge.svg :target: https://github.com/pygments/pygments/actions?query=workflow%3APygments -Contribution guidelines are found in Contributing.md_. +Please read our `Contributing instructions `_. -.. _Contributing.md: https://github.com/pygments/pygments/blob/master/Contributing.md +Security considerations +----------------------- + +Pygments provides no guarantees on execution time, which needs to be taken +into consideration when using Pygments to process arbitrary user inputs. For +example, if you have a web service which uses Pygments for highlighting, there +may be inputs which will cause the Pygments process to run "forever" and/or use +significant amounts of memory. This can subsequently be used to perform a +remote denial-of-service attack on the server if the processes are not +terminated quickly. + +Unfortunately, it's practically impossible to harden Pygments itself against +those issues: Some regular expressions can result in "catastrophic +backtracking", but other bugs like incorrect matchers can also +cause similar problems, and there is no way to find them in an automated fashion +(short of solving the halting problem.) Pygments has extensive unit tests, +automated randomized testing, and is also tested by `OSS-Fuzz `_, +but we will never be able to eliminate all bugs in this area. + +Our recommendations are: + +* Ensure that the Pygments process is *terminated* after a reasonably short + timeout. In general Pygments should take seconds at most for reasonably-sized + input. +* *Limit* the number of concurrent Pygments processes to avoid oversubscription + of resources. + +The Pygments authors will treat any bug resulting in long processing times with +high priority -- it's one of those things that will be fixed in a patch release. +When reporting a bug where you suspect super-linear execution times, please make +sure to attach an input to reproduce it. The authors ----------- -Pygments is maintained by **Georg Brandl**, e-mail address *georg*\ *@*\ *python.org* -and **Matthäus Chajdas**. +Pygments is maintained by **Georg Brandl**, e-mail address *georg*\ *@*\ *python.org*, **Matthäus Chajdas** and **Jean Abou-Samra**. Many lexers and fixes have been contributed by **Armin Ronacher**, the rest of the `Pocoo `_ team and **Tim Hatch**. diff --git a/description.rst b/description.rst index de34cf1339..9bd598e361 100644 --- a/description.rst +++ b/description.rst @@ -14,5 +14,5 @@ are: formats that PIL supports and ANSI sequences * it is usable as a command-line tool and as a library -Copyright 2006-2022 by the Pygments team, see ``AUTHORS``. -Licensed under the BSD, see ``LICENSE`` for details. \ No newline at end of file +Copyright 2006-2025 by the Pygments team, see ``AUTHORS``. +Licensed under the BSD, see ``LICENSE`` for details. diff --git a/doc/Makefile b/doc/Makefile index a0dcaaa4cb..7fb7541191 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -14,11 +14,10 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean pyodide html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" - @echo " pyodide to make Pyodide with currently checked out Pygments" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @@ -42,21 +41,12 @@ help: clean: -rm -rf $(BUILDDIR)/* -pyodide: - $(if $(test ! -f docker), $(error "Could not find Docker. Please install that before continuing.")) - # Enable the BuildKit backend to use the --output option. - DOCKER_BUILDKIT=1 docker build --file pyodide/Dockerfile --output $(BUILDDIR)/pyodide/pyodide .. - @echo - @echo "Pyodide build finished. The Pyodide artifacts are in $(BUILDDIR)/pyodide." - html: - $(if $(WEBSITE_BUILD), $(MAKE) pyodide) $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - $(if $(WEBSITE_BUILD), $(MAKE) pyodide) $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." diff --git a/doc/_static/demo.css b/doc/_static/demo.css index e14379af38..eaa4410418 100644 --- a/doc/_static/demo.css +++ b/doc/_static/demo.css @@ -17,6 +17,7 @@ padding: 2px; width: 100%; min-height: 150px; + resize: vertical; } #hlcode { diff --git a/doc/_templates/languages.html b/doc/_templates/languages.html deleted file mode 100644 index e046a90612..0000000000 --- a/doc/_templates/languages.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "layout.html" %} - -{% block htmltitle %}Languages {{ titlesuffix }}{% endblock %} - -{% block body %} - -

Languages

- -
    -{% for language in languages %} -
  • - {% if language.url %} - {{language.name}} - {% else %} - {{language.name}} - {% endif %} -
    • -{% endfor %} -
-{{ body }} - -

... that's all?

- -Well, why not write your own? Contributing to Pygments is easy and fun. Take a -look at the docs on lexer development. Pull -requests are welcome on GitHub. - -Note: the languages listed here are supported in the development version. The -latest release may lack a few of them. - -{% endblock %} diff --git a/doc/_themes/pygments14/layout.html b/doc/_themes/pygments14/layout.html index 34e86ef4e6..ba630e9d9c 100644 --- a/doc/_themes/pygments14/layout.html +++ b/doc/_themes/pygments14/layout.html @@ -85,8 +85,8 @@ {% block footer %} {# closes "flexwrapper" div #}