From 50c276412880c1a3dde8a6d6c909e3ed8ef47e43 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 22 Jun 2022 09:10:10 +0200 Subject: [PATCH 0001/1154] Delete unused Travis CI config and reference in docs (GH-345) --- .travis.yml | 86 ----------------------------------------------------- README.rst | 2 +- 2 files changed, 1 insertion(+), 87 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9d8a9f424..000000000 --- a/.travis.yml +++ /dev/null @@ -1,86 +0,0 @@ -os: linux -language: python - -cache: - pip: true - directories: - - $HOME/.ccache - - libs - -python: - - nightly - - 3.10 - - 2.7 - - 3.9 - - 3.8 - - 3.7 - - 3.6 - - 3.5 - -env: - global: - - USE_CCACHE=1 - - CCACHE_SLOPPINESS=pch_defines,time_macros - - CCACHE_COMPRESS=1 - - CCACHE_MAXSIZE=70M - - PATH="/usr/lib/ccache:$PATH" - - LIBXML2_VERSION=2.9.10 - - LIBXSLT_VERSION=1.1.34 - matrix: - - STATIC_DEPS=false - - STATIC_DEPS=true - -matrix: - include: - - python: 3.8 - env: - - STATIC_DEPS=false - - EXTRA_DEPS="docutils pygments sphinx sphinx-rtd-theme" - script: make html - - python: 3.8 - env: - - STATIC_DEPS=false - - EXTRA_DEPS="coverage<5" - - python: 3.8 - env: - - STATIC_DEPS=true - - LIBXML2_VERSION=2.9.2 # minimum version requirements - - LIBXSLT_VERSION=1.1.27 - - python: pypy - env: STATIC_DEPS=false - - python: pypy3 - env: STATIC_DEPS=false - - python: 3.8 - env: STATIC_DEPS=false - arch: arm64 - - python: 3.8 - env: STATIC_DEPS=true - arch: arm64 - - python: 3.8 - env: STATIC_DEPS=false - arch: ppc64le - - python: 3.8 - env: STATIC_DEPS=true - arch: ppc64le - allow_failures: - - python: nightly - - python: pypy - - python: pypy3 - -install: - - pip install -U pip wheel - - if [ -z "${TRAVIS_PYTHON_VERSION##*-dev}" ]; - then pip install --install-option=--no-cython-compile https://github.com/cython/cython/archive/master.zip; - else pip install -r requirements.txt; - fi - - pip install -U beautifulsoup4 cssselect html5lib rnc2rng==2.6.5 ${EXTRA_DEPS} - -script: - - CFLAGS="-O0 -g -fPIC" python -u setup.py build_ext --inplace - $(if [ -n "${TRAVIS_PYTHON_VERSION##2.*}" -a -n "${TRAVIS_PYTHON_VERSION##3.[34]*}" ]; then echo -n " -j7 "; fi ) - $(if [ -n "$EXTRA_DEPS" -a -z "${EXTRA_DEPS##*coverage*}" ]; then echo -n "--with-coverage"; fi ) - - ccache -s || true - - CFLAGS="-O0 -g -fPIC" PYTHONUNBUFFERED=x make test - - ccache -s || true - - python setup.py install - - python -c "from lxml import etree" diff --git a/README.rst b/README.rst index e8705ab92..a0434b379 100644 --- a/README.rst +++ b/README.rst @@ -63,7 +63,7 @@ Crypto currencies do not fit into that ambition. .. _`doc/main.txt`: https://github.com/lxml/lxml/blob/master/doc/main.txt .. _`INSTALL.txt`: http://lxml.de/installation.html -`Travis-CI `_ and `AppVeyor `_ +`AppVeyor `_ and `GitHub Actions `_ support the lxml project with their build and CI servers. Jetbrains supports the lxml project by donating free licenses of their `PyCharm IDE `_. From 86368e9cf70a0ad23cccd5ee32de847149af0c6f Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 1 Jul 2022 21:06:10 +0200 Subject: [PATCH 0002/1154] Fix a crash when incorrect parser input occurs together with usages of iterwalk() on trees generated by the same parser. --- src/lxml/apihelpers.pxi | 7 ++++--- src/lxml/iterparse.pxi | 11 ++++++----- src/lxml/tests/test_etree.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/lxml/apihelpers.pxi b/src/lxml/apihelpers.pxi index c16627629..9fae9fb12 100644 --- a/src/lxml/apihelpers.pxi +++ b/src/lxml/apihelpers.pxi @@ -246,9 +246,10 @@ cdef dict _build_nsmap(xmlNode* c_node): while c_node is not NULL and c_node.type == tree.XML_ELEMENT_NODE: c_ns = c_node.nsDef while c_ns is not NULL: - prefix = funicodeOrNone(c_ns.prefix) - if prefix not in nsmap: - nsmap[prefix] = funicodeOrNone(c_ns.href) + if c_ns.prefix or c_ns.href: + prefix = funicodeOrNone(c_ns.prefix) + if prefix not in nsmap: + nsmap[prefix] = funicodeOrNone(c_ns.href) c_ns = c_ns.next c_node = c_node.parent return nsmap diff --git a/src/lxml/iterparse.pxi b/src/lxml/iterparse.pxi index 138c23a6a..a7299da6d 100644 --- a/src/lxml/iterparse.pxi +++ b/src/lxml/iterparse.pxi @@ -420,7 +420,7 @@ cdef int _countNsDefs(xmlNode* c_node): count = 0 c_ns = c_node.nsDef while c_ns is not NULL: - count += 1 + count += (c_ns.href is not NULL) c_ns = c_ns.next return count @@ -431,9 +431,10 @@ cdef int _appendStartNsEvents(xmlNode* c_node, list event_list) except -1: count = 0 c_ns = c_node.nsDef while c_ns is not NULL: - ns_tuple = (funicode(c_ns.prefix) if c_ns.prefix is not NULL else '', - funicode(c_ns.href)) - event_list.append( (u"start-ns", ns_tuple) ) - count += 1 + if c_ns.href: + ns_tuple = (funicodeOrEmpty(c_ns.prefix), + funicode(c_ns.href)) + event_list.append( (u"start-ns", ns_tuple) ) + count += 1 c_ns = c_ns.next return count diff --git a/src/lxml/tests/test_etree.py b/src/lxml/tests/test_etree.py index e5f084692..285313f6e 100644 --- a/src/lxml/tests/test_etree.py +++ b/src/lxml/tests/test_etree.py @@ -1460,6 +1460,26 @@ def test_iterwalk_getiterator(self): [1,2,1,4], counts) + def test_walk_after_parse_failure(self): + # This used to be an issue because libxml2 can leak empty namespaces + # between failed parser runs. iterwalk() failed to handle such a tree. + try: + etree.XML('''''') + except etree.XMLSyntaxError: + pass + else: + assert False, "invalid input did not fail to parse" + + et = etree.XML(''' ''') + try: + ns = next(etree.iterwalk(et, events=('start-ns',))) + except StopIteration: + # This would be the expected result, because there was no namespace + pass + else: + # This is a bug in libxml2 + assert not ns, repr(ns) + def test_itertext_comment_pi(self): # https://bugs.launchpad.net/lxml/+bug/1844674 XML = self.etree.XML From d65e63229e8958bc08344a85cd3f09ceeef933c3 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 1 Jul 2022 21:09:05 +0200 Subject: [PATCH 0003/1154] Prepare release of lxml 4.9.1. --- CHANGES.txt | 12 ++++++++++++ doc/main.txt | 10 +++++++--- src/lxml/__init__.py | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b2e0c8f03..64bba1c22 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,18 @@ lxml changelog ============== +4.9.1 (2022-07-01) +================== + +Bugs fixed +---------- + +* A crash was resolved when using ``iterwalk()`` (or ``canonicalize()``) + after parsing certain incorrect input. Note that ``iterwalk()`` can crash + on *valid* input parsed with the same parser *after* failing to parse the + incorrect input. + + 4.9.0 (2022-06-01) ================== diff --git a/doc/main.txt b/doc/main.txt index e9a0a4637..578f92dcf 100644 --- a/doc/main.txt +++ b/doc/main.txt @@ -160,8 +160,8 @@ Index `_ (PyPI). It has the source that compiles on various platforms. The source distribution is signed with `this key `_. -The latest version is `lxml 4.9.0`_, released 2022-06-01 -(`changes for 4.9.0`_). `Older versions <#old-versions>`_ +The latest version is `lxml 4.9.1`_, released 2022-07-01 +(`changes for 4.9.1`_). `Older versions <#old-versions>`_ are listed below. Please take a look at the @@ -256,7 +256,9 @@ See the websites of lxml .. and the `latest in-development version `_. -.. _`PDF documentation`: lxmldoc-4.9.0.pdf +.. _`PDF documentation`: lxmldoc-4.9.1.pdf + +* `lxml 4.9.1`_, released 2022-07-01 (`changes for 4.9.1`_) * `lxml 4.9.0`_, released 2022-06-01 (`changes for 4.9.0`_) @@ -280,6 +282,7 @@ See the websites of lxml * `older releases `_ +.. _`lxml 4.9.1`: /files/lxml-4.9.1.tgz .. _`lxml 4.9.0`: /files/lxml-4.9.0.tgz .. _`lxml 4.8.0`: /files/lxml-4.8.0.tgz .. _`lxml 4.7.1`: /files/lxml-4.7.1.tgz @@ -291,6 +294,7 @@ See the websites of lxml .. _`lxml 4.6.1`: /files/lxml-4.6.1.tgz .. _`lxml 4.6.0`: /files/lxml-4.6.0.tgz +.. _`changes for 4.9.1`: /changes-4.9.1.html .. _`changes for 4.9.0`: /changes-4.9.0.html .. _`changes for 4.8.0`: /changes-4.8.0.html .. _`changes for 4.7.1`: /changes-4.7.1.html diff --git a/src/lxml/__init__.py b/src/lxml/__init__.py index 0e0083413..f8be68f71 100644 --- a/src/lxml/__init__.py +++ b/src/lxml/__init__.py @@ -1,6 +1,6 @@ # this is a package -__version__ = "4.9.0" +__version__ = "4.9.1" def get_include(): From d01872ccdf7e1e5e825b6c6292b43e7d27ae5fc4 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 1 Jul 2022 21:19:44 +0200 Subject: [PATCH 0004/1154] Prevent parse failure in new test from leaking into later test runs. --- src/lxml/tests/test_etree.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lxml/tests/test_etree.py b/src/lxml/tests/test_etree.py index 285313f6e..3e52258ed 100644 --- a/src/lxml/tests/test_etree.py +++ b/src/lxml/tests/test_etree.py @@ -1463,14 +1463,16 @@ def test_iterwalk_getiterator(self): def test_walk_after_parse_failure(self): # This used to be an issue because libxml2 can leak empty namespaces # between failed parser runs. iterwalk() failed to handle such a tree. + parser = etree.XMLParser() + try: - etree.XML('''''') + etree.XML('''''', parser=parser) except etree.XMLSyntaxError: pass else: assert False, "invalid input did not fail to parse" - et = etree.XML(''' ''') + et = etree.XML(''' ''', parser=parser) try: ns = next(etree.iterwalk(et, events=('start-ns',))) except StopIteration: From 1ee0778d001a898c78ef548b2dc153c0ad885359 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 9 Jul 2022 09:12:32 +0200 Subject: [PATCH 0005/1154] Mention download count in readme. --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index a0434b379..dea167ba3 100644 --- a/README.rst +++ b/README.rst @@ -74,6 +74,9 @@ Another supporter of the lxml project is Project income report --------------------- +lxml has `more than 50 million downloads `_ +per month on PyPI. + * Total project income in 2021: EUR 4890.37 (407.53 € / month) - Tidelift: EUR 4066.66 From 05636df54bff39bb6017e8fa6b979458f61a7bd9 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sun, 10 Jul 2022 09:54:03 +0200 Subject: [PATCH 0006/1154] Include Cython source files to allow showing source lines in tracebacks. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 97dd973fe..fdbb76580 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,9 @@ def static_env_list(name, separator=None): 'etree_api.h', 'lxml.etree.h', 'lxml.etree_api.h', + # Include Cython source files for better traceback output. + '*.pyx', + '*.pxi', ], 'lxml.includes': [ '*.pxd', '*.h' From ff82753bd09408ecfd6cacf7da67257495dc1335 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 15 Jul 2022 08:40:19 +0200 Subject: [PATCH 0007/1154] LP#1981760: Register "Element.attrib" as collections.abc.MutableMapping. --- CHANGES.txt | 6 ++++++ src/lxml/etree.pyx | 5 ++++- src/lxml/tests/test_etree.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 64bba1c22..8a1174ea8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,12 @@ lxml changelog ============== +Under development +================= + +* LP#1981760: ``Element.attrib`` now registers as ``collections.abc.MutableMapping``. + + 4.9.1 (2022-07-01) ================== diff --git a/src/lxml/etree.pyx b/src/lxml/etree.pyx index 95dd21ee5..c0d236bd8 100644 --- a/src/lxml/etree.pyx +++ b/src/lxml/etree.pyx @@ -87,6 +87,7 @@ from itertools import islice cdef object ITER_EMPTY = iter(()) +cdef object MutableMapping try: from collections.abc import MutableMapping # Py3.3+ except ImportError: @@ -113,7 +114,7 @@ class _ImmutableMapping(MutableMapping): iterkeys = itervalues = iteritems = __iter__ cdef object IMMUTABLE_EMPTY_MAPPING = _ImmutableMapping() -del MutableMapping, _ImmutableMapping +del _ImmutableMapping # the rules @@ -2568,6 +2569,8 @@ cdef class _Attrib: return NotImplemented return python.PyObject_RichCompare(one, other, op) +MutableMapping.register(_Attrib) + @cython.final @cython.internal diff --git a/src/lxml/tests/test_etree.py b/src/lxml/tests/test_etree.py index 3e52258ed..8bf82c084 100644 --- a/src/lxml/tests/test_etree.py +++ b/src/lxml/tests/test_etree.py @@ -241,6 +241,18 @@ def test_clear_keep_tail(self): a[0].clear(keep_tail=True) self.assertEqual(_bytes('B2C1C2'), tostring(a)) + def test_attrib_is_Mapping(self): + try: + from collections.abc import Mapping, MutableMapping + except ImportError: + from collections import Mapping, MutableMapping # Py2 + + Element = self.etree.Element + root = Element("root") + + self.assertTrue(isinstance(root.attrib, Mapping)) + self.assertTrue(isinstance(root.attrib, MutableMapping)) + def test_attribute_has_key(self): # ET in Py 3.x has no "attrib.has_key()" method XML = self.etree.XML From f5d4694a2cd1e2ae5b18c0f463984c91a379b714 Mon Sep 17 00:00:00 2001 From: Henning Janssen Date: Fri, 15 Jul 2022 08:59:17 +0200 Subject: [PATCH 0008/1154] Correct the condition for building the docs in CI (GH-347) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51d77a4e4..5868a3af3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: run: bash ./tools/ci-run.sh - name: Build docs - if: contains( env.EXTRA_DEPS, 'sphinx') + if: contains( matrix.env.EXTRA_DEPS, 'sphinx') run: make html - name: Upload docs From 56de38e2745f01427c9cd8c518ba4f50731f388c Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 15 Jul 2022 09:12:13 +0200 Subject: [PATCH 0009/1154] Add missing apidoc files to sdist. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index f05c25735..1592ed3e7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,5 +15,6 @@ recursive-include src/lxml/html/tests *.data *.txt recursive-include samples *.xml recursive-include benchmark *.py recursive-include doc *.py *.txt *.html *.css *.xml *.mgp pubkey.asc Makefile +recursive-include doc/html/apidoc *.js *.png *.inv recursive-include doc/s5/ui *.gif *.htc *.png *.js recursive-include doc/s5/ep2008 *.py *.png *.rng From c742576c105f40fc8b754fcae56fee4aa35840a3 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 19 Jul 2022 08:25:20 +0200 Subject: [PATCH 0010/1154] Work around libxml2 bug in affected versions that failed to reset the namespace count in the parser context. See https://gitlab.gnome.org/GNOME/libxml2/-/issues/378 --- src/lxml/includes/xmlparser.pxd | 1 + src/lxml/parser.pxi | 3 +++ src/lxml/tests/test_etree.py | 3 +-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lxml/includes/xmlparser.pxd b/src/lxml/includes/xmlparser.pxd index a196e34bd..45acfc846 100644 --- a/src/lxml/includes/xmlparser.pxd +++ b/src/lxml/includes/xmlparser.pxd @@ -144,6 +144,7 @@ cdef extern from "libxml/parser.h": void* userData int* spaceTab int spaceMax + int nsNr bint html bint progressive int inSubset diff --git a/src/lxml/parser.pxi b/src/lxml/parser.pxi index f5baf29b9..f0c8c6b64 100644 --- a/src/lxml/parser.pxi +++ b/src/lxml/parser.pxi @@ -569,6 +569,9 @@ cdef class _ParserContext(_ResolverContext): self._c_ctxt.disableSAX = 0 # work around bug in libxml2 else: xmlparser.xmlClearParserCtxt(self._c_ctxt) + # work around bug in libxml2 [2.9.10 .. 2.9.14]: + # https://gitlab.gnome.org/GNOME/libxml2/-/issues/378 + self._c_ctxt.nsNr = 0 cdef int prepare(self, bint set_document_loader=True) except -1: cdef int result diff --git a/src/lxml/tests/test_etree.py b/src/lxml/tests/test_etree.py index 8bf82c084..0339796d6 100644 --- a/src/lxml/tests/test_etree.py +++ b/src/lxml/tests/test_etree.py @@ -1491,8 +1491,7 @@ def test_walk_after_parse_failure(self): # This would be the expected result, because there was no namespace pass else: - # This is a bug in libxml2 - assert not ns, repr(ns) + assert False, "Found unexpected namespace '%s'" % ns def test_itertext_comment_pi(self): # https://bugs.launchpad.net/lxml/+bug/1844674 From d8c7d97c11e4823ea39536e22dd4c757e226b5b9 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 13 Aug 2022 10:15:08 +0200 Subject: [PATCH 0011/1154] Improve the docstring about the "profile_run" XSLT() argument. --- src/lxml/xslt.pxi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lxml/xslt.pxi b/src/lxml/xslt.pxi index d483cfa30..559b277bc 100644 --- a/src/lxml/xslt.pxi +++ b/src/lxml/xslt.pxi @@ -353,7 +353,8 @@ cdef class XSLT: Keyword arguments of the XSLT call: - - profile_run: enable XSLT profiling (default: False) + - profile_run: enable XSLT profiling and make the profile available + as XML document in ``result.xslt_profile`` (default: False) Other keyword arguments of the call are passed to the stylesheet as parameters. From 7bafe60135c4bae61b250b5e236407a8e579be5e Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 30 Aug 2022 17:57:29 +0200 Subject: [PATCH 0012/1154] Keep libxml2 download working with libxml2 2.10.x when requesting a 2.9.x version. --- buildlibxml.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/buildlibxml.py b/buildlibxml.py index e0c558fad..b87697bf6 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -178,7 +178,7 @@ def _list_dir_urllib(url): return files -def http_find_latest_version_directory(url): +def http_find_latest_version_directory(url, version=None): with closing(urlopen(url)) as res: charset = _find_content_encoding(res) data = res.read() @@ -189,7 +189,13 @@ def http_find_latest_version_directory(url): ] if not directories: return url - latest_dir = "%s.%s" % max(directories) + best_version = max(directories) + if version: + major, minor, _ = version.split(".", 2) + major, minor = int(major), int(minor) + if (major, minor) in directories: + best_version = (major, minor) + latest_dir = "%s.%s" % best_version return urljoin(url, latest_dir) + "/" @@ -248,7 +254,7 @@ def download_libxml2(dest_dir, version=None): from_location = "https://gitlab.gnome.org/GNOME/libxml2/-/archive/dea91c97debeac7c1aaf9c19f79029809e23a353/" version = "dea91c97debeac7c1aaf9c19f79029809e23a353" else: - from_location = http_find_latest_version_directory(LIBXML2_LOCATION) + from_location = http_find_latest_version_directory(LIBXML2_LOCATION, version=version) return download_library(dest_dir, from_location, 'libxml2', version_re, filename, version=version) @@ -259,7 +265,7 @@ def download_libxslt(dest_dir, version=None): #version_re = re.compile(r'LATEST_LIBXSLT_IS_([0-9.]+[0-9](?:-[abrc0-9]+)?)') version_re = re.compile(r'libxslt-([0-9.]+[0-9]).tar.xz') filename = 'libxslt-%s.tar.xz' - from_location = http_find_latest_version_directory(LIBXSLT_LOCATION) + from_location = http_find_latest_version_directory(LIBXSLT_LOCATION, version=version) return download_library(dest_dir, from_location, 'libxslt', version_re, filename, version=version) From 95f8006336889df154c45130db1ec897b424e529 Mon Sep 17 00:00:00 2001 From: Maximilian Date: Fri, 16 Sep 2022 12:55:48 +0200 Subject: [PATCH 0013/1154] Use proper SPDX license identifier in setup.py script (GH-351) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fdbb76580..a9e4c20d4 100644 --- a/setup.py +++ b/setup.py @@ -211,7 +211,7 @@ def build_packages(files): author_email="lxml-dev@lxml.de", maintainer="lxml dev team", maintainer_email="lxml-dev@lxml.de", - license="BSD", + license="BSD-3-Clause", url="https://lxml.de/", # Commented out because this causes distutils to emit warnings # `Unknown distribution option: 'bugtrack_url'` From 0b10088e4a358d5153defee68c1f7dfd2c06f18b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 24 Oct 2022 16:17:48 +0200 Subject: [PATCH 0014/1154] Turn buildlibxml.py into a download/build script, also for debugging platform specific issues. --- buildlibxml.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/buildlibxml.py b/buildlibxml.py index b87697bf6..5a7a4c64b 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -1,6 +1,6 @@ import os, re, sys, subprocess, platform import tarfile -from distutils import log, version +from distutils import log from contextlib import closing, contextmanager from ftplib import FTP @@ -23,6 +23,10 @@ pass +# overridable to control script usage +sys_platform = sys.platform + + # use pre-built libraries on Windows def download_and_extract_windows_binaries(destdir): @@ -109,7 +113,7 @@ def unpack_zipfile(zipfn, destdir): def get_prebuilt_libxml2xslt(download_dir, static_include_dirs, static_library_dirs): - assert sys.platform.startswith('win') + assert sys_platform.startswith('win') libs = download_and_extract_windows_binaries(download_dir) for libname, path in libs.items(): i = os.path.join(path, 'include') @@ -450,7 +454,7 @@ def has_current_lib(name, build_dir, _build_all_following=[False]): return found call_setup = {} - if sys.platform == 'darwin': + if sys_platform == 'darwin': configure_darwin_env(call_setup) configure_cmd = ['./configure', @@ -531,3 +535,27 @@ def has_current_lib(name, build_dir, _build_all_following=[False]): if lib in filename and filename.endswith('.a')] return xml2_config, xslt_config + + +def main(): + static_include_dirs = [] + static_library_dirs = [] + download_dir = "libs" + + if sys_platform.startswith('win'): + return get_prebuilt_libxml2xslt( + download_dir, static_include_dirs, static_library_dirs) + else: + return build_libxml2xslt( + download_dir, 'build/tmp', + static_include_dirs, static_library_dirs, + static_cflags=[], + static_binaries=[] + ) + + +if __name__ == '__main__': + if len(sys.argv) > 1: + # change global sys_platform setting + sys_platform = sys.argv[1] + main() From 77413c28e237f3f4bf2363150aaab9d2f7705f7b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 24 Oct 2022 17:19:14 +0200 Subject: [PATCH 0015/1154] Resolve build failures on appveyor/Windows due to incomplete (abbreviated) download lists of github releases. Closes https://bugs.launchpad.net/lxml/+bug/1993962 --- buildlibxml.py | 62 +++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/buildlibxml.py b/buildlibxml.py index 5a7a4c64b..8a7504ebb 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -1,3 +1,4 @@ +import json import os, re, sys, subprocess, platform import tarfile from distutils import log @@ -5,11 +6,12 @@ from ftplib import FTP try: - from urlparse import urljoin, unquote, urlparse - from urllib import urlretrieve, urlopen, urlcleanup -except ImportError: from urllib.parse import urljoin, unquote, urlparse - from urllib.request import urlretrieve, urlopen, urlcleanup + from urllib.request import urlretrieve, urlopen, urlcleanup, Request +except ImportError: # Py2 + from urlparse import urljoin, unquote, urlparse + from urllib import urlretrieve, urlcleanup + from urllib2 import urlopen, Request multi_make_options = [] try: @@ -30,17 +32,16 @@ # use pre-built libraries on Windows def download_and_extract_windows_binaries(destdir): - url = "https://github.com/lxml/libxml2-win-binaries/releases" - filenames = list(_list_dir_urllib(url)) - - release_path = "/download/%s/" % find_max_version( - "library release", filenames, re.compile(r"/releases/tag/([0-9.]+[0-9])$")) - url += release_path - filenames = [ - filename.rsplit('/', 1)[1] - for filename in filenames - if release_path in filename - ] + url = "https://api.github.com/repos/lxml/libxml2-win-binaries/releases" + releases, _ = read_url(url, accept="application/vnd.github+json", as_json=True) + + max_release = {'tag_name': ''} + for release in releases: + if max_release['tag_name'] < release.get('tag_name', ''): + max_release = release + + url = "https://github.com/lxml/libxml2-win-binaries/releases/download/%s/" % max_release['tag_name'] + filenames = [asset['name'] for asset in max_release.get('assets', ())] # Check for native ARM64 build or the environment variable that is set by # Visual Studio for cross-compilation (same variable as setuptools uses) @@ -168,13 +169,26 @@ def _list_dir_ftplib(url): return parse_text_ftplist("\n".join(data)) -def _list_dir_urllib(url): - with closing(urlopen(url)) as res: +def read_url(url, decode=True, accept=None, as_json=False): + if accept: + request = Request(url, headers={'Accept': accept}) + else: + request = Request(url) + + with closing(urlopen(request)) as res: charset = _find_content_encoding(res) content_type = res.headers.get('Content-Type') data = res.read() - data = data.decode(charset) + if decode: + data = data.decode(charset) + if as_json: + data = json.loads(data) + return data, content_type + + +def _list_dir_urllib(url): + data, content_type = read_url(url) if content_type and content_type.startswith('text/html'): files = parse_html_filelist(data) else: @@ -183,13 +197,11 @@ def _list_dir_urllib(url): def http_find_latest_version_directory(url, version=None): - with closing(urlopen(url)) as res: - charset = _find_content_encoding(res) - data = res.read() + data, _ = read_url(url) # e.g. directories = [ (int(v[0]), int(v[1])) - for v in re.findall(r' href=["\']([0-9]+)\.([0-9]+)/?["\']', data.decode(charset)) + for v in re.findall(r' href=["\']([0-9]+)\.([0-9]+)/?["\']', data) ] if not directories: return url @@ -204,10 +216,8 @@ def http_find_latest_version_directory(url, version=None): def http_listfiles(url, re_pattern): - with closing(urlopen(url)) as res: - charset = _find_content_encoding(res) - data = res.read() - files = re.findall(re_pattern, data.decode(charset)) + data, _ = read_url(url) + files = re.findall(re_pattern, data) return files From 1f1927fd618cae32e010c8c303dc9c44a319de1c Mon Sep 17 00:00:00 2001 From: Quentin Leffray Date: Mon, 7 Nov 2022 10:27:54 +0100 Subject: [PATCH 0016/1154] Provide the correct C compiler configuration for local wheel building on ARM Mac (GH-359) --- buildlibxml.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/buildlibxml.py b/buildlibxml.py index 8a7504ebb..8faf59378 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -416,11 +416,18 @@ def configure_darwin_env(env_setup): # configure target architectures on MacOS-X (x86_64 only, by default) major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) if major_version > 7: - env_default = { - 'CFLAGS': "-arch x86_64 -O2", - 'LDFLAGS': "-arch x86_64", - 'MACOSX_DEPLOYMENT_TARGET': "10.6" - } + if platform.mac_ver()[2] == "arm64": + env_default = { + 'CFLAGS': "-arch arm64 -O2", + 'LDFLAGS': "-arch arm64", + 'MACOSX_DEPLOYMENT_TARGET': "10.6" + } + else: + env_default = { + 'CFLAGS': "-arch x86_64 -O2", + 'LDFLAGS': "-arch x86_64", + 'MACOSX_DEPLOYMENT_TARGET': "10.6" + } env_default.update(os.environ) env_setup['env'] = env_default From 36201eccc024cd2ead7549998da2857256ed61c3 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 8 Nov 2022 08:38:37 +0100 Subject: [PATCH 0017/1154] Remove outdated files. --- DD.py | 916 ---------------------------------------------- bisect_crashes.py | 66 ---- 2 files changed, 982 deletions(-) delete mode 100644 DD.py delete mode 100644 bisect_crashes.py diff --git a/DD.py b/DD.py deleted file mode 100644 index 47dfec767..000000000 --- a/DD.py +++ /dev/null @@ -1,916 +0,0 @@ -#! /usr/bin/env python -# $Id: DD.py,v 1.2 2001/11/05 19:53:33 zeller Exp $ -# Enhanced Delta Debugging class -# Copyright (c) 1999, 2000, 2001 Andreas Zeller. - -# This module (written in Python) implements the base delta debugging -# algorithms and is at the core of all our experiments. This should -# easily run on any platform and any Python version since 1.6. -# -# To plug this into your system, all you have to do is to create a -# subclass with a dedicated `test()' method. Basically, you would -# invoke the DD test case minimization algorithm (= the `ddmin()' -# method) with a list of characters; the `test()' method would combine -# them to a document and run the test. This should be easy to realize -# and give you some good starting results; the file includes a simple -# sample application. -# -# This file is in the public domain; feel free to copy, modify, use -# and distribute this software as you wish - with one exception. -# Passau University has filed a patent for the use of delta debugging -# on program states (A. Zeller: `Isolating cause-effect chains', -# Saarland University, 2001). The fact that this file is publicly -# available does not imply that I or anyone else grants you any rights -# related to this patent. -# -# The use of Delta Debugging to isolate failure-inducing code changes -# (A. Zeller: `Yesterday, my program worked', ESEC/FSE 1999) or to -# simplify failure-inducing input (R. Hildebrandt, A. Zeller: -# `Simplifying failure-inducing input', ISSTA 2000) is, as far as I -# know, not covered by any patent, nor will it ever be. If you use -# this software in any way, I'd appreciate if you include a citation -# such as `This software uses the delta debugging algorithm as -# described in (insert one of the papers above)'. -# -# All about Delta Debugging is found at the delta debugging web site, -# -# http://www.st.cs.uni-sb.de/dd/ -# -# Happy debugging, -# -# Andreas Zeller - - -# Start with some helpers. -class OutcomeCache(object): - # This class holds test outcomes for configurations. This avoids - # running the same test twice. - - # The outcome cache is implemented as a tree. Each node points - # to the outcome of the remaining list. - # - # Example: ([1, 2, 3], PASS), ([1, 2], FAIL), ([1, 4, 5], FAIL): - # - # (2, FAIL)--(3, PASS) - # / - # (1, None) - # \ - # (4, None)--(5, FAIL) - - def __init__(self): - self.tail = {} # Points to outcome of tail - self.result = None # Result so far - - def add(self, c, result): - """Add (C, RESULT) to the cache. C must be a list of scalars.""" - cs = c[:] - cs.sort() - - p = self - for start in c: - if start not in p.tail: - p.tail[start] = OutcomeCache() - p = p.tail[start] - - p.result = result - - def lookup(self, c): - """Return RESULT if (C, RESULT) is in the cache; None, otherwise.""" - p = self - for start in c: - if start not in p.tail: - return None - p = p.tail[start] - - return p.result - - def lookup_superset(self, c, start = 0): - """Return RESULT if there is some (C', RESULT) in the cache with - C' being a superset of C or equal to C. Otherwise, return None.""" - - # FIXME: Make this non-recursive! - if start >= len(c): - if self.result: - return self.result - elif self.tail != {}: - # Select some superset - superset = self.tail[list(self.tail.keys())[0]] - return superset.lookup_superset(c, start + 1) - else: - return None - - if c[start] in self.tail: - return self.tail[c[start]].lookup_superset(c, start + 1) - - # Let K0 be the largest element in TAIL such that K0 <= C[START] - k0 = None - for k in self.tail.keys(): - if (k0 is None or k > k0) and k <= c[start]: - k0 = k - - if k0 is not None: - return self.tail[k0].lookup_superset(c, start) - - return None - - def lookup_subset(self, c): - """Return RESULT if there is some (C', RESULT) in the cache with - C' being a subset of C or equal to C. Otherwise, return None.""" - p = self - for start in range(len(c)): - if c[start] in p.tail: - p = p.tail[c[start]] - - return p.result - - - - -# Test the outcome cache -def oc_test(): - oc = OutcomeCache() - - assert oc.lookup([1, 2, 3]) is None - oc.add([1, 2, 3], 4) - assert oc.lookup([1, 2, 3]) == 4 - assert oc.lookup([1, 2, 3, 4]) is None - - assert oc.lookup([5, 6, 7]) is None - oc.add([5, 6, 7], 8) - assert oc.lookup([5, 6, 7]) == 8 - - assert oc.lookup([]) is None - oc.add([], 0) - assert oc.lookup([]) == 0 - - assert oc.lookup([1, 2]) is None - oc.add([1, 2], 3) - assert oc.lookup([1, 2]) == 3 - assert oc.lookup([1, 2, 3]) == 4 - - assert oc.lookup_superset([1]) == 3 or oc.lookup_superset([1]) == 4 - assert oc.lookup_superset([1, 2]) == 3 or oc.lookup_superset([1, 2]) == 4 - assert oc.lookup_superset([5]) == 8 - assert oc.lookup_superset([5, 6]) == 8 - assert oc.lookup_superset([6, 7]) == 8 - assert oc.lookup_superset([7]) == 8 - assert oc.lookup_superset([]) is not None - - assert oc.lookup_superset([9]) is None - assert oc.lookup_superset([7, 9]) is None - assert oc.lookup_superset([-5, 1]) is None - assert oc.lookup_superset([1, 2, 3, 9]) is None - assert oc.lookup_superset([4, 5, 6, 7]) is None - - assert oc.lookup_subset([]) == 0 - assert oc.lookup_subset([1, 2, 3]) == 4 - assert oc.lookup_subset([1, 2, 3, 4]) == 4 - assert oc.lookup_subset([1, 3]) is None - assert oc.lookup_subset([1, 2]) == 3 - - assert oc.lookup_subset([-5, 1]) is None - assert oc.lookup_subset([-5, 1, 2]) == 3 - assert oc.lookup_subset([-5]) == 0 - - -# Main Delta Debugging algorithm. -class DD(object): - # Delta debugging base class. To use this class for a particular - # setting, create a subclass with an overloaded `test()' method. - # - # Main entry points are: - # - `ddmin()' which computes a minimal failure-inducing configuration, and - # - `dd()' which computes a minimal failure-inducing difference. - # - # See also the usage sample at the end of this file. - # - # For further fine-tuning, you can implement an own `resolve()' - # method (tries to add or remove configuration elements in case of - # inconsistencies), or implement an own `split()' method, which - # allows you to split configurations according to your own - # criteria. - # - # The class includes other previous delta debugging algorithms, - # which are obsolete now; they are only included for comparison - # purposes. - - # Test outcomes. - PASS = "PASS" - FAIL = "FAIL" - UNRESOLVED = "UNRESOLVED" - - # Resolving directions. - ADD = "ADD" # Add deltas to resolve - REMOVE = "REMOVE" # Remove deltas to resolve - - # Debugging output (set to 1 to enable) - debug_test = 0 - debug_dd = 0 - debug_split = 0 - debug_resolve = 0 - - def __init__(self): - self.__resolving = 0 - self.__last_reported_length = 0 - self.monotony = 0 - self.outcome_cache = OutcomeCache() - self.cache_outcomes = 1 - self.minimize = 1 - self.maximize = 1 - self.assume_axioms_hold = 1 - - # Helpers - def __listminus(self, c1, c2): - """Return a list of all elements of C1 that are not in C2.""" - s2 = {} - for delta in c2: - s2[delta] = 1 - - c = [] - for delta in c1: - if delta not in s2: - c.append(delta) - - return c - - def __listintersect(self, c1, c2): - """Return the common elements of C1 and C2.""" - s2 = {} - for delta in c2: - s2[delta] = 1 - - c = [] - for delta in c1: - if delta in s2: - c.append(delta) - - return c - - def __listunion(self, c1, c2): - """Return the union of C1 and C2.""" - s1 = {} - for delta in c1: - s1[delta] = 1 - - c = c1[:] - for delta in c2: - if delta not in s1: - c.append(delta) - - return c - - def __listsubseteq(self, c1, c2): - """Return 1 if C1 is a subset or equal to C2.""" - s2 = {} - for delta in c2: - s2[delta] = 1 - - for delta in c1: - if delta not in s2: - return 0 - - return 1 - - # Output - def coerce(self, c): - """Return the configuration C as a compact string""" - # Default: use printable representation - return repr(c) - - def pretty(self, c): - """Like coerce(), but sort beforehand""" - sorted_c = c[:] - sorted_c.sort() - return self.coerce(sorted_c) - - # Testing - def test(self, c): - """Test the configuration C. Return PASS, FAIL, or UNRESOLVED""" - c.sort() - - # If we had this test before, return its result - if self.cache_outcomes: - cached_result = self.outcome_cache.lookup(c) - if cached_result is not None: - return cached_result - - if self.monotony: - # Check whether we had a passing superset of this test before - cached_result = self.outcome_cache.lookup_superset(c) - if cached_result == self.PASS: - return self.PASS - - cached_result = self.outcome_cache.lookup_subset(c) - if cached_result == self.FAIL: - return self.FAIL - - if self.debug_test: - print('') - print("test(%s)..." % (self.coerce(c),)) - - outcome = self._test(c) - - if self.debug_test: - print("test(%s) = %r" % (self.coerce(c), outcome)) - - if self.cache_outcomes: - self.outcome_cache.add(c, outcome) - - return outcome - - def _test(self, c): - """Stub to overload in subclasses""" - return self.UNRESOLVED # Placeholder - - - # Splitting - def split(self, c, n): - """Split C into [C_1, C_2, ..., C_n].""" - if self.debug_split: - print("split(%s, %r)..." % (self.coerce(c), n)) - - outcome = self._split(c, n) - - if self.debug_split: - print("split(%s, %r) = %r" % (self.coerce(c), n, outcome)) - - return outcome - - def _split(self, c, n): - """Stub to overload in subclasses""" - subsets = [] - start = 0 - for i in range(n): - subset = c[start:start + (len(c) - start) // (n - i)] - subsets.append(subset) - start = start + len(subset) - return subsets - - - # Resolving - def resolve(self, csub, c, direction): - """If direction == ADD, resolve inconsistency by adding deltas - to CSUB. Otherwise, resolve by removing deltas from CSUB.""" - - if self.debug_resolve: - print("resolve(%r, %s, %r)..." % (csub, self.coerce(c), direction)) - - outcome = self._resolve(csub, c, direction) - - if self.debug_resolve: - print("resolve(%r, %s, %r) = %r" % (csub, self.coerce(c), direction, outcome)) - - return outcome - - - def _resolve(self, csub, c, direction): - """Stub to overload in subclasses.""" - # By default, no way to resolve - return None - - - # Test with fixes - def test_and_resolve(self, csub, r, c, direction): - """Repeat testing CSUB + R while unresolved.""" - - initial_csub = csub[:] - c2 = self.__listunion(r, c) - - csubr = self.__listunion(csub, r) - t = self.test(csubr) - - # necessary to use more resolving mechanisms which can reverse each - # other, can (but needn't) be used in subclasses - self._resolve_type = 0 - - while t == self.UNRESOLVED: - self.__resolving = 1 - csubr = self.resolve(csubr, c, direction) - - if csubr is None: - # Nothing left to resolve - break - - if len(csubr) >= len(c2): - # Added everything: csub == c2. ("Upper" Baseline) - # This has already been tested. - csubr = None - break - - if len(csubr) <= len(r): - # Removed everything: csub == r. (Baseline) - # This has already been tested. - csubr = None - break - - t = self.test(csubr) - - self.__resolving = 0 - if csubr is None: - return self.UNRESOLVED, initial_csub - - # assert t == self.PASS or t == self.FAIL - csub = self.__listminus(csubr, r) - return t, csub - - # Inquiries - def resolving(self): - """Return 1 while resolving.""" - return self.__resolving - - - # Logging - def report_progress(self, c, title): - if len(c) != self.__last_reported_length: - print('') - print("%s: %d deltas left: %s" % (title, len(c), self.coerce(c))) - self.__last_reported_length = len(c) - - - # Delta Debugging (old ESEC/FSE version) - def old_dd(self, c, r = [], n = 2): - """Return the failure-inducing subset of C""" - - assert self.test([]) == dd.PASS - assert self.test(c) == dd.FAIL - - if self.debug_dd: - print("dd(%s, %r, %r)..." % (self.pretty(c), r, n)) - - outcome = self._old_dd(c, r, n) - - if self.debug_dd: - print("dd(%s, %r, %r) = %r" % (self.pretty(c), r, n, outcome)) - - return outcome - - def _old_dd(self, c, r, n): - """Stub to overload in subclasses""" - - if not r: - assert self.test([]) == self.PASS - assert self.test(c) == self.FAIL - else: - assert self.test(r) != self.FAIL - assert self.test(c + r) != self.PASS - - assert self.__listintersect(c, r) == [] - - if len(c) == 1: - # Nothing to split - return c - - run = 1 - next_c = c[:] - next_r = r[:] - - # We replace the tail recursion from the paper by a loop - while 1: - self.report_progress(c, "dd") - - cs = self.split(c, n) - - print('') - print("dd (run #%r): trying %s" % (run, ' + '.join(map(str, cs)))) - print('') - - # Check subsets - ts = [] - for i in range(n): - if self.debug_dd: - print("dd: trying cs[%d] = %s" % (i, self.pretty(cs[i]))) - - t, cs[i] = self.test_and_resolve(cs[i], r, c, self.REMOVE) - ts.append(t) - if t == self.FAIL: - # Found - if self.debug_dd: - print("dd: found %d deltas: %s" % (len(cs[i]), self.pretty(cs[i]))) - return self.dd(cs[i], r) - - # Check complements - cbars = [] - tbars = [] - - for i in range(n): - cbar = self.__listminus(c, cs[i] + r) - tbar, cbar = self.test_and_resolve(cbar, r, c, self.ADD) - - - doubled = self.__listintersect(cbar, cs[i]) - if doubled: - cs[i] = self.__listminus(cs[i], doubled) - - - cbars.append(cbar) - tbars.append(tbar) - - if ts[i] == self.PASS and tbars[i] == self.PASS: - # Interference - if self.debug_dd: - print("dd: interference of %s and %s" % (self.pretty(cs[i]), self.pretty(cbars[i]))) - - d = self.dd(cs[i][:], cbars[i] + r) - dbar = self.dd(cbars[i][:], cs[i] + r) - return d + dbar - - if ts[i] == self.UNRESOLVED and tbars[i] == self.PASS: - # Preference - if self.debug_dd: - print("dd: preferring %d deltas: %s" % (len(cs[i]), self.pretty(cs[i]))) - - return self.dd(cs[i][:], cbars[i] + r) - - if ts[i] == self.PASS or tbars[i] == self.FAIL: - if self.debug_dd: - excluded = self.__listminus(next_c, cbars[i]) - print("dd: excluding %d deltas: %s" % (len(excluded), self.pretty(excluded))) - - if ts[i] == self.PASS: - next_r = self.__listunion(next_r, cs[i]) - next_c = self.__listintersect(next_c, cbars[i]) - self.report_progress(next_c, "dd") - - next_n = min(len(next_c), n * 2) - - if next_n == n and next_c[:] == c[:] and next_r[:] == r[:]: - # Nothing left - if self.debug_dd: - print("dd: nothing left") - return next_c - - # Try again - if self.debug_dd: - print("dd: try again") - - c = next_c - r = next_r - n = next_n - run = run + 1 - - - def test_mix(self, csub, c, direction): - if self.minimize: - (t, csub) = self.test_and_resolve(csub, [], c, direction) - if t == self.FAIL: - return t, csub - - if self.maximize: - csubbar = self.__listminus(self.CC, csub) - cbar = self.__listminus(self.CC, c) - if direction == self.ADD: - directionbar = self.REMOVE - else: - directionbar = self.ADD - - (tbar, csubbar) = self.test_and_resolve(csubbar, [], cbar, - directionbar) - - csub = self.__listminus(self.CC, csubbar) - - if tbar == self.PASS: - t = self.FAIL - elif tbar == self.FAIL: - t = self.PASS - else: - t = self.UNRESOLVED - - return t, csub - - - # Delta Debugging (new ISSTA version) - def ddgen(self, c, minimize, maximize): - """Return a 1-minimal failing subset of C""" - - self.minimize = minimize - self.maximize = maximize - - n = 2 - self.CC = c - - if self.debug_dd: - print("dd(%s, %r)..." % (self.pretty(c), n)) - - outcome = self._dd(c, n) - - if self.debug_dd: - print("dd(%s, %r) = %r" % (self.pretty(c), n, outcome)) - - return outcome - - def _dd(self, c, n): - """Stub to overload in subclasses""" - - assert self.test([]) == self.PASS - - run = 1 - cbar_offset = 0 - - # We replace the tail recursion from the paper by a loop - while 1: - tc = self.test(c) - assert tc == self.FAIL or tc == self.UNRESOLVED - - if n > len(c): - # No further minimizing - print("dd: done") - return c - - self.report_progress(c, "dd") - - cs = self.split(c, n) - - print('') - print("dd (run #%d): trying %s" % (run, ' + '.join(map(str, cs)))) - print('') - - c_failed = 0 - cbar_failed = 0 - - next_c = c[:] - next_n = n - - # Check subsets - for i in range(n): - if self.debug_dd: - print("dd: trying %s" % (self.pretty(cs[i]),)) - - (t, cs[i]) = self.test_mix(cs[i], c, self.REMOVE) - - if t == self.FAIL: - # Found - if self.debug_dd: - print("dd: found %d deltas: %s" % (len(cs[i]), self.pretty(cs[i]))) - - c_failed = 1 - next_c = cs[i] - next_n = 2 - cbar_offset = 0 - self.report_progress(next_c, "dd") - break - - if not c_failed: - # Check complements - cbars = n * [self.UNRESOLVED] - - # print "cbar_offset =", cbar_offset - - for j in range(n): - i = int((j + cbar_offset) % n) - cbars[i] = self.__listminus(c, cs[i]) - t, cbars[i] = self.test_mix(cbars[i], c, self.ADD) - - doubled = self.__listintersect(cbars[i], cs[i]) - if doubled: - cs[i] = self.__listminus(cs[i], doubled) - - if t == self.FAIL: - if self.debug_dd: - print("dd: reduced to %d deltas: %s" % (len(cbars[i]), self.pretty(cbars[i]))) - - cbar_failed = 1 - next_c = self.__listintersect(next_c, cbars[i]) - next_n = next_n - 1 - self.report_progress(next_c, "dd") - - # In next run, start removing the following subset - cbar_offset = i - break - - if not c_failed and not cbar_failed: - if n >= len(c): - # No further minimizing - print("dd: done") - return c - - next_n = min(len(c), n * 2) - print("dd: increase granularity to %d" % next_n) - cbar_offset = (cbar_offset * next_n) / n - - c = next_c - n = next_n - run = run + 1 - - def ddmin(self, c): - return self.ddgen(c, 1, 0) - - def ddmax(self, c): - return self.ddgen(c, 0, 1) - - def ddmix(self, c): - return self.ddgen(c, 1, 1) - - - # General delta debugging (new TSE version) - def dddiff(self, c): - n = 2 - - if self.debug_dd: - print("dddiff(%s, %d)..." % (self.pretty(c), n)) - - outcome = self._dddiff([], c, n) - - if self.debug_dd: - print("dddiff(%s, %d) = %r" % (self.pretty(c), n, outcome)) - - return outcome - - def _dddiff(self, c1, c2, n): - run = 1 - cbar_offset = 0 - - # We replace the tail recursion from the paper by a loop - while 1: - if self.debug_dd: - print("dd: c1 = %s" % (self.pretty(c1),)) - print("dd: c2 = %s" % (self.pretty(c2),)) - - if self.assume_axioms_hold: - t1 = self.PASS - t2 = self.FAIL - else: - t1 = self.test(c1) - t2 = self.test(c2) - - assert t1 == self.PASS - assert t2 == self.FAIL - assert self.__listsubseteq(c1, c2) - - c = self.__listminus(c2, c1) - - if self.debug_dd: - print("dd: c2 - c1 = %s" % (self.pretty(c),)) - - if n > len(c): - # No further minimizing - print("dd: done") - return c, c1, c2 - - self.report_progress(c, "dd") - - cs = self.split(c, n) - - print('') - print("dd (run #%d): trying %s" % (run, ' + '.join(map(str, cs)))) - print('') - - progress = 0 - - next_c1 = c1[:] - next_c2 = c2[:] - next_n = n - - # Check subsets - for j in range(n): - i = int((j + cbar_offset) % n) - - if self.debug_dd: - print("dd: trying %s" % (self.pretty(cs[i]),)) - - (t, csub) = self.test_and_resolve(cs[i], c1, c, self.REMOVE) - csub = self.__listunion(c1, csub) - - if t == self.FAIL and t1 == self.PASS: - # Found - progress = 1 - next_c2 = csub - next_n = 2 - cbar_offset = 0 - - if self.debug_dd: - print("dd: reduce c2 to %d deltas: %s" % (len(next_c2), self.pretty(next_c2))) - break - - if t == self.PASS and t2 == self.FAIL: - # Reduce to complement - progress = 1 - next_c1 = csub - next_n = max(next_n - 1, 2) - cbar_offset = i - - if self.debug_dd: - print("dd: increase c1 to %d deltas: %s", (len(next_c1), self.pretty(next_c1))) - break - - - csub = self.__listminus(c, cs[i]) - (t, csub) = self.test_and_resolve(csub, c1, c, self.ADD) - csub = self.__listunion(c1, csub) - - if t == self.PASS and t2 == self.FAIL: - # Found - progress = 1 - next_c1 = csub - next_n = 2 - cbar_offset = 0 - - if self.debug_dd: - print("dd: increase c1 to %d deltas: %s" % (len(next_c1), self.pretty(next_c1))) - break - - if t == self.FAIL and t1 == self.PASS: - # Increase - progress = 1 - next_c2 = csub - next_n = max(next_n - 1, 2) - cbar_offset = i - - if self.debug_dd: - print("dd: reduce c2 to %d deltas: %s" % (len(next_c2), self.pretty(next_c2))) - break - - if progress: - self.report_progress(self.__listminus(next_c2, next_c1), "dd") - else: - if n >= len(c): - # No further minimizing - print("dd: done") - return c, c1, c2 - - next_n = min(len(c), n * 2) - print("dd: increase granularity to %d" % next_n) - cbar_offset = (cbar_offset * next_n) / n - - c1 = next_c1 - c2 = next_c2 - n = next_n - run = run + 1 - - def dd(self, c): - return self.dddiff(c) # Backwards compatibility - - - - - -if __name__ == '__main__': - # Test the outcome cache - oc_test() - - # Define our own DD class, with its own test method - class MyDD(DD): - def _test_a(self, c): - "Test the configuration C. Return PASS, FAIL, or UNRESOLVED." - - # Just a sample - # if 2 in c and not 3 in c: - # return self.UNRESOLVED - # if 3 in c and not 7 in c: - # return self.UNRESOLVED - if 7 in c and not 2 in c: - return self.UNRESOLVED - if 5 in c and 8 in c: - return self.FAIL - return self.PASS - - def _test_b(self, c): - if not c: - return self.PASS - if 1 in c and 2 in c and 3 in c and 4 in c and \ - 5 in c and 6 in c and 7 in c and 8 in c: - return self.FAIL - return self.UNRESOLVED - - def _test_c(self, c): - if 1 in c and 2 in c and 3 in c and 4 in c and \ - 6 in c and 8 in c: - if 5 in c and 7 in c: - return self.UNRESOLVED - else: - return self.FAIL - if 1 in c or 2 in c or 3 in c or 4 in c or \ - 6 in c or 8 in c: - return self.UNRESOLVED - return self.PASS - - def __init__(self): - self._test = self._test_c - DD.__init__(self) - - - print("WYNOT - a tool for delta debugging.") - mydd = MyDD() - # mydd.debug_test = 1 # Enable debugging output - # mydd.debug_dd = 1 # Enable debugging output - # mydd.debug_split = 1 # Enable debugging output - # mydd.debug_resolve = 1 # Enable debugging output - - # mydd.cache_outcomes = 0 - # mydd.monotony = 0 - - print("Minimizing failure-inducing input...") - c = mydd.ddmin([1, 2, 3, 4, 5, 6, 7, 8]) # Invoke DDMIN - print("The 1-minimal failure-inducing input is %s" % (c,)) - print("Removing any element will make the failure go away.") - print('') - - print("Computing the failure-inducing difference...") - (c, c1, c2) = mydd.dd([1, 2, 3, 4, 5, 6, 7, 8]) # Invoke DD - print("The 1-minimal failure-inducing difference is %s" % (c,)) - print("%s passes, %s fails" % (c1, c2)) - - - -# Local Variables: -# mode: python -# End: diff --git a/bisect_crashes.py b/bisect_crashes.py deleted file mode 100644 index 7a3fe6cf6..000000000 --- a/bisect_crashes.py +++ /dev/null @@ -1,66 +0,0 @@ - -import os -import sys -import unittest - -# make sure we import test.py from the right place -script_path = os.path.abspath(os.path.dirname(sys.argv[0])) -sys.path.insert(0, script_path) - -test_base_path = os.path.join(script_path, 'src') -sys.path.insert(1, test_base_path) - -import test -from DD import DD - -cfg = test.Options() -cfg.verbosity = 0 -cfg.basedir = test_base_path -cfg.unit_tests = True - -def write(line, *args): - if args: - line = line % args - sys.stderr.write(line + '\n') - - -def find_tests(): - test_files = test.get_test_files(cfg) - return test.get_test_cases(test_files, cfg) - -class DDTester(DD): - def _test(self, test_cases): - if not test_cases: - return self.PASS - write('Running subset of %d tests %s', - len(test_cases), self.coerce(test_cases)) - test_cases = [ item[-1] for item in test_cases ] - pid = os.fork() - if not pid: - # child executes tests - runner = test.CustomTestRunner(cfg, None) - suite = unittest.TestSuite() - suite.addTests(test_cases) - os._exit( not runner.run(suite).wasSuccessful() ) - cid, retval = os.waitpid(pid, 0) - if retval: - write('exit status: %d, signal: %d', retval >> 8, retval % 0xFF) - if (retval % 0xFF) > 2: # signal received? - return self.FAIL - return self.PASS - - def coerce(self, test_cases): - if not test_cases: - return '[]' - test_cases = [ item[-1] for item in test_cases ] - return '[%s .. %s]' % (test_cases[0].id(), test_cases[-1].id()) - -def dd_tests(): - tests = find_tests() - write('Found %d tests', len(tests)) - dd = DDTester() - min_tests = dd.ddmin( list(enumerate(tests)) ) - return [ item[-1] for item in min_tests ] - -if __name__ == '__main__': - write('Failing tests:\n%s', '\n'.join([test.id() for test in dd_tests()])) From 2ca71bd8a1f26635e003a47dd045b0089b8f942c Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 8 Nov 2022 08:39:48 +0100 Subject: [PATCH 0018/1154] Add intersphinx docs mapping. --- doc/api/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/api/conf.py b/doc/api/conf.py index 7c5f134d2..29db736a0 100644 --- a/doc/api/conf.py +++ b/doc/api/conf.py @@ -51,6 +51,10 @@ autodoc_member_order = 'groupwise' +intersphinx_mapping = { + "lxml": ("https://lxml.de/apidoc/", None), +} + # -- Options for todo extension ---------------------------------------------- # If true, `todo` and `todoList` produce output, else they produce nothing. From 0d5c72cc7d7f361221164ed379b525c5e1109f1f Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 8 Nov 2022 09:04:02 +0100 Subject: [PATCH 0019/1154] Add script to retrieve PyPI download stats by Python version. --- tools/pypistats.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tools/pypistats.py diff --git a/tools/pypistats.py b/tools/pypistats.py new file mode 100644 index 000000000..e8d9c0b91 --- /dev/null +++ b/tools/pypistats.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import json +from collections import defaultdict +from urllib.request import urlopen +import ssl + +PACKAGE = "lxml" + + +def get_pyver_stats(package=PACKAGE, period="month"): + stats_url = f"https://www.pypistats.org/api/packages/{package}/python_minor?period={period}" + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + with urlopen(stats_url, context=ctx) as stats: + data = json.load(stats) + return data + + +def aggregate(stats): + counts = defaultdict(int) + days = defaultdict(int) + for entry in stats['data']: + category = entry['category'] + counts[category] += entry['downloads'] + days[category] += 1 + return {category: counts[category] / days[category] for category in counts} + + +def main(): + stats = get_pyver_stats() + for version, count in sorted(aggregate(stats).items()): + print(f"{version:4}: {count:-12.1f} / day") + + +if __name__ == '__main__': + main() From f136f95b735c000134ec03d8abf23a405d0f2222 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 9 Nov 2022 12:09:29 +0100 Subject: [PATCH 0020/1154] Use final Python 3.11 in CI runs (GH-346) Also add an older refnanny job for comparison with the latest stable Python release. --- .github/workflows/ci.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5868a3af3..780248613 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,17 +30,20 @@ jobs: - 3.8 - 3.9 - "3.10" # quotes to avoid being interpreted as the number 3.1 - - "3.11-dev" - # - "3.12-dev" + - 3.11 + # - 3.12-dev env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: - # Temporary - Allow failure on all 3.11-dev jobs until beta comes out. - os: ubuntu-18.04 - python-version: 3.11-dev + python-version: 3.11 + - os: ubuntu-18.04 + python-version: 3.9 + env: {STATIC_DEPS: true, WITH_REFNANNY: true} + extra_hash: "-refnanny" allowed_failure: true - os: ubuntu-18.04 - python-version: 3.11-dev + python-version: 3.11 env: {STATIC_DEPS: true, WITH_REFNANNY: true} extra_hash: "-refnanny" allowed_failure: true @@ -103,12 +106,12 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 1 - - name: Setup python - uses: actions/setup-python@v2 + - name: Setup Python + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -129,7 +132,7 @@ jobs: run: make html - name: Upload docs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ matrix.extra_hash == '-docs' }} with: name: website_html @@ -137,14 +140,14 @@ jobs: if-no-files-found: ignore - name: Upload Coverage Report - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: pycoverage_html path: coverage* if-no-files-found: ignore - name: Upload Wheel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ matrix.env.STATIC_DEPS == 'true' && env.COVERAGE == 'false' }} with: name: wheels-${{ runner.os }} From eddd78d320b97a9170704009d871db337ece8069 Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Sat, 12 Nov 2022 16:31:05 +0100 Subject: [PATCH 0021/1154] AppVeyor CI: Add Python 3.11 jobs (GH-360) --- appveyor.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 344019035..2d6529e3f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,6 +3,8 @@ image: Visual Studio 2019 environment: matrix: + - python: 311 + - python: 311-x64 - python: 310 - python: 310-x64 - python: 39 @@ -19,6 +21,10 @@ environment: - python: 36-x64 - python: 35 - python: 35-x64 + + - python: 311 + arch: arm64 + env: STATIC_DEPS=true - python: 310 arch: arm64 env: STATIC_DEPS=true From b2359e197b3624bf95ee482261f6277f7a65c7a7 Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Thu, 1 Dec 2022 08:49:01 +0100 Subject: [PATCH 0022/1154] Add Dependabot configuration for GitHub Actions updates (GH-358) Add a Dependabot configuration that checks once a week if the GitHub Actions are still using the latest version. If not, it opens a PR to update them. It will actually open very few PRs, since we only have major versions specified (like v3), so only on a major v4 release it will update and open a PR. See https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..ac27a8486 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" From cc526589d062affb0432683724866e2afdc55eae Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 1 Dec 2022 08:52:06 +0100 Subject: [PATCH 0023/1154] GHA wheel CI: Upgrade GitHub Actions and PyPy version (GH-362) --- .github/workflows/wheels.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 09dc7c9d7..82574137a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: 3.9 @@ -33,13 +33,13 @@ jobs: files: dist/*.tar.gz - name: Upload sdist - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: sdist path: dist/*.tar.gz - name: Upload website - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: website path: doc/html @@ -96,10 +96,10 @@ jobs: pyversion: "cp310*" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 @@ -117,7 +117,7 @@ jobs: files: wheelhouse/*/*-m*linux*.whl # manylinux / musllinux - name: Upload wheels - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: wheels-${{ matrix.image }} path: wheelhouse/*/*-m*linux*.whl # manylinux / musllinux @@ -132,16 +132,16 @@ jobs: #os: [macos-10.15, windows-latest] #os: [macos-10.15, macOS-M1] os: [macos-10.15] - python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7"] + python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] runs-on: ${{ matrix.os }} env: { LIBXML2_VERSION: 2.9.14, LIBXSLT_VERSION: 1.1.35, MACOSX_DEPLOYMENT_TARGET: 10.15 } steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} @@ -165,7 +165,7 @@ jobs: files: dist/lxml-*.whl - name: Upload wheels - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: wheels-${{ matrix.os }} path: dist/lxml-*.whl From 7ed79b2b233b55c5d9ee944651a8857a0e7494ab Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 1 Dec 2022 09:37:12 +0100 Subject: [PATCH 0024/1154] CI: Avoid duplicate CI runs for PRs and tighten triggers and rights somewhat. --- .github/workflows/ci.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 780248613..a908a9289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,24 @@ name: CI -on: [push, pull_request] +on: + push: + paths: + - '**' + - '!.github/**' + - '.github/workflows/ci.yml' + pull_request: + paths: + - '**' + - '!.github/**' + - '.github/workflows/ci.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) jobs: ci: From d734076b894eae4945f9343eee249109e9b786f1 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 1 Dec 2022 09:39:24 +0100 Subject: [PATCH 0025/1154] CI: Minor bash code adaptation. --- tools/ci-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index f9b43fbdd..7e9167bda 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -54,7 +54,7 @@ fi # Build CFLAGS="-Og -g -fPIC -Wall -Wextra" python -u setup.py build_ext --inplace \ $(if [ -n "${PYTHON_VERSION##2.*}" ]; then echo -n " -j7 "; fi ) \ - $(if [ "$COVERAGE" == "true" ]; then echo -n " --with-coverage"; fi ) \ + $(if [[ "$COVERAGE" == "true" ]]; then echo -n " --with-coverage"; fi ) \ || exit 1 ccache -s || true From 14ff7f365168b96ae9ccb9a8127d48fb8a87e265 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 1 Dec 2022 09:41:39 +0100 Subject: [PATCH 0026/1154] Show Python versions in numerical order, not ascii order. Also show the aggregated percentage up to a given Python version. --- tools/pypistats.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/pypistats.py b/tools/pypistats.py index e8d9c0b91..92b96fbd1 100644 --- a/tools/pypistats.py +++ b/tools/pypistats.py @@ -29,10 +29,23 @@ def aggregate(stats): return {category: counts[category] / days[category] for category in counts} +def version_sorter(version_and_count): + version = version_and_count[0] + return tuple(map(int, version.split("."))) if version.replace(".", "").isdigit() else (2**32,) + + def main(): - stats = get_pyver_stats() - for version, count in sorted(aggregate(stats).items()): - print(f"{version:4}: {count:-12.1f} / day") + import sys + package_name = sys.argv[1] if len(sys.argv) > 1 else PACKAGE + + counts = get_pyver_stats(package=package_name) + stats = aggregate(counts) + total = sum(stats.values()) + + agg_sum = 0.0 + for version, count in sorted(stats.items(), key=version_sorter): + agg_sum += count + print(f"{version:4}: {count:-12.1f} / day ({agg_sum / total * 100:-5.1f}%)") if __name__ == '__main__': From 2728bbf620787e9c230587018521edd860f26a1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 09:47:35 +0100 Subject: [PATCH 0027/1154] Bump pat-s/always-upload-cache from 2.1.3 to 3.0.11 (GH-363) Bumps [pat-s/always-upload-cache](https://github.com/pat-s/always-upload-cache) from 2.1.3 to 3.0.11. - [Release notes](https://github.com/pat-s/always-upload-cache/releases) - [Changelog](https://github.com/pat-s/always-upload-cache/blob/main/RELEASES.md) - [Commits](https://github.com/pat-s/always-upload-cache/compare/v2.1.3...v3.0.11) --- updated-dependencies: - dependency-name: pat-s/always-upload-cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a908a9289..2a7ca3a89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Cache [ccache] - uses: pat-s/always-upload-cache@v2.1.3 + uses: pat-s/always-upload-cache@v3.0.11 if: startsWith(runner.os, 'Linux') with: path: ~/.ccache From aabb5ee34815a6d182a293add0e71c3b0601be93 Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Fri, 2 Dec 2022 10:00:00 +0100 Subject: [PATCH 0028/1154] GHA wheel CI: Update images, used actions and Python version (GH-356) * GHA wheel CI: Update images, used actions and Python version A bit of maintenance on the GitHub Actions wheel CI: - Update the used Ubuntu and macOS images to the latest versions, and enable the Windows run - Update the used actions to their latest versions - Use Python 3.10 to build wheels * Wheel CI: Add Python 3.11 jobs for Windows, macOS and manylinux * wheel CI: Use Python "3.x" to automatically use latest stable version Co-authored-by: scoder --- .github/workflows/wheels.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 82574137a..67f3e8eb1 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -6,7 +6,7 @@ on: jobs: sdist: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.x" - name: Install lib dependencies run: sudo apt-get update -y -q && sudo apt-get install -y -q "libxml2=2.9.10*" "libxml2-dev=2.9.10*" libxslt1.1 libxslt1-dev @@ -83,6 +83,8 @@ jobs: pyversion: "cp39*" - image: manylinux_2_24_aarch64 pyversion: "cp310*" + - image: manylinux_2_24_aarch64 + pyversion: "cp311*" - image: musllinux_1_1_aarch64 pyversion: "cp36*" @@ -94,6 +96,8 @@ jobs: pyversion: "cp39*" - image: musllinux_1_1_aarch64 pyversion: "cp310*" + - image: musllinux_1_1_aarch64 + pyversion: "cp311*" steps: - uses: actions/checkout@v3 @@ -101,7 +105,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.x" - name: Install dependencies run: python -m pip install -r requirements.txt @@ -129,10 +133,11 @@ jobs: fail-fast: false matrix: + os: [macos-latest, windows-latest] #os: [macos-10.15, windows-latest] #os: [macos-10.15, macOS-M1] - os: [macos-10.15] - python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] + #os: [macos-10.15] + python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] runs-on: ${{ matrix.os }} env: { LIBXML2_VERSION: 2.9.14, LIBXSLT_VERSION: 1.1.35, MACOSX_DEPLOYMENT_TARGET: 10.15 } From e8f088ae9710216e92e88eb5a432476ce582d76d Mon Sep 17 00:00:00 2001 From: BenHacker <56912908+MrBenHacker@users.noreply.github.com> Date: Tue, 6 Dec 2022 09:28:23 +0100 Subject: [PATCH 0029/1154] Fix build failed on windows server 2022 (GH-365) --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 67f3e8eb1..3e0852730 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -133,7 +133,7 @@ jobs: fail-fast: false matrix: - os: [macos-latest, windows-latest] + os: [macos-latest, windows-2019] #os: [macos-10.15, windows-latest] #os: [macos-10.15, macOS-M1] #os: [macos-10.15] From b636db9b9471756a88dcc3c570aee7e41e3880c0 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 2 Dec 2022 10:11:43 +0100 Subject: [PATCH 0030/1154] Prepare release of 4.9.2. --- CHANGES.txt | 18 ++++++++++++++++-- doc/main.txt | 10 +++++++--- src/lxml/__init__.py | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8a1174ea8..677f0bae8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,11 +2,25 @@ lxml changelog ============== -Under development -================= +4.9.2 (2022-12-02) +================== + +Bugs fixed +---------- + +* CVE-2022-2309: A Bug in libxml2 2.9.1[0-4] could let namespace declarations + from a failed parser run leak into later parser runs. This bug was worked around + in lxml and resolved in libxml2 2.10.0. + https://gitlab.gnome.org/GNOME/libxml2/-/issues/378 + +Other changes +------------- * LP#1981760: ``Element.attrib`` now registers as ``collections.abc.MutableMapping``. +* lxml now has a static build setup for macOS on ARM64 machines (not used for building wheels). + Patch by Quentin Leffray. + 4.9.1 (2022-07-01) ================== diff --git a/doc/main.txt b/doc/main.txt index 578f92dcf..a652c565e 100644 --- a/doc/main.txt +++ b/doc/main.txt @@ -160,8 +160,8 @@ Index `_ (PyPI). It has the source that compiles on various platforms. The source distribution is signed with `this key `_. -The latest version is `lxml 4.9.1`_, released 2022-07-01 -(`changes for 4.9.1`_). `Older versions <#old-versions>`_ +The latest version is `lxml 4.9.2`_, released 2022-12-02 +(`changes for 4.9.2`_). `Older versions <#old-versions>`_ are listed below. Please take a look at the @@ -256,7 +256,9 @@ See the websites of lxml .. and the `latest in-development version `_. -.. _`PDF documentation`: lxmldoc-4.9.1.pdf +.. _`PDF documentation`: lxmldoc-4.9.2.pdf + +* `lxml 4.9.2`_, released 2022-12-02 (`changes for 4.9.2`_) * `lxml 4.9.1`_, released 2022-07-01 (`changes for 4.9.1`_) @@ -282,6 +284,7 @@ See the websites of lxml * `older releases `_ +.. _`lxml 4.9.2`: /files/lxml-4.9.2.tgz .. _`lxml 4.9.1`: /files/lxml-4.9.1.tgz .. _`lxml 4.9.0`: /files/lxml-4.9.0.tgz .. _`lxml 4.8.0`: /files/lxml-4.8.0.tgz @@ -294,6 +297,7 @@ See the websites of lxml .. _`lxml 4.6.1`: /files/lxml-4.6.1.tgz .. _`lxml 4.6.0`: /files/lxml-4.6.0.tgz +.. _`changes for 4.9.2`: /changes-4.9.2.html .. _`changes for 4.9.1`: /changes-4.9.1.html .. _`changes for 4.9.0`: /changes-4.9.0.html .. _`changes for 4.8.0`: /changes-4.8.0.html diff --git a/src/lxml/__init__.py b/src/lxml/__init__.py index f8be68f71..f90fccc6a 100644 --- a/src/lxml/__init__.py +++ b/src/lxml/__init__.py @@ -1,6 +1,6 @@ # this is a package -__version__ = "4.9.1" +__version__ = "4.9.2" def get_include(): From 85b962c42bcac7ab45dd90325ab5ce32788e7eb0 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 25 Oct 2022 15:59:02 +0200 Subject: [PATCH 0031/1154] Add Windows CI builds. --- .github/workflows/ci.yml | 25 +++++++++++++++---------- tools/ci-run.sh | 2 +- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a7ca3a89..ccbc46ac3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: matrix: # Tests [amd64] # - os: [ubuntu-18.04, macos-10.15] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: - 2.7 - 3.5 @@ -53,31 +53,31 @@ jobs: env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: 3.11 - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: 3.9 env: {STATIC_DEPS: true, WITH_REFNANNY: true} extra_hash: "-refnanny" allowed_failure: true - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: 3.11 env: {STATIC_DEPS: true, WITH_REFNANNY: true} extra_hash: "-refnanny" allowed_failure: true # Coverage setup - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: 3.9 env: { COVERAGE: true } extra_hash: "-coverage" allowed_failure: true # shouldn't fail but currently does... - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: 3.9 env: { STATIC_DEPS: false, EXTRA_DEPS: "docutils pygments sphinx sphinx-rtd-theme" } extra_hash: "-docs" allowed_failure: true # shouldn't fail but currently does... # Old library setup with minimum version requirements - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: 3.9 env: { STATIC_DEPS: true, @@ -89,20 +89,25 @@ jobs: # Ubuntu sub-jobs: # ================ # Pypy - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: pypy-2.7 env: { STATIC_DEPS: false } allowed_failure: true - - os: ubuntu-18.04 + - os: ubuntu-latest python-version: pypy-3.7 env: { STATIC_DEPS: false } allowed_failure: true # MacOS sub-jobs # ============== - - os: macos-10.15 + - os: macos-latest allowed_failure: true # Unicode parsing fails in Py3 + # Windows sub-jobs + # ============== + - os: windows-latest + env: [{ STATIC_DEPS: true }] # always static + # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. # From testing, the runs tend to take ~3 minutes, so a limit of 20 minutes should be enough. This can always be # changed in the future if needed. diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 7e9167bda..6cd4fad6e 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -7,7 +7,7 @@ if [ -z "${OS_NAME##ubuntu*}" ]; then echo "Installing requirements [apt]" sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" sudo apt-get update -y -q - sudo apt-get install -y -q ccache gcc-$GCC_VERSION "libxml2=2.9.4*" "libxml2-dev=2.9.4*" libxslt1.1 libxslt1-dev || exit 1 + sudo apt-get install -y -q ccache gcc-$GCC_VERSION "libxml2=2.9.13*" "libxml2-dev=2.9.13*" libxslt1.1 libxslt1-dev || exit 1 sudo /usr/sbin/update-ccache-symlinks echo "/usr/lib/ccache" >> $GITHUB_PATH # export ccache to path From 714173737dd16f185df67451aca9bc7329bdd940 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 3 Nov 2022 10:19:46 +0100 Subject: [PATCH 0032/1154] Adapt download script to Github web changes by using their JSON API instead. --- download_artefacts.py | 49 ++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/download_artefacts.py b/download_artefacts.py index 268f0ed76..814525b26 100755 --- a/download_artefacts.py +++ b/download_artefacts.py @@ -1,34 +1,29 @@ #!/usr/bin/python3 -import itertools import json import logging -import re import shutil import datetime from concurrent.futures import ProcessPoolExecutor as Pool, as_completed from pathlib import Path -from urllib.request import urlopen +from urllib.request import urlopen, Request from urllib.parse import urljoin logger = logging.getLogger() PARALLEL_DOWNLOADS = 6 -GITHUB_PACKAGE_URL = "https://github.com/lxml/lxml" +GITHUB_API_URL = "https://api.github.com/repos/lxml/lxml" APPVEYOR_PACKAGE_URL = "https://ci.appveyor.com/api/projects/scoder/lxml" APPVEYOR_BUILDJOBS_URL = "https://ci.appveyor.com/api/buildjobs" -def find_github_files(version, base_package_url=GITHUB_PACKAGE_URL): - file_url_pattern = r'href="([^"]+/releases/download/[^"]+\.(?:whl|tar\.gz))"' - url = f"{base_package_url}/releases/tag/lxml-{version}" +def find_github_files(version, api_url=GITHUB_API_URL): + url = f"{api_url}/releases/tags/{version}" + release, _ = read_url(url, accept="application/vnd.github+json", as_json=True) - with urlopen(url) as p: - page = p.read().decode() - - for wheel_url, _ in itertools.groupby(sorted(re.findall(file_url_pattern, page))): - yield urljoin(base_package_url, wheel_url) + for asset in release.get('assets', ()): + yield asset['browser_download_url'] def find_appveyor_files(version, base_package_url=APPVEYOR_PACKAGE_URL, base_job_url=APPVEYOR_BUILDJOBS_URL): @@ -57,6 +52,36 @@ def find_appveyor_files(version, base_package_url=APPVEYOR_PACKAGE_URL, base_job yield urljoin(artifacts_url, artifact['fileName']) +def read_url(url, decode=True, accept=None, as_json=False): + if accept: + request = Request(url, headers={'Accept': accept}) + else: + request = Request(url) + + with urlopen(request) as res: + charset = _find_content_encoding(res) + content_type = res.headers.get('Content-Type') + data = res.read() + + if decode: + data = data.decode(charset) + if as_json: + data = json.loads(data) + return data, content_type + + +def _find_content_encoding(response, default='iso8859-1'): + from email.message import Message + content_type = response.headers.get('Content-Type') + if content_type: + msg = Message() + msg.add_header('Content-Type', content_type) + charset = msg.get_content_charset(default) + else: + charset = default + return charset + + def download1(wheel_url, dest_dir): wheel_name = wheel_url.rsplit("/", 1)[1] logger.info(f"Downloading {wheel_url} ...") From d8eafd7954a1b872a0ea5d9039d8be0ecfe9a620 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 8 Nov 2022 15:50:10 +0100 Subject: [PATCH 0033/1154] Some Windows fixes in CI build script. --- tools/ci-run.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 6cd4fad6e..5dc6e69ae 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -1,6 +1,8 @@ #!/usr/bin/bash GCC_VERSION=${GCC_VERSION:=8} +TEST_CFLAGS= +EXTRA_CFLAGS= # Set up compilers if [ -z "${OS_NAME##ubuntu*}" ]; then @@ -15,9 +17,13 @@ if [ -z "${OS_NAME##ubuntu*}" ]; then export CC="gcc" export PATH="/usr/lib/ccache:$PATH" + TEST_CFLAGS="-Og -g -fPIC" + EXTRA_CFLAGS="$TEST_CFLAGS -Wall -Wextra" elif [ -z "${OS_NAME##macos*}" ]; then export CC="clang -Wno-deprecated-declarations" + TEST_CFLAGS="-Og -g -fPIC" + EXTRA_CFLAGS="$TEST_CFLAGS -Wall -Wextra" fi # Log versions in use @@ -25,11 +31,13 @@ echo "====================" echo "|VERSIONS INSTALLED|" echo "====================" python -c 'import sys; print("Python %s" % (sys.version,))' -if [ "$CC" ]; then +if [[ "$CC" ]]; then which ${CC%% *} ${CC%% *} --version fi -pkg-config --modversion libxml-2.0 libxslt +if [ -z "${OS_NAME##win*}" ]; then + pkg-config --modversion libxml-2.0 libxslt +fi echo "====================" ccache -s || true @@ -46,13 +54,13 @@ if [ -z "${PYTHON_VERSION##2*}" ]; then else python -m pip install -U beautifulsoup4 cssselect html5lib rnc2rng ${EXTRA_DEPS} || exit 1 fi -if [ "$COVERAGE" == "true" ]; then +if [[ "$COVERAGE" == "true" ]]; then python -m pip install "coverage<5" || exit 1 python -m pip install --pre 'Cython>=3.0a0' || exit 1 fi # Build -CFLAGS="-Og -g -fPIC -Wall -Wextra" python -u setup.py build_ext --inplace \ +CFLAGS="$CFLAGS $EXTRA_CFLAGS" python -u setup.py build_ext --inplace \ $(if [ -n "${PYTHON_VERSION##2.*}" ]; then echo -n " -j7 "; fi ) \ $(if [[ "$COVERAGE" == "true" ]]; then echo -n " --with-coverage"; fi ) \ || exit 1 @@ -60,7 +68,7 @@ CFLAGS="-Og -g -fPIC -Wall -Wextra" python -u setup.py build_ext --inplace \ ccache -s || true # Run tests -CFLAGS="-Og -g -fPIC" PYTHONUNBUFFERED=x make test || exit 1 +CFLAGS="$TEST_CFLAGS" PYTHONUNBUFFERED=x make test || exit 1 python setup.py install || exit 1 python -c "from lxml import etree" || exit 1 From 980f47fa022ab5ba8d93eb2f951690512be89bde Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 6 Dec 2022 20:49:30 +0100 Subject: [PATCH 0034/1154] Remove outdated and not disfunctional PyPy3.7 wheel target. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3e0852730..2ec15bd02 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -137,7 +137,7 @@ jobs: #os: [macos-10.15, windows-latest] #os: [macos-10.15, macOS-M1] #os: [macos-10.15] - python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] + python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] runs-on: ${{ matrix.os }} env: { LIBXML2_VERSION: 2.9.14, LIBXSLT_VERSION: 1.1.35, MACOSX_DEPLOYMENT_TARGET: 10.15 } From a498a52056897a3595ad81af7a8393b1fc99c67d Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 7 Dec 2022 09:52:18 +0100 Subject: [PATCH 0035/1154] Send the project URL as user agent when downloading libraries during the build. --- buildlibxml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildlibxml.py b/buildlibxml.py index 8faf59378..15d6e3383 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -170,10 +170,10 @@ def _list_dir_ftplib(url): def read_url(url, decode=True, accept=None, as_json=False): + headers = {'User-Agent': 'https://github.com/lxml/lxml'} if accept: - request = Request(url, headers={'Accept': accept}) - else: - request = Request(url) + headers['Accept'] = accept + request = Request(url, headers=headers) with closing(urlopen(request)) as res: charset = _find_content_encoding(res) From 360a81cb9165a5b903641990f60f044d97e716d9 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 09:28:24 +0100 Subject: [PATCH 0036/1154] Extend PyPI stats script to include OS download statistics. --- tools/pypistats.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tools/pypistats.py b/tools/pypistats.py index 92b96fbd1..c5528970f 100644 --- a/tools/pypistats.py +++ b/tools/pypistats.py @@ -7,8 +7,8 @@ PACKAGE = "lxml" -def get_pyver_stats(package=PACKAGE, period="month"): - stats_url = f"https://www.pypistats.org/api/packages/{package}/python_minor?period={period}" +def get_stats(stats_type, package=PACKAGE, period="month"): + stats_url = f"https://www.pypistats.org/api/packages/{package}/{stats_type}?period={period}" ctx = ssl.create_default_context() ctx.check_hostname = False @@ -34,18 +34,37 @@ def version_sorter(version_and_count): return tuple(map(int, version.split("."))) if version.replace(".", "").isdigit() else (2**32,) -def main(): - import sys - package_name = sys.argv[1] if len(sys.argv) > 1 else PACKAGE +def system_sorter(name_and_count): + order = ('linux', 'windows', 'darwin') + system = name_and_count[0] + try: + return order.index(system.lower()) + except ValueError: + return len(order) - counts = get_pyver_stats(package=package_name) + +def print_agg_stats(counts, sort_key=None): stats = aggregate(counts) total = sum(stats.values()) - + max_len = max(len(category) for category in stats) agg_sum = 0.0 - for version, count in sorted(stats.items(), key=version_sorter): + for category, count in sorted(stats.items(), key=sort_key): agg_sum += count - print(f"{version:4}: {count:-12.1f} / day ({agg_sum / total * 100:-5.1f}%)") + print(f" {category:{max_len}}: {count:-12.1f} / day ({agg_sum / total * 100:-5.1f}%)") + + +def main(): + import sys + package_name = sys.argv[1] if len(sys.argv) > 1 else PACKAGE + + counts = get_stats("python_minor", package=package_name) + print("Downloads by Python version:") + print_agg_stats(counts, sort_key=version_sorter) + + print() + counts = get_stats("system", package=package_name) + print("Downloads by system:") + print_agg_stats(counts, sort_key=system_sorter) if __name__ == '__main__': From 30432f949f1031dad380ad9ba515a75fc6eda0d0 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 09:33:52 +0100 Subject: [PATCH 0037/1154] Use windows-2019 images also for CI builds (now only for wheel builds) and use strings for all Python versions to help with upgrading. --- .github/workflows/ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccbc46ac3..29bf5be48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,46 +39,46 @@ jobs: matrix: # Tests [amd64] # - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-2019] python-version: - - 2.7 - - 3.5 - - 3.6 - - 3.7 - - 3.8 - - 3.9 + - "2.7" + - "3.5" + - "3.6" + - "3.7" + - "3.8" + - "3.9" - "3.10" # quotes to avoid being interpreted as the number 3.1 - - 3.11 - # - 3.12-dev + - "3.11" + # - "3.12-dev" env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: - os: ubuntu-latest - python-version: 3.11 + python-version: "3.11" - os: ubuntu-latest - python-version: 3.9 + python-version: "3.9" env: {STATIC_DEPS: true, WITH_REFNANNY: true} extra_hash: "-refnanny" allowed_failure: true - os: ubuntu-latest - python-version: 3.11 + python-version: "3.11" env: {STATIC_DEPS: true, WITH_REFNANNY: true} extra_hash: "-refnanny" allowed_failure: true # Coverage setup - os: ubuntu-latest - python-version: 3.9 + python-version: "3.9" env: { COVERAGE: true } extra_hash: "-coverage" allowed_failure: true # shouldn't fail but currently does... - os: ubuntu-latest - python-version: 3.9 + python-version: "3.9" env: { STATIC_DEPS: false, EXTRA_DEPS: "docutils pygments sphinx sphinx-rtd-theme" } extra_hash: "-docs" allowed_failure: true # shouldn't fail but currently does... # Old library setup with minimum version requirements - os: ubuntu-latest - python-version: 3.9 + python-version: "3.9" env: { STATIC_DEPS: true, LIBXML2_VERSION: 2.9.2, @@ -105,7 +105,7 @@ jobs: # Windows sub-jobs # ============== - - os: windows-latest + - os: windows-2019 env: [{ STATIC_DEPS: true }] # always static # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. From 6b55a2f4652ff00507565d57b15ffecfd8af54da Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 10:08:15 +0100 Subject: [PATCH 0038/1154] Use gcc-9 in CI instead of gcc-8, which is no longer available. --- .github/workflows/ci.yml | 2 +- tools/ci-run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29bf5be48..68b6a43ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: LIBXML2_VERSION: 2.9.14 LIBXSLT_VERSION: 1.1.35 COVERAGE: false - GCC_VERSION: 8 + GCC_VERSION: 9 USE_CCACHE: 1 CCACHE_SLOPPINESS: "pch_defines,time_macros" CCACHE_COMPRESS: 1 diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 5dc6e69ae..bddd043e3 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -1,6 +1,6 @@ #!/usr/bin/bash -GCC_VERSION=${GCC_VERSION:=8} +GCC_VERSION=${GCC_VERSION:=9} TEST_CFLAGS= EXTRA_CFLAGS= From fc53d6ff9d8b6770e45dccc4bd88fff2c92cd38a Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 10:33:21 +0100 Subject: [PATCH 0039/1154] Show executed commands in CI runs. --- tools/ci-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index bddd043e3..0d45329a5 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +set -x + GCC_VERSION=${GCC_VERSION:=9} TEST_CFLAGS= EXTRA_CFLAGS= From 487a1943167909839c838c6d837ec4c792778baa Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 10:33:48 +0100 Subject: [PATCH 0040/1154] CI: exclude non-static Windows jobs. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68b6a43ca..ff6043517 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,10 +103,11 @@ jobs: - os: macos-latest allowed_failure: true # Unicode parsing fails in Py3 + exclude: # Windows sub-jobs # ============== - os: windows-2019 - env: [{ STATIC_DEPS: true }] # always static + env: { STATIC_DEPS: false } # always static # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. # From testing, the runs tend to take ~3 minutes, so a limit of 20 minutes should be enough. This can always be From b848b82e2ffd25d6a59271d4d80c2b3e0c2fdae3 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 10:47:52 +0100 Subject: [PATCH 0041/1154] Try to fix CI "setup.py install" in Py3.11. --- tools/ci-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 0d45329a5..db3c7e879 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -72,6 +72,7 @@ ccache -s || true # Run tests CFLAGS="$TEST_CFLAGS" PYTHONUNBUFFERED=x make test || exit 1 +python setup.py build || exit 1 python setup.py install || exit 1 python -c "from lxml import etree" || exit 1 From 0b0b2b9b73fe5c632e6ff5e88d34917620a37016 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 10:48:27 +0100 Subject: [PATCH 0042/1154] Exclude missing Python versions from CI jobs. --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff6043517..c5ec41c36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,8 +53,6 @@ jobs: env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: - - os: ubuntu-latest - python-version: "3.11" - os: ubuntu-latest python-version: "3.9" env: {STATIC_DEPS: true, WITH_REFNANNY: true} @@ -104,6 +102,11 @@ jobs: allowed_failure: true # Unicode parsing fails in Py3 exclude: + - os: ubuntu-latest + python-version: "3.5" + - os: ubuntu-latest + python-version: "3.6" + # Windows sub-jobs # ============== - os: windows-2019 From 2c2308e96303847c4306ae1aa836542e7a5a786f Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 10:57:26 +0100 Subject: [PATCH 0043/1154] Try to add a Windows CI build for Py2.7. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5ec41c36..e680c23d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,10 @@ jobs: env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: + - os: windows-2016 + python-version: 2.7 + env: { STATIC_DEPS: true } # always static + - os: ubuntu-latest python-version: "3.9" env: {STATIC_DEPS: true, WITH_REFNANNY: true} @@ -109,6 +113,8 @@ jobs: # Windows sub-jobs # ============== + - os: windows-2019 + python-version: 2.7 # needs older image - os: windows-2019 env: { STATIC_DEPS: false } # always static From cece238fd608e01cf5f791a7e97b08dd052e0e45 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 16:40:29 +0100 Subject: [PATCH 0044/1154] Add PyPy-3.8 CI target. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e680c23d3..dd1b9ea6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,10 @@ jobs: python-version: pypy-3.7 env: { STATIC_DEPS: false } allowed_failure: true + - os: ubuntu-latest + python-version: pypy-3.8 + env: { STATIC_DEPS: false } + allowed_failure: true # MacOS sub-jobs # ============== From ce4e5bcc37dc82110569714a5f7b4ed2d910bb36 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 16:41:51 +0100 Subject: [PATCH 0045/1154] Fix release date. --- CHANGES.txt | 2 +- doc/main.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 677f0bae8..c684ad5e1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,7 +2,7 @@ lxml changelog ============== -4.9.2 (2022-12-02) +4.9.2 (2022-12-13) ================== Bugs fixed diff --git a/doc/main.txt b/doc/main.txt index a652c565e..60e831b4c 100644 --- a/doc/main.txt +++ b/doc/main.txt @@ -160,7 +160,7 @@ Index `_ (PyPI). It has the source that compiles on various platforms. The source distribution is signed with `this key `_. -The latest version is `lxml 4.9.2`_, released 2022-12-02 +The latest version is `lxml 4.9.2`_, released 2022-12-13 (`changes for 4.9.2`_). `Older versions <#old-versions>`_ are listed below. @@ -258,7 +258,7 @@ See the websites of lxml .. _`PDF documentation`: lxmldoc-4.9.2.pdf -* `lxml 4.9.2`_, released 2022-12-02 (`changes for 4.9.2`_) +* `lxml 4.9.2`_, released 2022-12-13 (`changes for 4.9.2`_) * `lxml 4.9.1`_, released 2022-07-01 (`changes for 4.9.1`_) From 98224b31106e23d0ffeb1101ff4bc2c4627e8fd6 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 20:22:19 +0100 Subject: [PATCH 0046/1154] Install more recent library versions for the wheel build. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2ec15bd02..bc4d2d588 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -17,7 +17,7 @@ jobs: python-version: "3.x" - name: Install lib dependencies - run: sudo apt-get update -y -q && sudo apt-get install -y -q "libxml2=2.9.10*" "libxml2-dev=2.9.10*" libxslt1.1 libxslt1-dev + run: sudo apt-get update -y -q && sudo apt-get install -y -q "libxml2=2.9.13*" "libxml2-dev=2.9.13*" libxslt1.1 libxslt1-dev - name: Install Python dependencies run: python -m pip install -U pip setuptools && python -m pip install -U docutils pygments sphinx sphinx-rtd-theme -r requirements.txt From fc2f7ea8108544b9a2f2dd6cb8fdfb1f9f78dc2a Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 20:23:07 +0100 Subject: [PATCH 0047/1154] Use windows-2016 image instead of windows-2019 to fix the Py2.7 build. --- .github/workflows/wheels.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index bc4d2d588..2d2b7ac89 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -139,6 +139,13 @@ jobs: #os: [macos-10.15] python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] + include: + - os: windows-2016 + python-version: 2.7 + exclude: + - os: windows-2019 + python-version: 2.7 # needs older image + runs-on: ${{ matrix.os }} env: { LIBXML2_VERSION: 2.9.14, LIBXSLT_VERSION: 1.1.35, MACOSX_DEPLOYMENT_TARGET: 10.15 } From c17c1ca0496fbde3827c01cecd6a37d848142772 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 13 Dec 2022 20:27:16 +0100 Subject: [PATCH 0048/1154] Use same naming for Python version matrix variable in wheel workflow as in CI workflow. --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2d2b7ac89..89078587b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -137,7 +137,7 @@ jobs: #os: [macos-10.15, windows-latest] #os: [macos-10.15, macOS-M1] #os: [macos-10.15] - python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] + python-version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] include: - os: windows-2016 @@ -155,7 +155,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python_version }} + python-version: ${{ matrix.python-version }} - name: Install MacOS dependencies if: startsWith(matrix.os, 'mac') From 178f83b0e794ed5adfe48487e02986b85662e836 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 14 Dec 2022 09:52:29 +0100 Subject: [PATCH 0049/1154] Fix Github URL building in download script (as done for the appveyor URL). --- download_artefacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download_artefacts.py b/download_artefacts.py index 814525b26..8410d47d5 100755 --- a/download_artefacts.py +++ b/download_artefacts.py @@ -19,7 +19,7 @@ def find_github_files(version, api_url=GITHUB_API_URL): - url = f"{api_url}/releases/tags/{version}" + url = f"{api_url}/releases/tags/lxml-{version}" release, _ = read_url(url, accept="application/vnd.github+json", as_json=True) for asset in release.get('assets', ()): From dcbc0cc1cb0cedf8019184aaca805d2a649cd8de Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 15 Dec 2022 11:38:42 +0100 Subject: [PATCH 0050/1154] Use generic 'nogil' block markers in pxd declarations where possible, except for a few callback declarations that may be used for regular GIL functions. --- src/lxml/includes/c14n.pxd | 9 +- src/lxml/includes/htmlparser.pxd | 26 ++-- src/lxml/includes/relaxng.pxd | 22 +-- src/lxml/includes/schematron.pxd | 16 +- src/lxml/includes/tree.pxd | 260 +++++++++++++++---------------- src/lxml/includes/uri.pxd | 2 +- src/lxml/includes/xinclude.pxd | 20 +-- src/lxml/includes/xmlerror.pxd | 12 +- src/lxml/includes/xmlparser.pxd | 67 ++++---- src/lxml/includes/xmlschema.pxd | 2 +- src/lxml/includes/xpath.pxd | 73 ++++----- src/lxml/includes/xslt.pxd | 85 +++++----- 12 files changed, 297 insertions(+), 297 deletions(-) diff --git a/src/lxml/includes/c14n.pxd b/src/lxml/includes/c14n.pxd index d075e90e2..8b1f3c4c5 100644 --- a/src/lxml/includes/c14n.pxd +++ b/src/lxml/includes/c14n.pxd @@ -1,13 +1,13 @@ from lxml.includes.tree cimport xmlDoc, xmlOutputBuffer, xmlChar from lxml.includes.xpath cimport xmlNodeSet -cdef extern from "libxml/c14n.h": +cdef extern from "libxml/c14n.h" nogil: cdef int xmlC14NDocDumpMemory(xmlDoc* doc, xmlNodeSet* nodes, int exclusive, xmlChar** inclusive_ns_prefixes, int with_comments, - xmlChar** doc_txt_ptr) nogil + xmlChar** doc_txt_ptr) cdef int xmlC14NDocSave(xmlDoc* doc, xmlNodeSet* nodes, @@ -15,12 +15,11 @@ cdef extern from "libxml/c14n.h": xmlChar** inclusive_ns_prefixes, int with_comments, char* filename, - int compression) nogil + int compression) cdef int xmlC14NDocSaveTo(xmlDoc* doc, xmlNodeSet* nodes, int exclusive, xmlChar** inclusive_ns_prefixes, int with_comments, - xmlOutputBuffer* buffer) nogil - + xmlOutputBuffer* buffer) diff --git a/src/lxml/includes/htmlparser.pxd b/src/lxml/includes/htmlparser.pxd index 145a69a06..31dcc406c 100644 --- a/src/lxml/includes/htmlparser.pxd +++ b/src/lxml/includes/htmlparser.pxd @@ -4,7 +4,7 @@ from lxml.includes.tree cimport xmlDoc from lxml.includes.tree cimport xmlInputReadCallback, xmlInputCloseCallback from lxml.includes.xmlparser cimport xmlParserCtxt, xmlSAXHandler, xmlSAXHandlerV1 -cdef extern from "libxml/HTMLparser.h": +cdef extern from "libxml/HTMLparser.h" nogil: ctypedef enum htmlParserOption: HTML_PARSE_NOERROR # suppress error reports HTML_PARSE_NOWARNING # suppress warning reports @@ -24,33 +24,33 @@ cdef extern from "libxml/HTMLparser.h": xmlSAXHandlerV1 htmlDefaultSAXHandler cdef xmlParserCtxt* htmlCreateMemoryParserCtxt( - char* buffer, int size) nogil + char* buffer, int size) cdef xmlParserCtxt* htmlCreateFileParserCtxt( - char* filename, char* encoding) nogil + char* filename, char* encoding) cdef xmlParserCtxt* htmlCreatePushParserCtxt(xmlSAXHandler* sax, void* user_data, char* chunk, int size, - char* filename, int enc) nogil - cdef void htmlFreeParserCtxt(xmlParserCtxt* ctxt) nogil - cdef void htmlCtxtReset(xmlParserCtxt* ctxt) nogil - cdef int htmlCtxtUseOptions(xmlParserCtxt* ctxt, int options) nogil - cdef int htmlParseDocument(xmlParserCtxt* ctxt) nogil + char* filename, int enc) + cdef void htmlFreeParserCtxt(xmlParserCtxt* ctxt) + cdef void htmlCtxtReset(xmlParserCtxt* ctxt) + cdef int htmlCtxtUseOptions(xmlParserCtxt* ctxt, int options) + cdef int htmlParseDocument(xmlParserCtxt* ctxt) cdef int htmlParseChunk(xmlParserCtxt* ctxt, - char* chunk, int size, int terminate) nogil + char* chunk, int size, int terminate) cdef xmlDoc* htmlCtxtReadFile(xmlParserCtxt* ctxt, char* filename, const_char* encoding, - int options) nogil + int options) cdef xmlDoc* htmlCtxtReadDoc(xmlParserCtxt* ctxt, char* buffer, char* URL, const_char* encoding, - int options) nogil + int options) cdef xmlDoc* htmlCtxtReadIO(xmlParserCtxt* ctxt, xmlInputReadCallback ioread, xmlInputCloseCallback ioclose, void* ioctx, char* URL, const_char* encoding, - int options) nogil + int options) cdef xmlDoc* htmlCtxtReadMemory(xmlParserCtxt* ctxt, char* buffer, int size, char* filename, const_char* encoding, - int options) nogil + int options) diff --git a/src/lxml/includes/relaxng.pxd b/src/lxml/includes/relaxng.pxd index 28e9212d2..5ac96711e 100644 --- a/src/lxml/includes/relaxng.pxd +++ b/src/lxml/includes/relaxng.pxd @@ -1,7 +1,7 @@ from lxml.includes.tree cimport xmlDoc from lxml.includes.xmlerror cimport xmlStructuredErrorFunc -cdef extern from "libxml/relaxng.h": +cdef extern from "libxml/relaxng.h" nogil: ctypedef struct xmlRelaxNG ctypedef struct xmlRelaxNGParserCtxt @@ -49,16 +49,16 @@ cdef extern from "libxml/relaxng.h": XML_RELAXNG_ERR_ELEMWRONG = 38 XML_RELAXNG_ERR_TEXTWRONG = 39 - cdef xmlRelaxNGValidCtxt* xmlRelaxNGNewValidCtxt(xmlRelaxNG* schema) nogil - cdef int xmlRelaxNGValidateDoc(xmlRelaxNGValidCtxt* ctxt, xmlDoc* doc) nogil - cdef xmlRelaxNG* xmlRelaxNGParse(xmlRelaxNGParserCtxt* ctxt) nogil - cdef xmlRelaxNGParserCtxt* xmlRelaxNGNewParserCtxt(char* URL) nogil - cdef xmlRelaxNGParserCtxt* xmlRelaxNGNewDocParserCtxt(xmlDoc* doc) nogil - cdef void xmlRelaxNGFree(xmlRelaxNG* schema) nogil - cdef void xmlRelaxNGFreeParserCtxt(xmlRelaxNGParserCtxt* ctxt) nogil - cdef void xmlRelaxNGFreeValidCtxt(xmlRelaxNGValidCtxt* ctxt) nogil + cdef xmlRelaxNGValidCtxt* xmlRelaxNGNewValidCtxt(xmlRelaxNG* schema) + cdef int xmlRelaxNGValidateDoc(xmlRelaxNGValidCtxt* ctxt, xmlDoc* doc) + cdef xmlRelaxNG* xmlRelaxNGParse(xmlRelaxNGParserCtxt* ctxt) + cdef xmlRelaxNGParserCtxt* xmlRelaxNGNewParserCtxt(char* URL) + cdef xmlRelaxNGParserCtxt* xmlRelaxNGNewDocParserCtxt(xmlDoc* doc) + cdef void xmlRelaxNGFree(xmlRelaxNG* schema) + cdef void xmlRelaxNGFreeParserCtxt(xmlRelaxNGParserCtxt* ctxt) + cdef void xmlRelaxNGFreeValidCtxt(xmlRelaxNGValidCtxt* ctxt) cdef void xmlRelaxNGSetValidStructuredErrors( - xmlRelaxNGValidCtxt* ctxt, xmlStructuredErrorFunc serror, void *ctx) nogil + xmlRelaxNGValidCtxt* ctxt, xmlStructuredErrorFunc serror, void *ctx) cdef void xmlRelaxNGSetParserStructuredErrors( - xmlRelaxNGParserCtxt* ctxt, xmlStructuredErrorFunc serror, void *ctx) nogil + xmlRelaxNGParserCtxt* ctxt, xmlStructuredErrorFunc serror, void *ctx) diff --git a/src/lxml/includes/schematron.pxd b/src/lxml/includes/schematron.pxd index f8e325284..181248afd 100644 --- a/src/lxml/includes/schematron.pxd +++ b/src/lxml/includes/schematron.pxd @@ -1,7 +1,7 @@ from lxml.includes cimport xmlerror from lxml.includes.tree cimport xmlDoc -cdef extern from "libxml/schematron.h": +cdef extern from "libxml/schematron.h" nogil: ctypedef struct xmlSchematron ctypedef struct xmlSchematronParserCtxt ctypedef struct xmlSchematronValidCtxt @@ -16,19 +16,19 @@ cdef extern from "libxml/schematron.h": XML_SCHEMATRON_OUT_IO = 1024 # output to I/O mechanism cdef xmlSchematronParserCtxt* xmlSchematronNewDocParserCtxt( - xmlDoc* doc) nogil + xmlDoc* doc) cdef xmlSchematronParserCtxt* xmlSchematronNewParserCtxt( char* filename) nogil cdef xmlSchematronValidCtxt* xmlSchematronNewValidCtxt( - xmlSchematron* schema, int options) nogil + xmlSchematron* schema, int options) - cdef xmlSchematron* xmlSchematronParse(xmlSchematronParserCtxt* ctxt) nogil + cdef xmlSchematron* xmlSchematronParse(xmlSchematronParserCtxt* ctxt) cdef int xmlSchematronValidateDoc(xmlSchematronValidCtxt* ctxt, - xmlDoc* instance) nogil + xmlDoc* instance) - cdef void xmlSchematronFreeParserCtxt(xmlSchematronParserCtxt* ctxt) nogil - cdef void xmlSchematronFreeValidCtxt(xmlSchematronValidCtxt* ctxt) nogil - cdef void xmlSchematronFree(xmlSchematron* schema) nogil + cdef void xmlSchematronFreeParserCtxt(xmlSchematronParserCtxt* ctxt) + cdef void xmlSchematronFreeValidCtxt(xmlSchematronValidCtxt* ctxt) + cdef void xmlSchematronFree(xmlSchematron* schema) cdef void xmlSchematronSetValidStructuredErrors( xmlSchematronValidCtxt* ctxt, xmlerror.xmlStructuredErrorFunc error_func, void *data) diff --git a/src/lxml/includes/tree.pxd b/src/lxml/includes/tree.pxd index 010af8090..03d558a33 100644 --- a/src/lxml/includes/tree.pxd +++ b/src/lxml/includes/tree.pxd @@ -9,19 +9,19 @@ cdef extern from "libxml/xmlversion.h": cdef const_char* xmlParserVersion cdef int LIBXML_VERSION -cdef extern from "libxml/xmlstring.h": +cdef extern from "libxml/xmlstring.h" nogil: ctypedef unsigned char xmlChar ctypedef const xmlChar const_xmlChar "const xmlChar" - cdef int xmlStrlen(const_xmlChar* str) nogil - cdef xmlChar* xmlStrdup(const_xmlChar* cur) nogil - cdef int xmlStrncmp(const_xmlChar* str1, const_xmlChar* str2, int length) nogil - cdef int xmlStrcmp(const_xmlChar* str1, const_xmlChar* str2) nogil - cdef int xmlStrcasecmp(const xmlChar *str1, const xmlChar *str2) nogil - cdef const_xmlChar* xmlStrstr(const_xmlChar* str1, const_xmlChar* str2) nogil - cdef const_xmlChar* xmlStrchr(const_xmlChar* str1, xmlChar ch) nogil + cdef int xmlStrlen(const_xmlChar* str) + cdef xmlChar* xmlStrdup(const_xmlChar* cur) + cdef int xmlStrncmp(const_xmlChar* str1, const_xmlChar* str2, int length) + cdef int xmlStrcmp(const_xmlChar* str1, const_xmlChar* str2) + cdef int xmlStrcasecmp(const xmlChar *str1, const xmlChar *str2) + cdef const_xmlChar* xmlStrstr(const_xmlChar* str1, const_xmlChar* str2) + cdef const_xmlChar* xmlStrchr(const_xmlChar* str1, xmlChar ch) cdef const_xmlChar* _xcstr "(const xmlChar*)PyBytes_AS_STRING" (object s) -cdef extern from "libxml/encoding.h": +cdef extern from "libxml/encoding.h" nogil: ctypedef enum xmlCharEncoding: XML_CHAR_ENCODING_ERROR = -1 # No char encoding detected XML_CHAR_ENCODING_NONE = 0 # No char encoding detected @@ -49,19 +49,19 @@ cdef extern from "libxml/encoding.h": XML_CHAR_ENCODING_ASCII = 22 # pure ASCII ctypedef struct xmlCharEncodingHandler - cdef xmlCharEncodingHandler* xmlFindCharEncodingHandler(char* name) nogil + cdef xmlCharEncodingHandler* xmlFindCharEncodingHandler(char* name) cdef xmlCharEncodingHandler* xmlGetCharEncodingHandler( - xmlCharEncoding enc) nogil - cdef int xmlCharEncCloseFunc(xmlCharEncodingHandler* handler) nogil - cdef xmlCharEncoding xmlDetectCharEncoding(const_xmlChar* text, int len) nogil - cdef const_char* xmlGetCharEncodingName(xmlCharEncoding enc) nogil - cdef xmlCharEncoding xmlParseCharEncoding(char* name) nogil + xmlCharEncoding enc) + cdef int xmlCharEncCloseFunc(xmlCharEncodingHandler* handler) + cdef xmlCharEncoding xmlDetectCharEncoding(const_xmlChar* text, int len) + cdef const_char* xmlGetCharEncodingName(xmlCharEncoding enc) + cdef xmlCharEncoding xmlParseCharEncoding(char* name) ctypedef int (*xmlCharEncodingOutputFunc)( unsigned char *out_buf, int *outlen, const_uchar *in_buf, int *inlen) -cdef extern from "libxml/chvalid.h": - cdef int xmlIsChar_ch(char c) nogil - cdef int xmlIsCharQ(int ch) nogil +cdef extern from "libxml/chvalid.h" nogil: + cdef int xmlIsChar_ch(char c) + cdef int xmlIsCharQ(int ch) cdef extern from "libxml/hash.h": ctypedef struct xmlHashTable @@ -69,20 +69,20 @@ cdef extern from "libxml/hash.h": void xmlHashScan(xmlHashTable* table, xmlHashScanner f, void* data) nogil void* xmlHashLookup(xmlHashTable* table, const_xmlChar* name) nogil ctypedef void (*xmlHashDeallocator)(void *payload, xmlChar *name) - cdef xmlHashTable* xmlHashCreate(int size) - cdef xmlHashTable* xmlHashCreateDict(int size, xmlDict *dict) - cdef int xmlHashSize(xmlHashTable* table) - cdef void xmlHashFree(xmlHashTable* table, xmlHashDeallocator f) + cdef xmlHashTable* xmlHashCreate(int size) nogil + cdef xmlHashTable* xmlHashCreateDict(int size, xmlDict *dict) nogil + cdef int xmlHashSize(xmlHashTable* table) nogil + cdef void xmlHashFree(xmlHashTable* table, xmlHashDeallocator f) nogil -cdef extern from *: # actually "libxml/dict.h" +cdef extern from * nogil: # actually "libxml/dict.h" # libxml/dict.h appears to be broken to include in C ctypedef struct xmlDict - cdef const_xmlChar* xmlDictLookup(xmlDict* dict, const_xmlChar* name, int len) nogil - cdef const_xmlChar* xmlDictExists(xmlDict* dict, const_xmlChar* name, int len) nogil - cdef int xmlDictOwns(xmlDict* dict, const_xmlChar* name) nogil - cdef size_t xmlDictSize(xmlDict* dict) nogil + cdef const_xmlChar* xmlDictLookup(xmlDict* dict, const_xmlChar* name, int len) + cdef const_xmlChar* xmlDictExists(xmlDict* dict, const_xmlChar* name, int len) + cdef int xmlDictOwns(xmlDict* dict, const_xmlChar* name) + cdef size_t xmlDictSize(xmlDict* dict) -cdef extern from "libxml/tree.h": +cdef extern from "libxml/tree.h" nogil: ctypedef struct xmlDoc ctypedef struct xmlAttr ctypedef struct xmlNotationTable @@ -305,100 +305,100 @@ cdef extern from "libxml/tree.h": const_xmlChar* XML_XML_NAMESPACE - cdef void xmlFreeDoc(xmlDoc* cur) nogil - cdef void xmlFreeDtd(xmlDtd* cur) nogil - cdef void xmlFreeNode(xmlNode* cur) nogil - cdef void xmlFreeNsList(xmlNs* ns) nogil - cdef void xmlFreeNs(xmlNs* ns) nogil - cdef void xmlFree(void* buf) nogil + cdef void xmlFreeDoc(xmlDoc* cur) + cdef void xmlFreeDtd(xmlDtd* cur) + cdef void xmlFreeNode(xmlNode* cur) + cdef void xmlFreeNsList(xmlNs* ns) + cdef void xmlFreeNs(xmlNs* ns) + cdef void xmlFree(void* buf) - cdef xmlNode* xmlNewNode(xmlNs* ns, const_xmlChar* name) nogil - cdef xmlNode* xmlNewDocText(xmlDoc* doc, const_xmlChar* content) nogil - cdef xmlNode* xmlNewDocComment(xmlDoc* doc, const_xmlChar* content) nogil - cdef xmlNode* xmlNewDocPI(xmlDoc* doc, const_xmlChar* name, const_xmlChar* content) nogil - cdef xmlNode* xmlNewReference(xmlDoc* doc, const_xmlChar* name) nogil - cdef xmlNode* xmlNewCDataBlock(xmlDoc* doc, const_xmlChar* text, int len) nogil - cdef xmlNs* xmlNewNs(xmlNode* node, const_xmlChar* href, const_xmlChar* prefix) nogil - cdef xmlNode* xmlAddChild(xmlNode* parent, xmlNode* cur) nogil - cdef xmlNode* xmlReplaceNode(xmlNode* old, xmlNode* cur) nogil - cdef xmlNode* xmlAddPrevSibling(xmlNode* cur, xmlNode* elem) nogil - cdef xmlNode* xmlAddNextSibling(xmlNode* cur, xmlNode* elem) nogil + cdef xmlNode* xmlNewNode(xmlNs* ns, const_xmlChar* name) + cdef xmlNode* xmlNewDocText(xmlDoc* doc, const_xmlChar* content) + cdef xmlNode* xmlNewDocComment(xmlDoc* doc, const_xmlChar* content) + cdef xmlNode* xmlNewDocPI(xmlDoc* doc, const_xmlChar* name, const_xmlChar* content) + cdef xmlNode* xmlNewReference(xmlDoc* doc, const_xmlChar* name) + cdef xmlNode* xmlNewCDataBlock(xmlDoc* doc, const_xmlChar* text, int len) + cdef xmlNs* xmlNewNs(xmlNode* node, const_xmlChar* href, const_xmlChar* prefix) + cdef xmlNode* xmlAddChild(xmlNode* parent, xmlNode* cur) + cdef xmlNode* xmlReplaceNode(xmlNode* old, xmlNode* cur) + cdef xmlNode* xmlAddPrevSibling(xmlNode* cur, xmlNode* elem) + cdef xmlNode* xmlAddNextSibling(xmlNode* cur, xmlNode* elem) cdef xmlNode* xmlNewDocNode(xmlDoc* doc, xmlNs* ns, - const_xmlChar* name, const_xmlChar* content) nogil - cdef xmlDoc* xmlNewDoc(const_xmlChar* version) nogil - cdef xmlAttr* xmlNewProp(xmlNode* node, const_xmlChar* name, const_xmlChar* value) nogil + const_xmlChar* name, const_xmlChar* content) + cdef xmlDoc* xmlNewDoc(const_xmlChar* version) + cdef xmlAttr* xmlNewProp(xmlNode* node, const_xmlChar* name, const_xmlChar* value) cdef xmlAttr* xmlNewNsProp(xmlNode* node, xmlNs* ns, - const_xmlChar* name, const_xmlChar* value) nogil - cdef xmlChar* xmlGetNoNsProp(xmlNode* node, const_xmlChar* name) nogil - cdef xmlChar* xmlGetNsProp(xmlNode* node, const_xmlChar* name, const_xmlChar* nameSpace) nogil - cdef void xmlSetNs(xmlNode* node, xmlNs* ns) nogil - cdef xmlAttr* xmlSetProp(xmlNode* node, const_xmlChar* name, const_xmlChar* value) nogil + const_xmlChar* name, const_xmlChar* value) + cdef xmlChar* xmlGetNoNsProp(xmlNode* node, const_xmlChar* name) + cdef xmlChar* xmlGetNsProp(xmlNode* node, const_xmlChar* name, const_xmlChar* nameSpace) + cdef void xmlSetNs(xmlNode* node, xmlNs* ns) + cdef xmlAttr* xmlSetProp(xmlNode* node, const_xmlChar* name, const_xmlChar* value) cdef xmlAttr* xmlSetNsProp(xmlNode* node, xmlNs* ns, - const_xmlChar* name, const_xmlChar* value) nogil - cdef int xmlRemoveID(xmlDoc* doc, xmlAttr* cur) nogil - cdef int xmlRemoveProp(xmlAttr* cur) nogil - cdef void xmlFreePropList(xmlAttr* cur) nogil - cdef xmlChar* xmlGetNodePath(xmlNode* node) nogil - cdef void xmlDocDumpMemory(xmlDoc* cur, char** mem, int* size) nogil + const_xmlChar* name, const_xmlChar* value) + cdef int xmlRemoveID(xmlDoc* doc, xmlAttr* cur) + cdef int xmlRemoveProp(xmlAttr* cur) + cdef void xmlFreePropList(xmlAttr* cur) + cdef xmlChar* xmlGetNodePath(xmlNode* node) + cdef void xmlDocDumpMemory(xmlDoc* cur, char** mem, int* size) cdef void xmlDocDumpMemoryEnc(xmlDoc* cur, char** mem, int* size, - char* encoding) nogil + char* encoding) cdef int xmlSaveFileTo(xmlOutputBuffer* out, xmlDoc* cur, - char* encoding) nogil - - cdef void xmlUnlinkNode(xmlNode* cur) nogil - cdef xmlNode* xmlDocSetRootElement(xmlDoc* doc, xmlNode* root) nogil - cdef xmlNode* xmlDocGetRootElement(xmlDoc* doc) nogil - cdef void xmlSetTreeDoc(xmlNode* tree, xmlDoc* doc) nogil - cdef xmlAttr* xmlHasProp(xmlNode* node, const_xmlChar* name) nogil - cdef xmlAttr* xmlHasNsProp(xmlNode* node, const_xmlChar* name, const_xmlChar* nameSpace) nogil - cdef xmlChar* xmlNodeGetContent(xmlNode* cur) nogil - cdef int xmlNodeBufGetContent(xmlBuffer* buffer, xmlNode* cur) nogil - cdef xmlNs* xmlSearchNs(xmlDoc* doc, xmlNode* node, const_xmlChar* prefix) nogil - cdef xmlNs* xmlSearchNsByHref(xmlDoc* doc, xmlNode* node, const_xmlChar* href) nogil - cdef int xmlIsBlankNode(xmlNode* node) nogil - cdef long xmlGetLineNo(xmlNode* node) nogil - cdef void xmlElemDump(stdio.FILE* f, xmlDoc* doc, xmlNode* cur) nogil + char* encoding) + + cdef void xmlUnlinkNode(xmlNode* cur) + cdef xmlNode* xmlDocSetRootElement(xmlDoc* doc, xmlNode* root) + cdef xmlNode* xmlDocGetRootElement(xmlDoc* doc) + cdef void xmlSetTreeDoc(xmlNode* tree, xmlDoc* doc) + cdef xmlAttr* xmlHasProp(xmlNode* node, const_xmlChar* name) + cdef xmlAttr* xmlHasNsProp(xmlNode* node, const_xmlChar* name, const_xmlChar* nameSpace) + cdef xmlChar* xmlNodeGetContent(xmlNode* cur) + cdef int xmlNodeBufGetContent(xmlBuffer* buffer, xmlNode* cur) + cdef xmlNs* xmlSearchNs(xmlDoc* doc, xmlNode* node, const_xmlChar* prefix) + cdef xmlNs* xmlSearchNsByHref(xmlDoc* doc, xmlNode* node, const_xmlChar* href) + cdef int xmlIsBlankNode(xmlNode* node) + cdef long xmlGetLineNo(xmlNode* node) + cdef void xmlElemDump(stdio.FILE* f, xmlDoc* doc, xmlNode* cur) cdef void xmlNodeDumpOutput(xmlOutputBuffer* buf, xmlDoc* doc, xmlNode* cur, int level, - int format, const_char* encoding) nogil + int format, const_char* encoding) cdef void xmlBufAttrSerializeTxtContent(xmlOutputBuffer *buf, xmlDoc *doc, - xmlAttr *attr, const_xmlChar *string) nogil - cdef void xmlNodeSetName(xmlNode* cur, const_xmlChar* name) nogil - cdef void xmlNodeSetContent(xmlNode* cur, const_xmlChar* content) nogil - cdef xmlDtd* xmlCopyDtd(xmlDtd* dtd) nogil - cdef xmlDoc* xmlCopyDoc(xmlDoc* doc, int recursive) nogil - cdef xmlNode* xmlCopyNode(xmlNode* node, int extended) nogil - cdef xmlNode* xmlDocCopyNode(xmlNode* node, xmlDoc* doc, int extended) nogil - cdef int xmlReconciliateNs(xmlDoc* doc, xmlNode* tree) nogil - cdef xmlNs* xmlNewReconciliedNs(xmlDoc* doc, xmlNode* tree, xmlNs* ns) nogil - cdef xmlBuffer* xmlBufferCreate() nogil - cdef void xmlBufferWriteChar(xmlBuffer* buf, char* string) nogil - cdef void xmlBufferFree(xmlBuffer* buf) nogil - cdef const_xmlChar* xmlBufferContent(xmlBuffer* buf) nogil - cdef int xmlBufferLength(xmlBuffer* buf) nogil - cdef const_xmlChar* xmlBufContent(xmlBuf* buf) nogil # new in libxml2 2.9 - cdef size_t xmlBufUse(xmlBuf* buf) nogil # new in libxml2 2.9 - cdef int xmlKeepBlanksDefault(int val) nogil - cdef xmlChar* xmlNodeGetBase(xmlDoc* doc, xmlNode* node) nogil + xmlAttr *attr, const_xmlChar *string) + cdef void xmlNodeSetName(xmlNode* cur, const_xmlChar* name) + cdef void xmlNodeSetContent(xmlNode* cur, const_xmlChar* content) + cdef xmlDtd* xmlCopyDtd(xmlDtd* dtd) + cdef xmlDoc* xmlCopyDoc(xmlDoc* doc, int recursive) + cdef xmlNode* xmlCopyNode(xmlNode* node, int extended) + cdef xmlNode* xmlDocCopyNode(xmlNode* node, xmlDoc* doc, int extended) + cdef int xmlReconciliateNs(xmlDoc* doc, xmlNode* tree) + cdef xmlNs* xmlNewReconciliedNs(xmlDoc* doc, xmlNode* tree, xmlNs* ns) + cdef xmlBuffer* xmlBufferCreate() + cdef void xmlBufferWriteChar(xmlBuffer* buf, char* string) + cdef void xmlBufferFree(xmlBuffer* buf) + cdef const_xmlChar* xmlBufferContent(xmlBuffer* buf) + cdef int xmlBufferLength(xmlBuffer* buf) + cdef const_xmlChar* xmlBufContent(xmlBuf* buf) # new in libxml2 2.9 + cdef size_t xmlBufUse(xmlBuf* buf) # new in libxml2 2.9 + cdef int xmlKeepBlanksDefault(int val) + cdef xmlChar* xmlNodeGetBase(xmlDoc* doc, xmlNode* node) cdef xmlDtd* xmlCreateIntSubset(xmlDoc* doc, const_xmlChar* name, - const_xmlChar* ExternalID, const_xmlChar* SystemID) nogil - cdef void xmlNodeSetBase(xmlNode* node, const_xmlChar* uri) nogil - cdef int xmlValidateNCName(const_xmlChar* value, int space) nogil + const_xmlChar* ExternalID, const_xmlChar* SystemID) + cdef void xmlNodeSetBase(xmlNode* node, const_xmlChar* uri) + cdef int xmlValidateNCName(const_xmlChar* value, int space) -cdef extern from "libxml/uri.h": - cdef const_xmlChar* xmlBuildURI(const_xmlChar* href, const_xmlChar* base) nogil +cdef extern from "libxml/uri.h" nogil: + cdef const_xmlChar* xmlBuildURI(const_xmlChar* href, const_xmlChar* base) -cdef extern from "libxml/HTMLtree.h": +cdef extern from "libxml/HTMLtree.h" nogil: cdef void htmlNodeDumpFormatOutput(xmlOutputBuffer* buf, xmlDoc* doc, xmlNode* cur, - char* encoding, int format) nogil - cdef xmlDoc* htmlNewDoc(const_xmlChar* uri, const_xmlChar* externalID) nogil + char* encoding, int format) + cdef xmlDoc* htmlNewDoc(const_xmlChar* uri, const_xmlChar* externalID) -cdef extern from "libxml/valid.h": - cdef xmlAttr* xmlGetID(xmlDoc* doc, const_xmlChar* ID) nogil +cdef extern from "libxml/valid.h" nogil: + cdef xmlAttr* xmlGetID(xmlDoc* doc, const_xmlChar* ID) cdef void xmlDumpNotationTable(xmlBuffer* buffer, - xmlNotationTable* table) nogil - cdef int xmlValidateNameValue(const_xmlChar* value) nogil + xmlNotationTable* table) + cdef int xmlValidateNameValue(const_xmlChar* value) cdef extern from "libxml/xmlIO.h": cdef int xmlOutputBufferWrite(xmlOutputBuffer* out, @@ -411,8 +411,8 @@ cdef extern from "libxml/xmlIO.h": cdef int xmlOutputBufferClose(xmlOutputBuffer* out) nogil ctypedef int (*xmlInputReadCallback)(void* context, - char* buffer, int len) - ctypedef int (*xmlInputCloseCallback)(void* context) + char* buffer, int len) nogil + ctypedef int (*xmlInputCloseCallback)(void* context) nogil ctypedef int (*xmlOutputWriteCallback)(void* context, char* buffer, int len) @@ -430,7 +430,7 @@ cdef extern from "libxml/xmlIO.h": cdef xmlOutputBuffer* xmlOutputBufferCreateFilename( char* URI, xmlCharEncodingHandler* encoder, int compression) nogil -cdef extern from "libxml/xmlsave.h": +cdef extern from "libxml/xmlsave.h" nogil: ctypedef struct xmlSaveCtxt ctypedef enum xmlSaveOption: @@ -443,20 +443,20 @@ cdef extern from "libxml/xmlsave.h": XML_SAVE_AS_HTML = 64 # force HTML serialization on XML doc (2.7.2) cdef xmlSaveCtxt* xmlSaveToFilename(char* filename, char* encoding, - int options) nogil + int options) cdef xmlSaveCtxt* xmlSaveToBuffer(xmlBuffer* buffer, char* encoding, - int options) nogil # libxml2 2.6.23 - cdef long xmlSaveDoc(xmlSaveCtxt* ctxt, xmlDoc* doc) nogil - cdef long xmlSaveTree(xmlSaveCtxt* ctxt, xmlNode* node) nogil - cdef int xmlSaveClose(xmlSaveCtxt* ctxt) nogil - cdef int xmlSaveFlush(xmlSaveCtxt* ctxt) nogil - cdef int xmlSaveSetAttrEscape(xmlSaveCtxt* ctxt, void* escape_func) nogil - cdef int xmlSaveSetEscape(xmlSaveCtxt* ctxt, void* escape_func) nogil - -cdef extern from "libxml/globals.h": - cdef int xmlThrDefKeepBlanksDefaultValue(int onoff) nogil - cdef int xmlThrDefLineNumbersDefaultValue(int onoff) nogil - cdef int xmlThrDefIndentTreeOutput(int onoff) nogil + int options) # libxml2 2.6.23 + cdef long xmlSaveDoc(xmlSaveCtxt* ctxt, xmlDoc* doc) + cdef long xmlSaveTree(xmlSaveCtxt* ctxt, xmlNode* node) + cdef int xmlSaveClose(xmlSaveCtxt* ctxt) + cdef int xmlSaveFlush(xmlSaveCtxt* ctxt) + cdef int xmlSaveSetAttrEscape(xmlSaveCtxt* ctxt, void* escape_func) + cdef int xmlSaveSetEscape(xmlSaveCtxt* ctxt, void* escape_func) + +cdef extern from "libxml/globals.h" nogil: + cdef int xmlThrDefKeepBlanksDefaultValue(int onoff) + cdef int xmlThrDefLineNumbersDefaultValue(int onoff) + cdef int xmlThrDefIndentTreeOutput(int onoff) cdef extern from "libxml/xmlmemory.h" nogil: cdef void* xmlMalloc(size_t size) @@ -466,15 +466,15 @@ cdef extern from "libxml/xmlmemory.h" nogil: cdef void xmlMemDisplayLast(stdio.FILE* file, long num_bytes) cdef void xmlMemShow(stdio.FILE* file, int count) -cdef extern from "etree_defs.h": - cdef bint _isElement(xmlNode* node) nogil - cdef bint _isElementOrXInclude(xmlNode* node) nogil - cdef const_xmlChar* _getNs(xmlNode* node) nogil +cdef extern from "etree_defs.h" nogil: + cdef bint _isElement(xmlNode* node) + cdef bint _isElementOrXInclude(xmlNode* node) + cdef const_xmlChar* _getNs(xmlNode* node) cdef void BEGIN_FOR_EACH_ELEMENT_FROM(xmlNode* tree_top, xmlNode* start_node, - bint inclusive) nogil - cdef void END_FOR_EACH_ELEMENT_FROM(xmlNode* start_node) nogil + bint inclusive) + cdef void END_FOR_EACH_ELEMENT_FROM(xmlNode* start_node) cdef void BEGIN_FOR_EACH_FROM(xmlNode* tree_top, xmlNode* start_node, - bint inclusive) nogil - cdef void END_FOR_EACH_FROM(xmlNode* start_node) nogil + bint inclusive) + cdef void END_FOR_EACH_FROM(xmlNode* start_node) diff --git a/src/lxml/includes/uri.pxd b/src/lxml/includes/uri.pxd index 2b6bb79f3..f886a54b9 100644 --- a/src/lxml/includes/uri.pxd +++ b/src/lxml/includes/uri.pxd @@ -1,4 +1,4 @@ -cdef extern from "libxml/uri.h": +cdef extern from "libxml/uri.h" nogil: ctypedef struct xmlURI cdef xmlURI* xmlParseURI(char* str) diff --git a/src/lxml/includes/xinclude.pxd b/src/lxml/includes/xinclude.pxd index 4232d3e43..68267175a 100644 --- a/src/lxml/includes/xinclude.pxd +++ b/src/lxml/includes/xinclude.pxd @@ -1,22 +1,22 @@ from lxml.includes.tree cimport xmlDoc, xmlNode -cdef extern from "libxml/xinclude.h": +cdef extern from "libxml/xinclude.h" nogil: ctypedef struct xmlXIncludeCtxt - cdef int xmlXIncludeProcess(xmlDoc* doc) nogil - cdef int xmlXIncludeProcessFlags(xmlDoc* doc, int parser_opts) nogil - cdef int xmlXIncludeProcessTree(xmlNode* doc) nogil - cdef int xmlXIncludeProcessTreeFlags(xmlNode* doc, int parser_opts) nogil + cdef int xmlXIncludeProcess(xmlDoc* doc) + cdef int xmlXIncludeProcessFlags(xmlDoc* doc, int parser_opts) + cdef int xmlXIncludeProcessTree(xmlNode* doc) + cdef int xmlXIncludeProcessTreeFlags(xmlNode* doc, int parser_opts) # libxml2 >= 2.7.4 cdef int xmlXIncludeProcessTreeFlagsData( - xmlNode* doc, int parser_opts, void* data) nogil + xmlNode* doc, int parser_opts, void* data) - cdef xmlXIncludeCtxt* xmlXIncludeNewContext(xmlDoc* doc) nogil - cdef int xmlXIncludeProcessNode(xmlXIncludeCtxt* ctxt, xmlNode* node) nogil - cdef int xmlXIncludeSetFlags(xmlXIncludeCtxt* ctxt, int flags) nogil + cdef xmlXIncludeCtxt* xmlXIncludeNewContext(xmlDoc* doc) + cdef int xmlXIncludeProcessNode(xmlXIncludeCtxt* ctxt, xmlNode* node) + cdef int xmlXIncludeSetFlags(xmlXIncludeCtxt* ctxt, int flags) # libxml2 >= 2.6.27 cdef int xmlXIncludeProcessFlagsData( - xmlDoc* doc, int flags, void* data) nogil + xmlDoc* doc, int flags, void* data) diff --git a/src/lxml/includes/xmlerror.pxd b/src/lxml/includes/xmlerror.pxd index 13c8f3782..c5ac6a0aa 100644 --- a/src/lxml/includes/xmlerror.pxd +++ b/src/lxml/includes/xmlerror.pxd @@ -823,7 +823,7 @@ cdef extern from "libxml/xmlerror.h": XML_RELAXNG_ERR_TEXTWRONG = 39 # --- END: GENERATED CONSTANTS --- -cdef extern from "libxml/xmlerror.h": +cdef extern from "libxml/xmlerror.h" nogil: ctypedef struct xmlError: int domain int code @@ -838,15 +838,15 @@ cdef extern from "libxml/xmlerror.h": int int2 void* node - ctypedef void (*xmlGenericErrorFunc)(void* ctxt, char* msg, ...) nogil + ctypedef void (*xmlGenericErrorFunc)(void* ctxt, char* msg, ...) ctypedef void (*xmlStructuredErrorFunc)(void* userData, - xmlError* error) nogil + xmlError* error) cdef void xmlSetGenericErrorFunc( - void* ctxt, xmlGenericErrorFunc func) nogil + void* ctxt, xmlGenericErrorFunc func) cdef void xmlSetStructuredErrorFunc( - void* ctxt, xmlStructuredErrorFunc func) nogil + void* ctxt, xmlStructuredErrorFunc func) -cdef extern from "libxml/globals.h": +cdef extern from "libxml/globals.h" nogil: cdef xmlStructuredErrorFunc xmlStructuredError cdef void* xmlStructuredErrorContext diff --git a/src/lxml/includes/xmlparser.pxd b/src/lxml/includes/xmlparser.pxd index 45acfc846..9f3056248 100644 --- a/src/lxml/includes/xmlparser.pxd +++ b/src/lxml/includes/xmlparser.pxd @@ -6,7 +6,7 @@ from lxml.includes.tree cimport xmlInputReadCallback, xmlInputCloseCallback from lxml.includes.xmlerror cimport xmlError, xmlStructuredErrorFunc -cdef extern from "libxml/parser.h": +cdef extern from "libxml/parser.h" nogil: ctypedef void (*startElementNsSAX2Func)(void* ctx, const_xmlChar* localname, const_xmlChar* prefix, @@ -49,7 +49,7 @@ cdef extern from "libxml/parser.h": cdef int XML_SAX2_MAGIC -cdef extern from "libxml/tree.h": +cdef extern from "libxml/tree.h" nogil: ctypedef struct xmlParserInput: int line int length @@ -93,12 +93,12 @@ cdef extern from "libxml/xmlIO.h" nogil: cdef xmlParserInputBuffer* xmlAllocParserInputBuffer(int enc) -cdef extern from "libxml/parser.h": +cdef extern from "libxml/parser.h" nogil: - cdef xmlDict* xmlDictCreate() nogil - cdef xmlDict* xmlDictCreateSub(xmlDict* subdict) nogil - cdef void xmlDictFree(xmlDict* sub) nogil - cdef int xmlDictReference(xmlDict* dict) nogil + cdef xmlDict* xmlDictCreate() + cdef xmlDict* xmlDictCreateSub(xmlDict* subdict) + cdef void xmlDictFree(xmlDict* sub) + cdef int xmlDictReference(xmlDict* dict) cdef int XML_COMPLETE_ATTRS # SAX option for adding DTD default attributes cdef int XML_SKIP_IDS # SAX option for not building an XML ID dict @@ -181,36 +181,36 @@ cdef extern from "libxml/parser.h": # libxml2 2.9.0+ only: XML_PARSE_BIG_LINES = 4194304 # Store big lines numbers in text PSVI field - cdef void xmlInitParser() nogil - cdef void xmlCleanupParser() nogil + cdef void xmlInitParser() + cdef void xmlCleanupParser() - cdef int xmlLineNumbersDefault(int onoff) nogil - cdef xmlParserCtxt* xmlNewParserCtxt() nogil + cdef int xmlLineNumbersDefault(int onoff) + cdef xmlParserCtxt* xmlNewParserCtxt() cdef xmlParserInput* xmlNewIOInputStream(xmlParserCtxt* ctxt, xmlParserInputBuffer* input, - int enc) nogil - cdef int xmlCtxtUseOptions(xmlParserCtxt* ctxt, int options) nogil - cdef void xmlFreeParserCtxt(xmlParserCtxt* ctxt) nogil - cdef void xmlCtxtReset(xmlParserCtxt* ctxt) nogil - cdef void xmlClearParserCtxt(xmlParserCtxt* ctxt) nogil + int enc) + cdef int xmlCtxtUseOptions(xmlParserCtxt* ctxt, int options) + cdef void xmlFreeParserCtxt(xmlParserCtxt* ctxt) + cdef void xmlCtxtReset(xmlParserCtxt* ctxt) + cdef void xmlClearParserCtxt(xmlParserCtxt* ctxt) cdef int xmlParseChunk(xmlParserCtxt* ctxt, - char* chunk, int size, int terminate) nogil + char* chunk, int size, int terminate) cdef xmlDoc* xmlCtxtReadDoc(xmlParserCtxt* ctxt, char* cur, char* URL, char* encoding, - int options) nogil + int options) cdef xmlDoc* xmlCtxtReadFile(xmlParserCtxt* ctxt, char* filename, char* encoding, - int options) nogil + int options) cdef xmlDoc* xmlCtxtReadIO(xmlParserCtxt* ctxt, xmlInputReadCallback ioread, xmlInputCloseCallback ioclose, void* ioctx, char* URL, char* encoding, - int options) nogil + int options) cdef xmlDoc* xmlCtxtReadMemory(xmlParserCtxt* ctxt, char* buffer, int size, char* filename, const_char* encoding, - int options) nogil + int options) # iterparse: @@ -218,33 +218,34 @@ cdef extern from "libxml/parser.h": void* user_data, char* chunk, int size, - char* filename) nogil + char* filename) cdef int xmlCtxtResetPush(xmlParserCtxt* ctxt, char* chunk, int size, char* filename, - char* encoding) nogil + char* encoding) # entity loaders: ctypedef xmlParserInput* (*xmlExternalEntityLoader)( - const_char * URL, const_char * ID, xmlParserCtxt* context) nogil - cdef xmlExternalEntityLoader xmlGetExternalEntityLoader() nogil - cdef void xmlSetExternalEntityLoader(xmlExternalEntityLoader f) nogil + const_char * URL, const_char * ID, xmlParserCtxt* context) + cdef xmlExternalEntityLoader xmlGetExternalEntityLoader() + cdef void xmlSetExternalEntityLoader(xmlExternalEntityLoader f) # DTDs: - cdef xmlDtd* xmlParseDTD(const_xmlChar* ExternalID, const_xmlChar* SystemID) nogil + cdef xmlDtd* xmlParseDTD(const_xmlChar* ExternalID, const_xmlChar* SystemID) cdef xmlDtd* xmlIOParseDTD(xmlSAXHandler* sax, xmlParserInputBuffer* input, - int enc) nogil + int enc) -cdef extern from "libxml/parserInternals.h": + +cdef extern from "libxml/parserInternals.h" nogil: cdef xmlParserInput* xmlNewInputStream(xmlParserCtxt* ctxt) cdef xmlParserInput* xmlNewStringInputStream(xmlParserCtxt* ctxt, - char* buffer) nogil + char* buffer) cdef xmlParserInput* xmlNewInputFromFile(xmlParserCtxt* ctxt, - char* filename) nogil - cdef void xmlFreeInputStream(xmlParserInput* input) nogil - cdef int xmlSwitchEncoding(xmlParserCtxt* ctxt, int enc) nogil + char* filename) + cdef void xmlFreeInputStream(xmlParserInput* input) + cdef int xmlSwitchEncoding(xmlParserCtxt* ctxt, int enc) diff --git a/src/lxml/includes/xmlschema.pxd b/src/lxml/includes/xmlschema.pxd index 8e93cc570..067411113 100644 --- a/src/lxml/includes/xmlschema.pxd +++ b/src/lxml/includes/xmlschema.pxd @@ -2,7 +2,7 @@ from lxml.includes.tree cimport xmlDoc from lxml.includes.xmlparser cimport xmlSAXHandler from lxml.includes.xmlerror cimport xmlStructuredErrorFunc -cdef extern from "libxml/xmlschemas.h": +cdef extern from "libxml/xmlschemas.h" nogil: ctypedef struct xmlSchema ctypedef struct xmlSchemaParserCtxt diff --git a/src/lxml/includes/xpath.pxd b/src/lxml/includes/xpath.pxd index d01735b68..22069eb7c 100644 --- a/src/lxml/includes/xpath.pxd +++ b/src/lxml/includes/xpath.pxd @@ -4,7 +4,8 @@ from lxml.includes cimport xmlerror from libc.string cimport const_char from lxml.includes.tree cimport xmlChar, const_xmlChar -cdef extern from "libxml/xpath.h": + +cdef extern from "libxml/xpath.h" nogil: ctypedef enum xmlXPathObjectType: XPATH_UNDEFINED = 0 XPATH_NODESET = 1 @@ -73,63 +74,63 @@ cdef extern from "libxml/xpath.h": ctypedef struct xmlXPathCompExpr - ctypedef void (*xmlXPathFunction)(xmlXPathParserContext* ctxt, int nargs) nogil + ctypedef void (*xmlXPathFunction)(xmlXPathParserContext* ctxt, int nargs) ctypedef xmlXPathFunction (*xmlXPathFuncLookupFunc)(void* ctxt, const_xmlChar* name, - const_xmlChar* ns_uri) nogil + const_xmlChar* ns_uri) - cdef xmlXPathContext* xmlXPathNewContext(tree.xmlDoc* doc) nogil + cdef xmlXPathContext* xmlXPathNewContext(tree.xmlDoc* doc) cdef xmlXPathObject* xmlXPathEvalExpression(const_xmlChar* str, - xmlXPathContext* ctxt) nogil + xmlXPathContext* ctxt) cdef xmlXPathObject* xmlXPathCompiledEval(xmlXPathCompExpr* comp, - xmlXPathContext* ctxt) nogil - cdef xmlXPathCompExpr* xmlXPathCompile(const_xmlChar* str) nogil + xmlXPathContext* ctxt) + cdef xmlXPathCompExpr* xmlXPathCompile(const_xmlChar* str) cdef xmlXPathCompExpr* xmlXPathCtxtCompile(xmlXPathContext* ctxt, - const_xmlChar* str) nogil - cdef void xmlXPathFreeContext(xmlXPathContext* ctxt) nogil - cdef void xmlXPathFreeCompExpr(xmlXPathCompExpr* comp) nogil - cdef void xmlXPathFreeObject(xmlXPathObject* obj) nogil + const_xmlChar* str) + cdef void xmlXPathFreeContext(xmlXPathContext* ctxt) + cdef void xmlXPathFreeCompExpr(xmlXPathCompExpr* comp) + cdef void xmlXPathFreeObject(xmlXPathObject* obj) cdef int xmlXPathRegisterNs(xmlXPathContext* ctxt, - const_xmlChar* prefix, const_xmlChar* ns_uri) nogil + const_xmlChar* prefix, const_xmlChar* ns_uri) - cdef xmlNodeSet* xmlXPathNodeSetCreate(tree.xmlNode* val) nogil - cdef void xmlXPathFreeNodeSet(xmlNodeSet* val) nogil + cdef xmlNodeSet* xmlXPathNodeSetCreate(tree.xmlNode* val) + cdef void xmlXPathFreeNodeSet(xmlNodeSet* val) -cdef extern from "libxml/xpathInternals.h": +cdef extern from "libxml/xpathInternals.h" nogil: cdef int xmlXPathRegisterFunc(xmlXPathContext* ctxt, const_xmlChar* name, - xmlXPathFunction f) nogil + xmlXPathFunction f) cdef int xmlXPathRegisterFuncNS(xmlXPathContext* ctxt, const_xmlChar* name, const_xmlChar* ns_uri, - xmlXPathFunction f) nogil + xmlXPathFunction f) cdef void xmlXPathRegisterFuncLookup(xmlXPathContext *ctxt, xmlXPathFuncLookupFunc f, - void *funcCtxt) nogil + void *funcCtxt) cdef int xmlXPathRegisterVariable(xmlXPathContext *ctxt, const_xmlChar* name, - xmlXPathObject* value) nogil + xmlXPathObject* value) cdef int xmlXPathRegisterVariableNS(xmlXPathContext *ctxt, const_xmlChar* name, const_xmlChar* ns_uri, - xmlXPathObject* value) nogil - cdef void xmlXPathRegisteredVariablesCleanup(xmlXPathContext *ctxt) nogil - cdef void xmlXPathRegisteredNsCleanup(xmlXPathContext *ctxt) nogil - cdef xmlXPathObject* valuePop (xmlXPathParserContext *ctxt) nogil - cdef int valuePush(xmlXPathParserContext* ctxt, xmlXPathObject *value) nogil + xmlXPathObject* value) + cdef void xmlXPathRegisteredVariablesCleanup(xmlXPathContext *ctxt) + cdef void xmlXPathRegisteredNsCleanup(xmlXPathContext *ctxt) + cdef xmlXPathObject* valuePop (xmlXPathParserContext *ctxt) + cdef int valuePush(xmlXPathParserContext* ctxt, xmlXPathObject *value) - cdef xmlXPathObject* xmlXPathNewCString(const_char *val) nogil - cdef xmlXPathObject* xmlXPathWrapCString(const_char * val) nogil - cdef xmlXPathObject* xmlXPathNewString(const_xmlChar *val) nogil - cdef xmlXPathObject* xmlXPathWrapString(const_xmlChar * val) nogil - cdef xmlXPathObject* xmlXPathNewFloat(double val) nogil - cdef xmlXPathObject* xmlXPathNewBoolean(int val) nogil - cdef xmlXPathObject* xmlXPathNewNodeSet(tree.xmlNode* val) nogil - cdef xmlXPathObject* xmlXPathNewValueTree(tree.xmlNode* val) nogil + cdef xmlXPathObject* xmlXPathNewCString(const_char *val) + cdef xmlXPathObject* xmlXPathWrapCString(const_char * val) + cdef xmlXPathObject* xmlXPathNewString(const_xmlChar *val) + cdef xmlXPathObject* xmlXPathWrapString(const_xmlChar * val) + cdef xmlXPathObject* xmlXPathNewFloat(double val) + cdef xmlXPathObject* xmlXPathNewBoolean(int val) + cdef xmlXPathObject* xmlXPathNewNodeSet(tree.xmlNode* val) + cdef xmlXPathObject* xmlXPathNewValueTree(tree.xmlNode* val) cdef void xmlXPathNodeSetAdd(xmlNodeSet* cur, - tree.xmlNode* val) nogil + tree.xmlNode* val) cdef void xmlXPathNodeSetAddUnique(xmlNodeSet* cur, - tree.xmlNode* val) nogil - cdef xmlXPathObject* xmlXPathWrapNodeSet(xmlNodeSet* val) nogil - cdef void xmlXPathErr(xmlXPathParserContext* ctxt, int error) nogil + tree.xmlNode* val) + cdef xmlXPathObject* xmlXPathWrapNodeSet(xmlNodeSet* val) + cdef void xmlXPathErr(xmlXPathParserContext* ctxt, int error) diff --git a/src/lxml/includes/xslt.pxd b/src/lxml/includes/xslt.pxd index 101fb7e78..05d2d4b9b 100644 --- a/src/lxml/includes/xslt.pxd +++ b/src/lxml/includes/xslt.pxd @@ -11,7 +11,7 @@ cdef extern from "libxslt/xslt.h": cdef extern from "libxslt/xsltconfig.h": cdef int LIBXSLT_VERSION -cdef extern from "libxslt/xsltInternals.h": +cdef extern from "libxslt/xsltInternals.h" nogil: ctypedef enum xsltTransformState: XSLT_STATE_OK # 0 XSLT_STATE_ERROR # 1 @@ -42,35 +42,35 @@ cdef extern from "libxslt/xsltInternals.h": ctypedef struct xsltTemplate - cdef xsltStylesheet* xsltParseStylesheetDoc(xmlDoc* doc) nogil - cdef void xsltFreeStylesheet(xsltStylesheet* sheet) nogil + cdef xsltStylesheet* xsltParseStylesheetDoc(xmlDoc* doc) + cdef void xsltFreeStylesheet(xsltStylesheet* sheet) -cdef extern from "libxslt/imports.h": +cdef extern from "libxslt/imports.h" nogil: # actually defined in "etree_defs.h" cdef void LXML_GET_XSLT_ENCODING(const_xmlChar* result_var, xsltStylesheet* style) -cdef extern from "libxslt/extensions.h": +cdef extern from "libxslt/extensions.h" nogil: ctypedef void (*xsltTransformFunction)(xsltTransformContext* ctxt, xmlNode* context_node, xmlNode* inst, - void* precomp_unused) nogil + void* precomp_unused) cdef int xsltRegisterExtFunction(xsltTransformContext* ctxt, const_xmlChar* name, const_xmlChar* URI, - xmlXPathFunction function) nogil + xmlXPathFunction function) cdef int xsltRegisterExtModuleFunction(const_xmlChar* name, const_xmlChar* URI, - xmlXPathFunction function) nogil + xmlXPathFunction function) cdef int xsltUnregisterExtModuleFunction(const_xmlChar* name, const_xmlChar* URI) cdef xmlXPathFunction xsltExtModuleFunctionLookup( - const_xmlChar* name, const_xmlChar* URI) nogil + const_xmlChar* name, const_xmlChar* URI) cdef int xsltRegisterExtPrefix(xsltStylesheet* style, - const_xmlChar* prefix, const_xmlChar* URI) nogil + const_xmlChar* prefix, const_xmlChar* URI) cdef int xsltRegisterExtElement(xsltTransformContext* ctxt, const_xmlChar* name, const_xmlChar* URI, - xsltTransformFunction function) nogil + xsltTransformFunction function) -cdef extern from "libxslt/documents.h": +cdef extern from "libxslt/documents.h" nogil: ctypedef enum xsltLoadType: XSLT_LOAD_START XSLT_LOAD_STYLESHEET @@ -79,48 +79,48 @@ cdef extern from "libxslt/documents.h": ctypedef xmlDoc* (*xsltDocLoaderFunc)(const_xmlChar* URI, xmlDict* dict, int options, void* ctxt, - xsltLoadType type) nogil + xsltLoadType type) cdef xsltDocLoaderFunc xsltDocDefaultLoader - cdef void xsltSetLoaderFunc(xsltDocLoaderFunc f) nogil + cdef void xsltSetLoaderFunc(xsltDocLoaderFunc f) -cdef extern from "libxslt/transform.h": +cdef extern from "libxslt/transform.h" nogil: cdef xmlDoc* xsltApplyStylesheet(xsltStylesheet* style, xmlDoc* doc, - const_char** params) nogil + const_char** params) cdef xmlDoc* xsltApplyStylesheetUser(xsltStylesheet* style, xmlDoc* doc, const_char** params, const_char* output, void* profile, - xsltTransformContext* context) nogil + xsltTransformContext* context) cdef void xsltProcessOneNode(xsltTransformContext* ctxt, xmlNode* contextNode, - xsltStackElem* params) nogil + xsltStackElem* params) cdef xsltTransformContext* xsltNewTransformContext(xsltStylesheet* style, - xmlDoc* doc) nogil - cdef void xsltFreeTransformContext(xsltTransformContext* context) nogil + xmlDoc* doc) + cdef void xsltFreeTransformContext(xsltTransformContext* context) cdef void xsltApplyOneTemplate(xsltTransformContext* ctxt, xmlNode* contextNode, xmlNode* list, xsltTemplate* templ, - xsltStackElem* params) nogil + xsltStackElem* params) -cdef extern from "libxslt/xsltutils.h": +cdef extern from "libxslt/xsltutils.h" nogil: cdef int xsltSaveResultToString(xmlChar** doc_txt_ptr, int* doc_txt_len, xmlDoc* result, - xsltStylesheet* style) nogil + xsltStylesheet* style) cdef int xsltSaveResultToFilename(const_char *URL, xmlDoc* result, xsltStylesheet* style, - int compression) nogil + int compression) cdef int xsltSaveResultTo(xmlOutputBuffer* buf, xmlDoc* result, - xsltStylesheet* style) nogil + xsltStylesheet* style) cdef xmlGenericErrorFunc xsltGenericError cdef void *xsltGenericErrorContext cdef void xsltSetGenericErrorFunc( - void* ctxt, void (*handler)(void* ctxt, char* msg, ...)) nogil + void* ctxt, void (*handler)(void* ctxt, char* msg, ...) nogil) cdef void xsltSetTransformErrorFunc( xsltTransformContext*, void* ctxt, - void (*handler)(void* ctxt, char* msg, ...) nogil) nogil + void (*handler)(void* ctxt, char* msg, ...) nogil) cdef void xsltTransformError(xsltTransformContext* ctxt, xsltStylesheet* style, xmlNode* node, char* msg, ...) @@ -128,7 +128,7 @@ cdef extern from "libxslt/xsltutils.h": xsltTransformContext* ctxt, int options) -cdef extern from "libxslt/security.h": +cdef extern from "libxslt/security.h" nogil: ctypedef struct xsltSecurityPrefs ctypedef enum xsltSecurityOption: XSLT_SECPREF_READ_FILE = 1 @@ -139,44 +139,44 @@ cdef extern from "libxslt/security.h": ctypedef int (*xsltSecurityCheck)(xsltSecurityPrefs* sec, xsltTransformContext* ctxt, - char* value) nogil + char* value) - cdef xsltSecurityPrefs* xsltNewSecurityPrefs() nogil - cdef void xsltFreeSecurityPrefs(xsltSecurityPrefs* sec) nogil + cdef xsltSecurityPrefs* xsltNewSecurityPrefs() + cdef void xsltFreeSecurityPrefs(xsltSecurityPrefs* sec) cdef int xsltSecurityForbid(xsltSecurityPrefs* sec, xsltTransformContext* ctxt, - char* value) nogil + char* value) cdef int xsltSecurityAllow(xsltSecurityPrefs* sec, xsltTransformContext* ctxt, - char* value) nogil + char* value) cdef int xsltSetSecurityPrefs(xsltSecurityPrefs* sec, xsltSecurityOption option, - xsltSecurityCheck func) nogil + xsltSecurityCheck func) cdef xsltSecurityCheck xsltGetSecurityPrefs( xsltSecurityPrefs* sec, - xsltSecurityOption option) nogil + xsltSecurityOption option) cdef int xsltSetCtxtSecurityPrefs(xsltSecurityPrefs* sec, - xsltTransformContext* ctxt) nogil - cdef xmlDoc* xsltGetProfileInformation(xsltTransformContext* ctxt) nogil + xsltTransformContext* ctxt) + cdef xmlDoc* xsltGetProfileInformation(xsltTransformContext* ctxt) -cdef extern from "libxslt/variables.h": +cdef extern from "libxslt/variables.h" nogil: cdef int xsltQuoteUserParams(xsltTransformContext* ctxt, const_char** params) cdef int xsltQuoteOneUserParam(xsltTransformContext* ctxt, const_xmlChar* name, const_xmlChar* value) -cdef extern from "libxslt/extra.h": +cdef extern from "libxslt/extra.h" nogil: const_xmlChar* XSLT_LIBXSLT_NAMESPACE const_xmlChar* XSLT_XALAN_NAMESPACE const_xmlChar* XSLT_SAXON_NAMESPACE const_xmlChar* XSLT_XT_NAMESPACE cdef xmlXPathFunction xsltFunctionNodeSet - cdef void xsltRegisterAllExtras() nogil + cdef void xsltRegisterAllExtras() -cdef extern from "libexslt/exslt.h": - cdef void exsltRegisterAll() nogil +cdef extern from "libexslt/exslt.h" nogil: + cdef void exsltRegisterAll() # libexslt 1.1.25+ const_xmlChar* EXSLT_DATE_NAMESPACE @@ -188,4 +188,3 @@ cdef extern from "libexslt/exslt.h": cdef int exsltSetsXpathCtxtRegister(xmlXPathContext* ctxt, const_xmlChar* prefix) cdef int exsltMathXpathCtxtRegister(xmlXPathContext* ctxt, const_xmlChar* prefix) cdef int exsltStrXpathCtxtRegister(xmlXPathContext* ctxt, const_xmlChar* prefix) - From 1abcacfca67b5aa6c791462916d77899a7a7db48 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 22 Dec 2022 07:36:40 +0100 Subject: [PATCH 0051/1154] Call super() from CSSSelector constructor to help with used side inheritance. --- src/lxml/cssselect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lxml/cssselect.py b/src/lxml/cssselect.py index 586a1427c..89d838850 100644 --- a/src/lxml/cssselect.py +++ b/src/lxml/cssselect.py @@ -92,7 +92,7 @@ def __init__(self, css, namespaces=None, translator='xml'): elif translator == 'xhtml': translator = LxmlHTMLTranslator(xhtml=True) path = translator.css_to_xpath(css) - etree.XPath.__init__(self, path, namespaces=namespaces) + super(CSSSelector, self).__init__(path, namespaces=namespaces) self.css = css def __repr__(self): From ec86e2c6b5241b330934d1aee81583a1b5897ca7 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 22 Dec 2022 07:38:01 +0100 Subject: [PATCH 0052/1154] Simplify code. --- src/lxml/cssselect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lxml/cssselect.py b/src/lxml/cssselect.py index 89d838850..8c1893668 100644 --- a/src/lxml/cssselect.py +++ b/src/lxml/cssselect.py @@ -96,7 +96,7 @@ def __init__(self, css, namespaces=None, translator='xml'): self.css = css def __repr__(self): - return '<%s %s for %r>' % ( + return '<%s %x for %r>' % ( self.__class__.__name__, - hex(abs(id(self)))[2:], + abs(id(self)), self.css) From 9a6cdd09b9be84b0dae875a4dc1e7bdb27dcc7d9 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 22 Dec 2022 07:42:35 +0100 Subject: [PATCH 0053/1154] Replace outdated URL in error message. --- src/lxml/cssselect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lxml/cssselect.py b/src/lxml/cssselect.py index 8c1893668..e8ce5c8e1 100644 --- a/src/lxml/cssselect.py +++ b/src/lxml/cssselect.py @@ -14,7 +14,7 @@ except ImportError: raise ImportError( 'cssselect does not seem to be installed. ' - 'See http://packages.python.org/cssselect/') + 'See https://pypi.org/project/cssselect/') SelectorSyntaxError = external_cssselect.SelectorSyntaxError From 6b2698d7678572edf81890f9494edafd65751627 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 22 Dec 2022 09:55:26 +0100 Subject: [PATCH 0054/1154] Update to latest library versions. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/wheels.yml | 2 +- Makefile | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd1b9ea6e..aa28205fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,8 +132,8 @@ jobs: OS_NAME: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} MACOSX_DEPLOYMENT_TARGET: 10.15 - LIBXML2_VERSION: 2.9.14 - LIBXSLT_VERSION: 1.1.35 + LIBXML2_VERSION: 2.10.3 + LIBXSLT_VERSION: 1.1.37 COVERAGE: false GCC_VERSION: 9 USE_CCACHE: 1 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 89078587b..66fca6067 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -147,7 +147,7 @@ jobs: python-version: 2.7 # needs older image runs-on: ${{ matrix.os }} - env: { LIBXML2_VERSION: 2.9.14, LIBXSLT_VERSION: 1.1.35, MACOSX_DEPLOYMENT_TARGET: 10.15 } + env: { LIBXML2_VERSION: 2.10.3, LIBXSLT_VERSION: 1.1.37, MACOSX_DEPLOYMENT_TARGET: 10.15 } steps: - uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index 1e0a9119a..ffe167adc 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ CYTHON_WITH_COVERAGE?=$(shell $(PYTHON) -c 'import Cython.Coverage; import sys; CYTHON3_WITH_COVERAGE?=$(shell $(PYTHON3) -c 'import Cython.Coverage; import sys; assert not hasattr(sys, "pypy_version_info")' >/dev/null 2>/dev/null && echo " --coverage" || true) PYTHON_BUILD_VERSION ?= * -MANYLINUX_LIBXML2_VERSION=2.9.14 -MANYLINUX_LIBXSLT_VERSION=1.1.35 +MANYLINUX_LIBXML2_VERSION=2.10.3 +MANYLINUX_LIBXSLT_VERSION=1.1.37 MANYLINUX_CFLAGS=-O3 -g1 -pipe -fPIC -flto MANYLINUX_LDFLAGS=-flto From 1166cfbfa01f808f9f4c07cd6c834b8d61a1e97b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 23 Dec 2022 09:12:19 +0100 Subject: [PATCH 0055/1154] Remove CI image that is not available. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa28205fa..11a2dfcb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,9 @@ jobs: env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: - - os: windows-2016 - python-version: 2.7 - env: { STATIC_DEPS: true } # always static + #- os: windows-2016 + # python-version: 2.7 + # env: { STATIC_DEPS: true } # always static - os: ubuntu-latest python-version: "3.9" From 3e0eea07932963b43ced494ea85830e472f73cc3 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 23 Dec 2022 09:13:18 +0100 Subject: [PATCH 0056/1154] Add latest Python 3.12 dev version to CI matrix. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11a2dfcb3..d49f0ae8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - "3.9" - "3.10" # quotes to avoid being interpreted as the number 3.1 - "3.11" - # - "3.12-dev" + - "3.12-dev" env: [{ STATIC_DEPS: true }, { STATIC_DEPS: false }] include: From 6ae1f5d5969f7b8aee0b7ce9a0436f6e235cd52a Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 23 Dec 2022 09:21:51 +0100 Subject: [PATCH 0057/1154] Add CI caching for the library download files to avoid excessive remote access. --- .github/workflows/ci.yml | 10 ++++++++++ .github/workflows/wheels.yml | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d49f0ae8b..605cf7c34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,16 @@ jobs: path: ~/.ccache key: ${{ runner.os }}-ccache${{ matrix.extra_hash }}-${{ matrix.python-version }}-${{ hashFiles('.github/workflows/ci.yml', 'tools/ci-run.sh') }} + - name: Cache [libs] + uses: actions/cache@v3 + if: ${{ matrix.env.STATIC_DEPS == 'true' }} + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: libs-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }}${{ matrix.extra_hash }} + - name: Run CI continue-on-error: ${{ matrix.allowed_failure || false }} env: ${{ matrix.env }} diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 66fca6067..4963a1489 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -110,6 +110,15 @@ jobs: - name: Install dependencies run: python -m pip install -r requirements.txt + - name: Cache [libs] + uses: actions/cache@v3 + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: linux-libs + - name: Build Linux wheels run: make sdist wheel_${{ matrix.image }} env: { STATIC_DEPS: true, PYTHON_BUILD_VERSION: "${{ matrix.pyversion }}" } @@ -166,6 +175,16 @@ jobs: - name: Install dependencies run: python -m pip install setuptools wheel -r requirements.txt + - name: Cache [libs] + uses: actions/cache@v3 + if: startsWith(runner.os, 'macos-') + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: non-linux-libs + - name: Build wheels run: make sdist wheel env: { STATIC_DEPS: true, RUN_TESTS: true } From a03a4b3c6b906d33c5ef1a15f3d5ca5fff600c76 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 23 Dec 2022 10:46:32 +0100 Subject: [PATCH 0058/1154] Use "noexcept" modifiers for callback functions and fix some exception value declaration bugs found by Cython 3.0. --- src/lxml/extensions.pxi | 6 +++--- src/lxml/includes/tree.pxd | 10 +++++----- src/lxml/includes/xmlerror.pxd | 4 ++-- src/lxml/includes/xmlparser.pxd | 24 ++++++++++++------------ src/lxml/parser.pxi | 10 +++++----- src/lxml/saxparser.pxi | 26 +++++++++++++------------- src/lxml/serializer.pxi | 4 ++-- src/lxml/xmlerror.pxi | 14 +++++++------- src/lxml/xpath.pxi | 2 +- 9 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/lxml/extensions.pxi b/src/lxml/extensions.pxi index 35a321b7a..80e53d7b5 100644 --- a/src/lxml/extensions.pxi +++ b/src/lxml/extensions.pxi @@ -393,7 +393,7 @@ cdef tuple LIBXML2_XPATH_ERROR_MESSAGES = ( b"?? Unknown error ??\n", ) -cdef void _forwardXPathError(void* c_ctxt, xmlerror.xmlError* c_error) with gil: +cdef void _forwardXPathError(void* c_ctxt, xmlerror.xmlError* c_error) noexcept with gil: cdef xmlerror.xmlError error cdef int xpath_code if c_error.message is not NULL: @@ -414,7 +414,7 @@ cdef void _forwardXPathError(void* c_ctxt, xmlerror.xmlError* c_error) with gil: (<_BaseContext>c_ctxt)._error_log._receive(&error) -cdef void _receiveXPathError(void* c_context, xmlerror.xmlError* error) nogil: +cdef void _receiveXPathError(void* c_context, xmlerror.xmlError* error) noexcept nogil: if not __DEBUG: return if c_context is NULL: @@ -851,7 +851,7 @@ cdef void _extension_function_call(_BaseContext context, function, # lookup the function by name and call it cdef void _xpath_function_call(xpath.xmlXPathParserContext* ctxt, - int nargs) with gil: + int nargs) noexcept with gil: cdef _BaseContext context cdef xpath.xmlXPathContext* rctxt = ctxt.context context = <_BaseContext> rctxt.userData diff --git a/src/lxml/includes/tree.pxd b/src/lxml/includes/tree.pxd index 03d558a33..312537cbd 100644 --- a/src/lxml/includes/tree.pxd +++ b/src/lxml/includes/tree.pxd @@ -65,7 +65,7 @@ cdef extern from "libxml/chvalid.h" nogil: cdef extern from "libxml/hash.h": ctypedef struct xmlHashTable - ctypedef void (*xmlHashScanner)(void* payload, void* data, const_xmlChar* name) # may require GIL! + ctypedef void (*xmlHashScanner)(void* payload, void* data, const_xmlChar* name) noexcept # may require GIL! void xmlHashScan(xmlHashTable* table, xmlHashScanner f, void* data) nogil void* xmlHashLookup(xmlHashTable* table, const_xmlChar* name) nogil ctypedef void (*xmlHashDeallocator)(void *payload, xmlChar *name) @@ -411,12 +411,12 @@ cdef extern from "libxml/xmlIO.h": cdef int xmlOutputBufferClose(xmlOutputBuffer* out) nogil ctypedef int (*xmlInputReadCallback)(void* context, - char* buffer, int len) nogil - ctypedef int (*xmlInputCloseCallback)(void* context) nogil + char* buffer, int len) except -1 nogil + ctypedef int (*xmlInputCloseCallback)(void* context) except -1 nogil ctypedef int (*xmlOutputWriteCallback)(void* context, - char* buffer, int len) - ctypedef int (*xmlOutputCloseCallback)(void* context) + char* buffer, int len) except -1 + ctypedef int (*xmlOutputCloseCallback)(void* context) except -1 cdef xmlOutputBuffer* xmlAllocOutputBuffer( xmlCharEncodingHandler* encoder) nogil diff --git a/src/lxml/includes/xmlerror.pxd b/src/lxml/includes/xmlerror.pxd index c5ac6a0aa..6967378b7 100644 --- a/src/lxml/includes/xmlerror.pxd +++ b/src/lxml/includes/xmlerror.pxd @@ -838,9 +838,9 @@ cdef extern from "libxml/xmlerror.h" nogil: int int2 void* node - ctypedef void (*xmlGenericErrorFunc)(void* ctxt, char* msg, ...) + ctypedef void (*xmlGenericErrorFunc)(void* ctxt, char* msg, ...) noexcept ctypedef void (*xmlStructuredErrorFunc)(void* userData, - xmlError* error) + xmlError* error) noexcept cdef void xmlSetGenericErrorFunc( void* ctxt, xmlGenericErrorFunc func) diff --git a/src/lxml/includes/xmlparser.pxd b/src/lxml/includes/xmlparser.pxd index 9f3056248..c94212cee 100644 --- a/src/lxml/includes/xmlparser.pxd +++ b/src/lxml/includes/xmlparser.pxd @@ -15,37 +15,37 @@ cdef extern from "libxml/parser.h" nogil: const_xmlChar** namespaces, int nb_attributes, int nb_defaulted, - const_xmlChar** attributes) + const_xmlChar** attributes) noexcept ctypedef void (*endElementNsSAX2Func)(void* ctx, const_xmlChar* localname, const_xmlChar* prefix, - const_xmlChar* URI) + const_xmlChar* URI) noexcept - ctypedef void (*startElementSAXFunc)(void* ctx, const_xmlChar* name, const_xmlChar** atts) + ctypedef void (*startElementSAXFunc)(void* ctx, const_xmlChar* name, const_xmlChar** atts) noexcept - ctypedef void (*endElementSAXFunc)(void* ctx, const_xmlChar* name) + ctypedef void (*endElementSAXFunc)(void* ctx, const_xmlChar* name) noexcept - ctypedef void (*charactersSAXFunc)(void* ctx, const_xmlChar* ch, int len) + ctypedef void (*charactersSAXFunc)(void* ctx, const_xmlChar* ch, int len) noexcept - ctypedef void (*cdataBlockSAXFunc)(void* ctx, const_xmlChar* value, int len) + ctypedef void (*cdataBlockSAXFunc)(void* ctx, const_xmlChar* value, int len) noexcept - ctypedef void (*commentSAXFunc)(void* ctx, const_xmlChar* value) + ctypedef void (*commentSAXFunc)(void* ctx, const_xmlChar* value) noexcept ctypedef void (*processingInstructionSAXFunc)(void* ctx, const_xmlChar* target, - const_xmlChar* data) + const_xmlChar* data) noexcept ctypedef void (*internalSubsetSAXFunc)(void* ctx, const_xmlChar* name, const_xmlChar* externalID, - const_xmlChar* systemID) + const_xmlChar* systemID) noexcept - ctypedef void (*endDocumentSAXFunc)(void* ctx) + ctypedef void (*endDocumentSAXFunc)(void* ctx) noexcept - ctypedef void (*startDocumentSAXFunc)(void* ctx) + ctypedef void (*startDocumentSAXFunc)(void* ctx) noexcept - ctypedef void (*referenceSAXFunc)(void * ctx, const_xmlChar* name) + ctypedef void (*referenceSAXFunc)(void * ctx, const_xmlChar* name) noexcept cdef int XML_SAX2_MAGIC diff --git a/src/lxml/parser.pxi b/src/lxml/parser.pxi index f0c8c6b64..e1e9da9f0 100644 --- a/src/lxml/parser.pxi +++ b/src/lxml/parser.pxi @@ -402,10 +402,10 @@ cdef class _FileReaderContext: finally: return c_byte_count # swallow any exceptions -cdef int _readFilelikeParser(void* ctxt, char* c_buffer, int c_size) with gil: +cdef int _readFilelikeParser(void* ctxt, char* c_buffer, int c_size) except -1 with gil: return (<_FileReaderContext>ctxt).copyToBuffer(c_buffer, c_size) -cdef int _readFileParser(void* ctxt, char* c_buffer, int c_size) nogil: +cdef int _readFileParser(void* ctxt, char* c_buffer, int c_size) except -1 nogil: return stdio.fread(c_buffer, 1, c_size, ctxt) ############################################################ @@ -626,10 +626,10 @@ cdef _initParserContext(_ParserContext context, if c_ctxt is not NULL: context._initParserContext(c_ctxt) -cdef void _forwardParserError(xmlparser.xmlParserCtxt* _parser_context, xmlerror.xmlError* error) with gil: +cdef void _forwardParserError(xmlparser.xmlParserCtxt* _parser_context, xmlerror.xmlError* error) noexcept with gil: (<_ParserContext>_parser_context._private)._error_log._receive(error) -cdef void _receiveParserError(void* c_context, xmlerror.xmlError* error) nogil: +cdef void _receiveParserError(void* c_context, xmlerror.xmlError* error) noexcept nogil: if __DEBUG: if c_context is NULL or (c_context)._private is NULL: _forwardError(NULL, error) @@ -1207,7 +1207,7 @@ cdef class _BaseParser: context.cleanup() -cdef void _initSaxDocument(void* ctxt) with gil: +cdef void _initSaxDocument(void* ctxt) noexcept with gil: xmlparser.xmlSAX2StartDocument(ctxt) c_ctxt = ctxt c_doc = c_ctxt.myDoc diff --git a/src/lxml/saxparser.pxi b/src/lxml/saxparser.pxi index 49e72beaf..1737f0801 100644 --- a/src/lxml/saxparser.pxi +++ b/src/lxml/saxparser.pxi @@ -294,7 +294,7 @@ cdef void _handleSaxStart( const_xmlChar* c_namespace, int c_nb_namespaces, const_xmlChar** c_namespaces, int c_nb_attributes, int c_nb_defaulted, - const_xmlChar** c_attributes) with gil: + const_xmlChar** c_attributes) noexcept with gil: cdef int i cdef size_t c_len c_ctxt = ctxt @@ -336,7 +336,7 @@ cdef void _handleSaxTargetStart( const_xmlChar* c_namespace, int c_nb_namespaces, const_xmlChar** c_namespaces, int c_nb_attributes, int c_nb_defaulted, - const_xmlChar** c_attributes) with gil: + const_xmlChar** c_attributes) noexcept with gil: cdef int i cdef size_t c_len c_ctxt = ctxt @@ -407,7 +407,7 @@ cdef void _handleSaxTargetStart( cdef void _handleSaxStartNoNs(void* ctxt, const_xmlChar* c_name, - const_xmlChar** c_attributes) with gil: + const_xmlChar** c_attributes) noexcept with gil: c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: return @@ -426,7 +426,7 @@ cdef void _handleSaxStartNoNs(void* ctxt, const_xmlChar* c_name, cdef void _handleSaxTargetStartNoNs(void* ctxt, const_xmlChar* c_name, - const_xmlChar** c_attributes) with gil: + const_xmlChar** c_attributes) noexcept with gil: c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: return @@ -483,7 +483,7 @@ cdef int _pushSaxStartEvent(_SaxParserContext context, cdef void _handleSaxEnd(void* ctxt, const_xmlChar* c_localname, const_xmlChar* c_prefix, - const_xmlChar* c_namespace) with gil: + const_xmlChar* c_namespace) noexcept with gil: c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: return @@ -506,7 +506,7 @@ cdef void _handleSaxEnd(void* ctxt, const_xmlChar* c_localname, return # swallow any further exceptions -cdef void _handleSaxEndNoNs(void* ctxt, const_xmlChar* c_name) with gil: +cdef void _handleSaxEndNoNs(void* ctxt, const_xmlChar* c_name) noexcept with gil: c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: return @@ -558,7 +558,7 @@ cdef int _pushSaxEndEvent(_SaxParserContext context, return 0 -cdef void _handleSaxData(void* ctxt, const_xmlChar* c_data, int data_len) with gil: +cdef void _handleSaxData(void* ctxt, const_xmlChar* c_data, int data_len) noexcept with gil: # can only be called if parsing with a target c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: @@ -575,7 +575,7 @@ cdef void _handleSaxData(void* ctxt, const_xmlChar* c_data, int data_len) with g cdef void _handleSaxTargetDoctype(void* ctxt, const_xmlChar* c_name, const_xmlChar* c_public, - const_xmlChar* c_system) with gil: + const_xmlChar* c_system) noexcept with gil: # can only be called if parsing with a target c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: @@ -592,7 +592,7 @@ cdef void _handleSaxTargetDoctype(void* ctxt, const_xmlChar* c_name, return # swallow any further exceptions -cdef void _handleSaxStartDocument(void* ctxt) with gil: +cdef void _handleSaxStartDocument(void* ctxt) noexcept with gil: c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: return @@ -608,7 +608,7 @@ cdef void _handleSaxStartDocument(void* ctxt) with gil: cdef void _handleSaxTargetPI(void* ctxt, const_xmlChar* c_target, - const_xmlChar* c_data) with gil: + const_xmlChar* c_data) noexcept with gil: # can only be called if parsing with a target c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: @@ -627,7 +627,7 @@ cdef void _handleSaxTargetPI(void* ctxt, const_xmlChar* c_target, cdef void _handleSaxPIEvent(void* ctxt, const_xmlChar* target, - const_xmlChar* data) with gil: + const_xmlChar* data) noexcept with gil: # can only be called when collecting pi events c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: @@ -645,7 +645,7 @@ cdef void _handleSaxPIEvent(void* ctxt, const_xmlChar* target, return # swallow any further exceptions -cdef void _handleSaxTargetComment(void* ctxt, const_xmlChar* c_data) with gil: +cdef void _handleSaxTargetComment(void* ctxt, const_xmlChar* c_data) noexcept with gil: # can only be called if parsing with a target c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: @@ -661,7 +661,7 @@ cdef void _handleSaxTargetComment(void* ctxt, const_xmlChar* c_data) with gil: return # swallow any further exceptions -cdef void _handleSaxComment(void* ctxt, const_xmlChar* text) with gil: +cdef void _handleSaxComment(void* ctxt, const_xmlChar* text) noexcept with gil: # can only be called when collecting comment events c_ctxt = ctxt if c_ctxt._private is NULL or c_ctxt.disableSAX: diff --git a/src/lxml/serializer.pxi b/src/lxml/serializer.pxi index 79a02829e..e1c76e1ba 100644 --- a/src/lxml/serializer.pxi +++ b/src/lxml/serializer.pxi @@ -699,10 +699,10 @@ cdef class _FilelikeWriter: finally: return retval # and swallow any further exceptions -cdef int _writeFilelikeWriter(void* ctxt, char* c_buffer, int length): +cdef int _writeFilelikeWriter(void* ctxt, char* c_buffer, int length) except -1: return (<_FilelikeWriter>ctxt).write(c_buffer, length) -cdef int _closeFilelikeWriter(void* ctxt): +cdef int _closeFilelikeWriter(void* ctxt) except -1: return (<_FilelikeWriter>ctxt).close() cdef _tofilelike(f, _Element element, encoding, doctype, method, diff --git a/src/lxml/xmlerror.pxi b/src/lxml/xmlerror.pxi index 1b50444fb..793e1d923 100644 --- a/src/lxml/xmlerror.pxi +++ b/src/lxml/xmlerror.pxi @@ -634,7 +634,7 @@ def use_global_python_log(PyErrorLog log not None): # local log functions: forward error to logger object -cdef void _forwardError(void* c_log_handler, xmlerror.xmlError* error) with gil: +cdef void _forwardError(void* c_log_handler, xmlerror.xmlError* error) noexcept with gil: cdef _BaseErrorLog log_handler if c_log_handler is not NULL: log_handler = <_BaseErrorLog>c_log_handler @@ -645,27 +645,27 @@ cdef void _forwardError(void* c_log_handler, xmlerror.xmlError* error) with gil: log_handler._receive(error) -cdef void _receiveError(void* c_log_handler, xmlerror.xmlError* error) nogil: +cdef void _receiveError(void* c_log_handler, xmlerror.xmlError* error) noexcept nogil: # no Python objects here, may be called without thread context ! if __DEBUG: _forwardError(c_log_handler, error) -cdef void _receiveXSLTError(void* c_log_handler, char* msg, ...) nogil: +cdef void _receiveXSLTError(void* c_log_handler, char* msg, ...) noexcept nogil: # no Python objects here, may be called without thread context ! cdef cvarargs.va_list args cvarargs.va_start(args, msg) _receiveGenericError(c_log_handler, xmlerror.XML_FROM_XSLT, msg, args) cvarargs.va_end(args) -cdef void _receiveRelaxNGParseError(void* c_log_handler, char* msg, ...) nogil: +cdef void _receiveRelaxNGParseError(void* c_log_handler, char* msg, ...) noexcept nogil: # no Python objects here, may be called without thread context ! cdef cvarargs.va_list args cvarargs.va_start(args, msg) _receiveGenericError(c_log_handler, xmlerror.XML_FROM_RELAXNGP, msg, args) cvarargs.va_end(args) -cdef void _receiveRelaxNGValidationError(void* c_log_handler, char* msg, ...) nogil: +cdef void _receiveRelaxNGValidationError(void* c_log_handler, char* msg, ...) noexcept nogil: # no Python objects here, may be called without thread context ! cdef cvarargs.va_list args cvarargs.va_start(args, msg) @@ -673,7 +673,7 @@ cdef void _receiveRelaxNGValidationError(void* c_log_handler, char* msg, ...) no cvarargs.va_end(args) # dummy function: no log output at all -cdef void _nullGenericErrorFunc(void* ctxt, char* msg, ...) nogil: +cdef void _nullGenericErrorFunc(void* ctxt, char* msg, ...) noexcept nogil: pass @@ -694,7 +694,7 @@ cdef void _connectGenericErrorLog(log, int c_domain=-1): cdef void _receiveGenericError(void* c_log_handler, int c_domain, - char* msg, cvarargs.va_list args) nogil: + char* msg, cvarargs.va_list args) noexcept nogil: # no Python objects here, may be called without thread context ! cdef xmlerror.xmlError c_error cdef char* c_text diff --git a/src/lxml/xpath.pxi b/src/lxml/xpath.pxi index a7cae4bff..704338e89 100644 --- a/src/lxml/xpath.pxi +++ b/src/lxml/xpath.pxi @@ -99,7 +99,7 @@ cdef class _XPathContext(_BaseContext): cdef void _registerExsltFunctionsForNamespaces( - void* _c_href, void* _ctxt, const_xmlChar* c_prefix): + void* _c_href, void* _ctxt, const_xmlChar* c_prefix) noexcept: c_href = _c_href ctxt = _ctxt From 170e9ea86ec9b37c922a65956e13bb216f6b9682 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 23 Dec 2022 11:04:23 +0100 Subject: [PATCH 0059/1154] Make CI failures show up also on macOS. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 605cf7c34..7884bca08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,8 +106,8 @@ jobs: # MacOS sub-jobs # ============== - - os: macos-latest - allowed_failure: true # Unicode parsing fails in Py3 + #- os: macos-latest + # allowed_failure: true # Unicode parsing fails in Py3 exclude: - os: ubuntu-latest From 3b7e43d0ef95c0c14501bdb357eb56da92251864 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 27 Dec 2022 09:45:35 +0100 Subject: [PATCH 0060/1154] Allow 3.12 CI jobs to fail since they run into a refcounting test failure that needs further investigation. --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7884bca08..65d873c39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,9 @@ jobs: # python-version: 2.7 # env: { STATIC_DEPS: true } # always static + - python-version: "3.12-dev" + allowed_failure: true + - os: ubuntu-latest python-version: "3.9" env: {STATIC_DEPS: true, WITH_REFNANNY: true} From 67d4856adeebf34ab58963c7ad82ec83a8fdda84 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 27 Dec 2022 09:59:38 +0100 Subject: [PATCH 0061/1154] Reduce the number of library releases that we look up in Windows builds. We do not need all of them, just the latest. --- buildlibxml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildlibxml.py b/buildlibxml.py index 15d6e3383..0c57936ff 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -32,7 +32,7 @@ # use pre-built libraries on Windows def download_and_extract_windows_binaries(destdir): - url = "https://api.github.com/repos/lxml/libxml2-win-binaries/releases" + url = "https://api.github.com/repos/lxml/libxml2-win-binaries/releases?per_page=5" releases, _ = read_url(url, accept="application/vnd.github+json", as_json=True) max_release = {'tag_name': ''} From 9e3ec4767f9c4766a0919c15ffe019c9d8b31499 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 27 Dec 2022 10:22:37 +0100 Subject: [PATCH 0062/1154] Pass GitHub token into API call when listing library releases. --- .github/workflows/ci.yml | 2 +- buildlibxml.py | 11 +++++++++-- tools/ci-run.sh | 6 +++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65d873c39..2d960a28b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: - name: Run CI continue-on-error: ${{ matrix.allowed_failure || false }} env: ${{ matrix.env }} - run: bash ./tools/ci-run.sh + run: GITHUB_API_TOKEN="${{ secrets.GITHUB_TOKEN }}" bash ./tools/ci-run.sh - name: Build docs if: contains( matrix.env.EXTRA_DEPS, 'sphinx') diff --git a/buildlibxml.py b/buildlibxml.py index 0c57936ff..d385217ee 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -33,7 +33,12 @@ def download_and_extract_windows_binaries(destdir): url = "https://api.github.com/repos/lxml/libxml2-win-binaries/releases?per_page=5" - releases, _ = read_url(url, accept="application/vnd.github+json", as_json=True) + releases, _ = read_url( + url, + accept="application/vnd.github+json", + as_json=True, + github_api_token=os.environ.get("GITHUB_API_TOKEN"), + ) max_release = {'tag_name': ''} for release in releases: @@ -169,10 +174,12 @@ def _list_dir_ftplib(url): return parse_text_ftplist("\n".join(data)) -def read_url(url, decode=True, accept=None, as_json=False): +def read_url(url, decode=True, accept=None, as_json=False, github_api_token=None): headers = {'User-Agent': 'https://github.com/lxml/lxml'} if accept: headers['Accept'] = accept + if github_api_token: + headers['authorization'] = "Bearer " + github_api_token request = Request(url, headers=headers) with closing(urlopen(request)) as res: diff --git a/tools/ci-run.sh b/tools/ci-run.sh index db3c7e879..893e73b22 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -5,6 +5,8 @@ set -x GCC_VERSION=${GCC_VERSION:=9} TEST_CFLAGS= EXTRA_CFLAGS= +SAVED_GITHUB_API_TOKEN="${GITHUB_API_TOKEN}" +unset GITHUB_API_TOKEN # remove from env # Set up compilers if [ -z "${OS_NAME##ubuntu*}" ]; then @@ -62,7 +64,9 @@ if [[ "$COVERAGE" == "true" ]]; then fi # Build -CFLAGS="$CFLAGS $EXTRA_CFLAGS" python -u setup.py build_ext --inplace \ +CFLAGS="$CFLAGS $EXTRA_CFLAGS" \ + GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + python -u setup.py build_ext --inplace \ $(if [ -n "${PYTHON_VERSION##2.*}" ]; then echo -n " -j7 "; fi ) \ $(if [[ "$COVERAGE" == "true" ]]; then echo -n " --with-coverage"; fi ) \ || exit 1 From 055bd2e261b247f54648068830939a23ccd04f42 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 2 Jan 2023 09:07:06 +0100 Subject: [PATCH 0063/1154] CI: Try to make token passing work in Windows. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d960a28b..0f03ddb1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: - name: Run CI continue-on-error: ${{ matrix.allowed_failure || false }} env: ${{ matrix.env }} - run: GITHUB_API_TOKEN="${{ secrets.GITHUB_TOKEN }}" bash ./tools/ci-run.sh + run: bash -c 'GITHUB_API_TOKEN="${{ secrets.GITHUB_TOKEN }}" bash ./tools/ci-run.sh' - name: Build docs if: contains( matrix.env.EXTRA_DEPS, 'sphinx') From ef48b57837fdb2b052b58624236c6174d4d40788 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 3 Jan 2023 08:02:21 +0100 Subject: [PATCH 0064/1154] CI: Use Github API token where necessary. --- tools/ci-run.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 893e73b22..a72aba67a 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -64,8 +64,8 @@ if [[ "$COVERAGE" == "true" ]]; then fi # Build -CFLAGS="$CFLAGS $EXTRA_CFLAGS" \ - GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ +GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + CFLAGS="$CFLAGS $EXTRA_CFLAGS" \ python -u setup.py build_ext --inplace \ $(if [ -n "${PYTHON_VERSION##2.*}" ]; then echo -n " -j7 "; fi ) \ $(if [[ "$COVERAGE" == "true" ]]; then echo -n " --with-coverage"; fi ) \ @@ -74,14 +74,19 @@ CFLAGS="$CFLAGS $EXTRA_CFLAGS" \ ccache -s || true # Run tests -CFLAGS="$TEST_CFLAGS" PYTHONUNBUFFERED=x make test || exit 1 +GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + CFLAGS="$TEST_CFLAGS" PYTHONUNBUFFERED=x \ + make test || exit 1 -python setup.py build || exit 1 -python setup.py install || exit 1 +GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + python setup.py build || exit 1 +GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + python setup.py install || exit 1 python -c "from lxml import etree" || exit 1 -CFLAGS="-O3 -g1 -mtune=generic -fPIC -flto" \ - LDFLAGS="-flto" \ - make clean wheel || exit 1 +GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + CFLAGS="-O3 -g1 -mtune=generic -fPIC -flto" \ + LDFLAGS="-flto" \ + make clean wheel || exit 1 ccache -s || true From 88cbcc4538416cca1eae4b8a92d902096cdb11bc Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 3 Jan 2023 08:05:17 +0100 Subject: [PATCH 0065/1154] Avoid using two different conditions for the same thing. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4963a1489..6d51b84fd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install MacOS dependencies - if: startsWith(matrix.os, 'mac') + if: startsWith(runner.os, 'macos-') run: | brew install automake libtool ln -s /usr/local/bin/glibtoolize /usr/local/bin/libtoolize From d5593c118856d3eba1eef181e3e0c666ded11f35 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 3 Jan 2023 08:12:59 +0100 Subject: [PATCH 0066/1154] Use MACOS_DEPLOYMENT_TARGET=11.0 instead of 10.15 to support arm64 builds. --- .github/workflows/ci.yml | 2 +- .github/workflows/wheels.yml | 2 +- buildlibxml.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f03ddb1d..959ab41ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: env: OS_NAME: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} - MACOSX_DEPLOYMENT_TARGET: 10.15 + MACOSX_DEPLOYMENT_TARGET: 11.0 LIBXML2_VERSION: 2.10.3 LIBXSLT_VERSION: 1.1.37 COVERAGE: false diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6d51b84fd..ae79122fc 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -156,7 +156,7 @@ jobs: python-version: 2.7 # needs older image runs-on: ${{ matrix.os }} - env: { LIBXML2_VERSION: 2.10.3, LIBXSLT_VERSION: 1.1.37, MACOSX_DEPLOYMENT_TARGET: 10.15 } + env: { LIBXML2_VERSION: 2.10.3, LIBXSLT_VERSION: 1.1.37, MACOSX_DEPLOYMENT_TARGET: 11.0 } steps: - uses: actions/checkout@v3 diff --git a/buildlibxml.py b/buildlibxml.py index d385217ee..b2d425b5f 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -427,7 +427,7 @@ def configure_darwin_env(env_setup): env_default = { 'CFLAGS': "-arch arm64 -O2", 'LDFLAGS': "-arch arm64", - 'MACOSX_DEPLOYMENT_TARGET': "10.6" + 'MACOSX_DEPLOYMENT_TARGET': "11.0" } else: env_default = { From d62806acd53046b506b86c6b7949cc8a989be19c Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 3 Jan 2023 08:20:45 +0100 Subject: [PATCH 0067/1154] CI: Try to use ccache on macos. --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 959ab41ed..bc90308b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,9 +155,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install MacOS dependencies + if: startsWith(runner.os, 'macos-') + run: | + brew install automake libtool ccache + ln -s /usr/local/bin/glibtoolize /usr/local/bin/libtoolize + - name: Cache [ccache] uses: pat-s/always-upload-cache@v3.0.11 - if: startsWith(runner.os, 'Linux') + if: startsWith(runner.os, 'Linux') || startsWith(runner.os, 'macos') with: path: ~/.ccache key: ${{ runner.os }}-ccache${{ matrix.extra_hash }}-${{ matrix.python-version }}-${{ hashFiles('.github/workflows/ci.yml', 'tools/ci-run.sh') }} From 6aa47a7ae23b3d622f19044c23181a197dcd209d Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 3 Jan 2023 08:26:58 +0100 Subject: [PATCH 0068/1154] Remove CI build+import tests that were previously not very useful and now fail on Py3.11+ due to distutils deprecation/changes. --- tools/ci-run.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index a72aba67a..8d79b5569 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -78,12 +78,6 @@ GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ CFLAGS="$TEST_CFLAGS" PYTHONUNBUFFERED=x \ make test || exit 1 -GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ - python setup.py build || exit 1 -GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ - python setup.py install || exit 1 -python -c "from lxml import etree" || exit 1 - GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ CFLAGS="-O3 -g1 -mtune=generic -fPIC -flto" \ LDFLAGS="-flto" \ From 4f7d52741149056dd4addde65d6257a80f8ade9e Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 3 Jan 2023 08:32:38 +0100 Subject: [PATCH 0069/1154] CI: Try to fix macOS build condition. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/wheels.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc90308b2..1745b73e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,14 +156,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install MacOS dependencies - if: startsWith(runner.os, 'macos-') + if: startsWith(runner.os, 'mac') run: | brew install automake libtool ccache ln -s /usr/local/bin/glibtoolize /usr/local/bin/libtoolize - name: Cache [ccache] uses: pat-s/always-upload-cache@v3.0.11 - if: startsWith(runner.os, 'Linux') || startsWith(runner.os, 'macos') + if: startsWith(runner.os, 'Linux') || startsWith(runner.os, 'mac') with: path: ~/.ccache key: ${{ runner.os }}-ccache${{ matrix.extra_hash }}-${{ matrix.python-version }}-${{ hashFiles('.github/workflows/ci.yml', 'tools/ci-run.sh') }} diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ae79122fc..29d31f9cf 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install MacOS dependencies - if: startsWith(runner.os, 'macos-') + if: startsWith(runner.os, 'mac') run: | brew install automake libtool ln -s /usr/local/bin/glibtoolize /usr/local/bin/libtoolize @@ -177,7 +177,7 @@ jobs: - name: Cache [libs] uses: actions/cache@v3 - if: startsWith(runner.os, 'macos-') + if: startsWith(runner.os, 'mac') with: path: | libs/*.xz From 3d4e60f2835e4d85fd357c182656d3eca534f2ff Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 3 Mar 2023 09:15:38 +0100 Subject: [PATCH 0070/1154] Prevent lxml.objectify from accepting non-decimal digit characters as integers/numbers and from representing them as IntElement. --- src/lxml/objectify.pyx | 2 +- src/lxml/tests/test_objectify.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lxml/objectify.pyx b/src/lxml/objectify.pyx index 376695a8b..791f2cf8c 100644 --- a/src/lxml/objectify.pyx +++ b/src/lxml/objectify.pyx @@ -976,7 +976,7 @@ cdef _checkNumber(bytes_unicode s, bint allow_float): cdef NumberParserState state = NPS_SPACE_PRE for c in s: - if c.isdigit() if (bytes_unicode is unicode) else c in b'0123456789': + if c in u'0123456789': if state in (NPS_DIGITS, NPS_FRACTION, NPS_DIGITS_EXP): pass elif state in (NPS_SPACE_PRE, NPS_SIGN): diff --git a/src/lxml/tests/test_objectify.py b/src/lxml/tests/test_objectify.py index f50a34474..a62df8522 100644 --- a/src/lxml/tests/test_objectify.py +++ b/src/lxml/tests/test_objectify.py @@ -2663,6 +2663,7 @@ def test_standard_lookup(self): t f + ²²²² 12_34 1.2_34 34E From 42cbc3c5c02307a9c0cc88de282e7f78a7251731 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 25 Mar 2023 11:47:34 +0100 Subject: [PATCH 0071/1154] Update project income report for 2022. --- README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index dea167ba3..39edf15f2 100644 --- a/README.rst +++ b/README.rst @@ -77,11 +77,16 @@ Project income report lxml has `more than 50 million downloads `_ per month on PyPI. -* Total project income in 2021: EUR 4890.37 (407.53 € / month) +* Total project income in 2022: EUR 2566.38 (213.87 € / month) + + - Tidelift: EUR 2539.38 + - Paypal: EUR 27.00 + +* Total project income in 2021: EUR 4640.37 (386.70 € / month) - Tidelift: EUR 4066.66 - Paypal: EUR 223.71 - - other: EUR 600.00 + - other: EUR 350.00 * Total project income in 2020: EUR 6065,86 (506.49 € / month) From 59c328ff388ef04643a9a3949923ce6f35d580b2 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 25 Mar 2023 11:52:48 +0100 Subject: [PATCH 0072/1154] Build universal x86_64 + Arm64 wheels on macOS by default. --- buildlibxml.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/buildlibxml.py b/buildlibxml.py index b2d425b5f..4d7295dee 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -420,21 +420,14 @@ def cmmi(configure_cmd, build_dir, multicore=None, **call_setup): def configure_darwin_env(env_setup): import platform - # configure target architectures on MacOS-X (x86_64 only, by default) + # configure target architectures on MacOS-X (x86_64 + Arm64, by default) major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) - if major_version > 7: - if platform.mac_ver()[2] == "arm64": - env_default = { - 'CFLAGS': "-arch arm64 -O2", - 'LDFLAGS': "-arch arm64", - 'MACOSX_DEPLOYMENT_TARGET': "11.0" - } - else: - env_default = { - 'CFLAGS': "-arch x86_64 -O2", - 'LDFLAGS': "-arch x86_64", - 'MACOSX_DEPLOYMENT_TARGET': "10.6" - } + if major_version >= 11: + env_default = { + 'CFLAGS': "-arch x86_64 -arch arm64 -O3", + 'LDFLAGS': "-arch x86_64 -arch arm64", + 'MACOSX_DEPLOYMENT_TARGET': "11.0" + } env_default.update(os.environ) env_setup['env'] = env_default From 4aa4d6cf26d9061a083dfe82ade8d1641237b73b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 25 Mar 2023 12:00:51 +0100 Subject: [PATCH 0073/1154] Update changelog. --- CHANGES.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index c684ad5e1..660407d9d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,22 @@ lxml changelog ============== +5.0.0 (2023-??-??) +================== + +Bugs fixed +---------- + +* ``lxml.objectify`` accepted non-decimal numbers like ``²²²`` as integers. + +* The internal exception handling in C callbacks was improved for Cython 3.0. + +Other changes +------------- + +* Linux/macOS wheels use the latest library versions libxml2 2.10.3 and libxslt 1.1.37. + + 4.9.2 (2022-12-13) ================== From 7a0e4a044d0bc7a1688e6cce93d2a0116346b1e1 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 25 Mar 2023 13:20:03 +0100 Subject: [PATCH 0074/1154] Update download count. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 39edf15f2..6adcfaafb 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ Another supporter of the lxml project is Project income report --------------------- -lxml has `more than 50 million downloads `_ +lxml has `about 60 million downloads `_ per month on PyPI. * Total project income in 2022: EUR 2566.38 (213.87 € / month) From e5a330c01cfcfb7bfc13fb0dd644a5bb761e1bfb Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 25 Mar 2023 21:24:46 +0100 Subject: [PATCH 0075/1154] Print download totals in pypistats. --- tools/pypistats.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/pypistats.py b/tools/pypistats.py index c5528970f..4ba7a5c3f 100644 --- a/tools/pypistats.py +++ b/tools/pypistats.py @@ -43,8 +43,7 @@ def system_sorter(name_and_count): return len(order) -def print_agg_stats(counts, sort_key=None): - stats = aggregate(counts) +def print_agg_stats(stats, sort_key=None): total = sum(stats.values()) max_len = max(len(category) for category in stats) agg_sum = 0.0 @@ -58,13 +57,19 @@ def main(): package_name = sys.argv[1] if len(sys.argv) > 1 else PACKAGE counts = get_stats("python_minor", package=package_name) + stats = aggregate(counts) print("Downloads by Python version:") - print_agg_stats(counts, sort_key=version_sorter) + print_agg_stats(stats, sort_key=version_sorter) print() counts = get_stats("system", package=package_name) + stats = aggregate(counts) print("Downloads by system:") - print_agg_stats(counts, sort_key=system_sorter) + print_agg_stats(stats, sort_key=system_sorter) + + total = sum(stats.values()) + days = {"month": 30, "week": 7, "day": 1} + print(f"Total downloads: {total * days['month']:-12,.1f}") if __name__ == '__main__': From 331f58a186bc52775c5e9a1ec54b93f9547f6b0a Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sun, 26 Mar 2023 12:26:27 +0200 Subject: [PATCH 0076/1154] Set master version to 5.0.0a0 to diverge from 4.9.x. --- src/lxml/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lxml/__init__.py b/src/lxml/__init__.py index f90fccc6a..06c17c1a7 100644 --- a/src/lxml/__init__.py +++ b/src/lxml/__init__.py @@ -1,6 +1,6 @@ # this is a package -__version__ = "4.9.2" +__version__ = "5.0.0a0" def get_include(): From 0919a045264f88a86f0aa8af289d1e68ad023c04 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sun, 26 Mar 2023 14:37:52 +0200 Subject: [PATCH 0077/1154] Exclude Py2.7 wheel on Windows since we don't have a windows-2016 build runner for it on GHA, it seems. --- .github/workflows/wheels.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 29d31f9cf..2f6b59a25 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -144,13 +144,12 @@ jobs: matrix: os: [macos-latest, windows-2019] #os: [macos-10.15, windows-latest] - #os: [macos-10.15, macOS-M1] #os: [macos-10.15] python-version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8-v7.3.7", "pypy-3.9-v7.3.9"] - include: - - os: windows-2016 - python-version: 2.7 + #include: + # - os: windows-2016 + # python-version: 2.7 exclude: - os: windows-2019 python-version: 2.7 # needs older image From da0c433cbb6e1ce74981484ff250d101a8078d51 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sun, 26 Mar 2023 14:42:53 +0200 Subject: [PATCH 0078/1154] Update Cython requirement to 0.29.33. --- CHANGES.txt | 4 +++- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 660407d9d..5b140d940 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -15,7 +15,9 @@ Bugs fixed Other changes ------------- -* Linux/macOS wheels use the latest library versions libxml2 2.10.3 and libxslt 1.1.37. +* Binary wheels use the latest library versions libxml2 2.10.3 and libxslt 1.1.37. + +* Built with Cython 0.29.33 to adapt to changes in Python 3.11 and 3.12. 4.9.2 (2022-12-13) diff --git a/requirements.txt b/requirements.txt index 988182be6..a5ca9089b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython>=0.29.7 +Cython>=0.29.33 From d35f44e2d3da16d756467e1b9dd4001cb1f60d44 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sun, 26 Mar 2023 16:12:24 +0200 Subject: [PATCH 0079/1154] Try to fix libiconv version parsing in Windows build ("1.15-1" instead of plain "1.14"). --- buildlibxml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildlibxml.py b/buildlibxml.py index 4d7295dee..d98334e20 100644 --- a/buildlibxml.py +++ b/buildlibxml.py @@ -315,7 +315,7 @@ def find_max_version(libname, filenames, version_re=None): match = version_re.search(fn) if match: version_string = match.group(1) - versions.append((tuple(map(tryint, version_string.split('.'))), + versions.append((tuple(map(tryint, version_string.replace("-", ".-").split('.'))), version_string)) if not versions: raise Exception( From dfdd910453e24f48dfe550060065cb7dbdd41728 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 27 Mar 2023 08:52:59 +0200 Subject: [PATCH 0080/1154] Add missing C macro define on Windows that is now required by libexslt 1.1.37. --- setupinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setupinfo.py b/setupinfo.py index 675891478..90b1de4ea 100644 --- a/setupinfo.py +++ b/setupinfo.py @@ -351,6 +351,7 @@ def define_macros(): if OPTION_BUILD_LIBXML2XSLT: macros.append(('LIBXML_STATIC', None)) macros.append(('LIBXSLT_STATIC', None)) + macros.append(('LIBEXSLT_STATIC', None)) # Disable showing C lines in tracebacks, unless explicitly requested. macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0')) return macros From 5dbc44b342479bb9eb551d542f73f09a734fe112 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 29 Mar 2023 10:53:34 +0200 Subject: [PATCH 0081/1154] Try to fix the macOS CI builds by always building universal-2 binaries. --- tools/ci-run.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 8d79b5569..afe521dc1 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -5,6 +5,7 @@ set -x GCC_VERSION=${GCC_VERSION:=9} TEST_CFLAGS= EXTRA_CFLAGS= +EXTRA_LDFLAGS= SAVED_GITHUB_API_TOKEN="${GITHUB_API_TOKEN}" unset GITHUB_API_TOKEN # remove from env @@ -22,12 +23,13 @@ if [ -z "${OS_NAME##ubuntu*}" ]; then export CC="gcc" export PATH="/usr/lib/ccache:$PATH" TEST_CFLAGS="-Og -g -fPIC" - EXTRA_CFLAGS="$TEST_CFLAGS -Wall -Wextra" + EXTRA_CFLAGS="-Wall -Wextra" elif [ -z "${OS_NAME##macos*}" ]; then export CC="clang -Wno-deprecated-declarations" - TEST_CFLAGS="-Og -g -fPIC" - EXTRA_CFLAGS="$TEST_CFLAGS -Wall -Wextra" + TEST_CFLAGS="-Og -g -fPIC -arch arm64 -arch x86_64" + EXTRA_LDFLAGS="-arch arm64 -arch x86_64" + EXTRA_CFLAGS="-Wall -Wextra -arch arm64 -arch x86_64" fi # Log versions in use @@ -60,12 +62,13 @@ else fi if [[ "$COVERAGE" == "true" ]]; then python -m pip install "coverage<5" || exit 1 - python -m pip install --pre 'Cython>=3.0a0' || exit 1 + python -m pip install --pre 'Cython>=3.0b2' || exit 1 fi # Build GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ - CFLAGS="$CFLAGS $EXTRA_CFLAGS" \ + CFLAGS="$CFLAGS $TEST_CFLAGS $EXTRA_CFLAGS" \ + LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" \ python -u setup.py build_ext --inplace \ $(if [ -n "${PYTHON_VERSION##2.*}" ]; then echo -n " -j7 "; fi ) \ $(if [[ "$COVERAGE" == "true" ]]; then echo -n " --with-coverage"; fi ) \ @@ -75,12 +78,14 @@ ccache -s || true # Run tests GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ - CFLAGS="$TEST_CFLAGS" PYTHONUNBUFFERED=x \ + CFLAGS="$TEST_CFLAGS $EXTRA_CFLAGS" \ + LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" \ + PYTHONUNBUFFERED=x \ make test || exit 1 GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ - CFLAGS="-O3 -g1 -mtune=generic -fPIC -flto" \ - LDFLAGS="-flto" \ + CFLAGS="$EXTRA_CFLAGS -O3 -g1 -mtune=generic -fPIC -flto" \ + LDFLAGS="-flto $EXTRA_LDFLAGS" \ make clean wheel || exit 1 ccache -s || true From 80d374b371ee2f3bfb64336548b7aad4acb6a66b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 29 Mar 2023 14:09:33 +0200 Subject: [PATCH 0082/1154] Exclude Py3.5 from CI builds - fails for too many platforms by now. --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1745b73e8..ac8be6642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,6 @@ jobs: os: [ubuntu-latest, macos-latest, windows-2019] python-version: - "2.7" - - "3.5" - "3.6" - "3.7" - "3.8" @@ -113,8 +112,6 @@ jobs: # allowed_failure: true # Unicode parsing fails in Py3 exclude: - - os: ubuntu-latest - python-version: "3.5" - os: ubuntu-latest python-version: "3.6" From 11b33a83ad689bd16bd0a98c14cda51a90572b73 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 29 Mar 2023 14:10:55 +0200 Subject: [PATCH 0083/1154] Update supported versions in package classifiers. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a9e4c20d4..756af767a 100644 --- a/setup.py +++ b/setup.py @@ -257,12 +257,12 @@ def build_packages(files): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: C', 'Operating System :: OS Independent', 'Topic :: Text Processing :: Markup :: HTML', From e119129957f5df90daeda4f5d426962b290d07a6 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 1 Apr 2023 18:16:55 +0200 Subject: [PATCH 0084/1154] Try to fix wheel upload in CI. --- .github/workflows/ci.yml | 2 +- tools/ci-run.sh | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac8be6642..a426416f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: - name: Upload Wheel uses: actions/upload-artifact@v3 - if: ${{ matrix.env.STATIC_DEPS == 'true' && env.COVERAGE == 'false' }} + if: ${{ matrix.env.STATIC_DEPS == 'true' }} with: name: wheels-${{ runner.os }} path: dist/*.whl diff --git a/tools/ci-run.sh b/tools/ci-run.sh index afe521dc1..ff256f9d5 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -77,15 +77,19 @@ GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ ccache -s || true # Run tests +echo "Running the tests ..." GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ CFLAGS="$TEST_CFLAGS $EXTRA_CFLAGS" \ LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" \ PYTHONUNBUFFERED=x \ make test || exit 1 -GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ - CFLAGS="$EXTRA_CFLAGS -O3 -g1 -mtune=generic -fPIC -flto" \ - LDFLAGS="-flto $EXTRA_LDFLAGS" \ - make clean wheel || exit 1 +if [[ "$COVERAGE" != "true" ]]; then + echo "Building a clean wheel ..." + GITHUB_API_TOKEN="${SAVED_GITHUB_API_TOKEN}" \ + CFLAGS="$EXTRA_CFLAGS -O3 -g1 -mtune=generic -fPIC -flto" \ + LDFLAGS="-flto $EXTRA_LDFLAGS" \ + make clean wheel || exit 1 +fi ccache -s || true From 438c4da8e418301e7a97bf63ff835db8b13ffbcf Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 1 Apr 2023 18:43:53 +0200 Subject: [PATCH 0085/1154] Split the library caches of the CI/wheel builds by OS since we have different downloads for Windows and others. --- .github/workflows/ci.yml | 2 +- .github/workflows/wheels.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a426416f0..ca3561b4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: libs/*.xz libs/*.gz libs/*.zip - key: libs-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }}${{ matrix.extra_hash }} + key: libs-${{ runner.os }}-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }}${{ matrix.extra_hash }} - name: Run CI continue-on-error: ${{ matrix.allowed_failure || false }} diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2f6b59a25..1d29ca9f4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -176,13 +176,12 @@ jobs: - name: Cache [libs] uses: actions/cache@v3 - if: startsWith(runner.os, 'mac') with: path: | libs/*.xz libs/*.gz libs/*.zip - key: non-linux-libs + key: libs-${{ runner.os }} - name: Build wheels run: make sdist wheel From aa790c4ff3e54ee9865ee2b1e3475a4f29b32499 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 20 Apr 2023 10:09:37 +0200 Subject: [PATCH 0086/1154] Require latest Cython release to support CPython 3.12a7. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5ca9089b..907611706 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython>=0.29.33 +Cython>=0.29.34 From 932dff3f59a000084ad89807c6e3d27ad0631fe6 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 20 Apr 2023 10:09:47 +0200 Subject: [PATCH 0087/1154] Update changelog. --- CHANGES.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5b140d940..99aad582f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,12 +12,14 @@ Bugs fixed * The internal exception handling in C callbacks was improved for Cython 3.0. +* A memory leak in ``lxml.html.clean`` was resolved by switching to Cython 0.29.34. + Other changes ------------- * Binary wheels use the latest library versions libxml2 2.10.3 and libxslt 1.1.37. -* Built with Cython 0.29.33 to adapt to changes in Python 3.11 and 3.12. +* Built with Cython 0.29.34 to adapt to changes in Python 3.11 and 3.12. 4.9.2 (2022-12-13) From e3ef31814f822f86e831d6c66020b3421462fc32 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 10 May 2023 15:49:00 +0200 Subject: [PATCH 0088/1154] pypistats: Reverse category output to make possible cut-offs visible. --- tools/pypistats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pypistats.py b/tools/pypistats.py index 4ba7a5c3f..31cbc9e0b 100644 --- a/tools/pypistats.py +++ b/tools/pypistats.py @@ -47,7 +47,7 @@ def print_agg_stats(stats, sort_key=None): total = sum(stats.values()) max_len = max(len(category) for category in stats) agg_sum = 0.0 - for category, count in sorted(stats.items(), key=sort_key): + for category, count in sorted(stats.items(), key=sort_key, reverse=True): agg_sum += count print(f" {category:{max_len}}: {count:-12.1f} / day ({agg_sum / total * 100:-5.1f}%)") From 07db761f9f027d1814a43686cda6fca26e37a931 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 11 May 2023 10:29:02 +0200 Subject: [PATCH 0089/1154] Avoid using the deprecated "imp" module. Closes https://bugs.launchpad.net/lxml/+bug/2018137 --- src/lxml/html/tests/test_html5parser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lxml/html/tests/test_html5parser.py b/src/lxml/html/tests/test_html5parser.py index 56afe98b7..bcf7f1b12 100644 --- a/src/lxml/html/tests/test_html5parser.py +++ b/src/lxml/html/tests/test_html5parser.py @@ -1,5 +1,4 @@ import os -import imp try: from StringIO import StringIO except ImportError: # python 3 @@ -45,7 +44,10 @@ def find_module(self, fullname, path=None): return None def load_module(self, fullname): - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + fake_module = object() + fake_module.__qualname__ = fullname + fake_module.__name__ = fullname.rsplit('.', 1)[-1] + mod = sys.modules.setdefault(fullname, fake_module) mod.__file__, mod.__loader__, mod.__path__ = "", self, [] mod.__dict__.update(self.mocks[fullname]) return mod From c6b7e621e4696c02bf8f6ea423ffbbf2109748ab Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 11 May 2023 10:30:15 +0200 Subject: [PATCH 0090/1154] Avoid using the deprecated "imp" module. Closes https://bugs.launchpad.net/lxml/+bug/2018137 --- src/lxml/html/tests/test_html5parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lxml/html/tests/test_html5parser.py b/src/lxml/html/tests/test_html5parser.py index bcf7f1b12..57878a5e3 100644 --- a/src/lxml/html/tests/test_html5parser.py +++ b/src/lxml/html/tests/test_html5parser.py @@ -44,7 +44,8 @@ def find_module(self, fullname, path=None): return None def load_module(self, fullname): - fake_module = object() + class Cls: pass + fake_module = Cls() fake_module.__qualname__ = fullname fake_module.__name__ = fullname.rsplit('.', 1)[-1] mod = sys.modules.setdefault(fullname, fake_module) From bf03e4507c0d5154dfd1b7e00427db403a7421fa Mon Sep 17 00:00:00 2001 From: Pedro Nacht Date: Fri, 19 May 2023 16:41:13 -0300 Subject: [PATCH 0091/1154] Add token permissions to wheels.yml (GH-369) --- .github/workflows/wheels.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 1d29ca9f4..2dde59041 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,10 +4,15 @@ on: release: types: [created] +permissions: {} + jobs: sdist: runs-on: ubuntu-latest + permissions: + contents: write + steps: - uses: actions/checkout@v3 @@ -47,6 +52,9 @@ jobs: Linux: runs-on: ubuntu-latest + permissions: + contents: write + strategy: # Allows for matrix sub-jobs to fail without canceling the rest fail-fast: false @@ -155,6 +163,10 @@ jobs: python-version: 2.7 # needs older image runs-on: ${{ matrix.os }} + + permissions: + contents: write + env: { LIBXML2_VERSION: 2.10.3, LIBXSLT_VERSION: 1.1.37, MACOSX_DEPLOYMENT_TARGET: 11.0 } steps: From 53eb0f82641cbfc968f0fb8ca6d81767fcfe1a5b Mon Sep 17 00:00:00 2001 From: Michael Schlenker Date: Thu, 25 May 2023 08:47:53 +0200 Subject: [PATCH 0092/1154] Fix order of initialization for threading on Windows with libxml2 2.11.4 (GH-370) This fixes an Access Violation on Windows when using a shared build of libxml2 2.11.4. Stacktrace: ntdll.dll!RtlpWaitOnCriticalSection() + 0xa6 Bytes ntdll.dll!RtlpEnterCriticalSectionContended() + 0x1c4 Bytes ntdll.dll!RtlEnterCriticalSection() + 0x42 Bytes > libxml2.dll!xmlRMutexLock(_xmlRMutex * tok=0x0000000002fea170) Zeile 234 C libxml2.dll!xmlThrDefIndentTreeOutput(int v=0) Zeile 934 C etree.pyd!00000000037fc566() python27.dll!_PyImport_LoadDynamicModule(char * name=0x0000000002fe4408, char * pathname=0x000000005a49c660, _iobuf * fp=0x0000000002839880) Zeile 54 C libxml2 changed the mutex initialization in this commit: https://gitlab.gnome.org/GNOME/libxml2/-/commit/65d381f32c0d2cb2361055fddbd5c4a9119031d1 xmlThrDefIndentTreeOutput tries to use the global Mutex, but this is only initialized by xmlInitParser(), so it crashes due to wrong initialization order. --- src/lxml/etree.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lxml/etree.pyx b/src/lxml/etree.pyx index c0d236bd8..ea5e43025 100644 --- a/src/lxml/etree.pyx +++ b/src/lxml/etree.pyx @@ -142,15 +142,15 @@ cdef struct qname: const_xmlChar* c_name python.PyObject* href +# initialize parser (and threading) +xmlparser.xmlInitParser() + # global per-thread setup tree.xmlThrDefIndentTreeOutput(1) tree.xmlThrDefLineNumbersDefaultValue(1) _initThreadLogging() -# initialize parser (and threading) -xmlparser.xmlInitParser() - # filename encoding cdef bytes _FILENAME_ENCODING = (sys.getfilesystemencoding() or sys.getdefaultencoding() or 'ascii').encode("UTF-8") cdef char* _C_FILENAME_ENCODING = _cstr(_FILENAME_ENCODING) From a1f2231dfdd31a37f2c55bc9ae4ac33e92221227 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 12 May 2023 09:49:34 +0200 Subject: [PATCH 0093/1154] Adapt unicode parsing to Py3.12. --- src/lxml/includes/etree_defs.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/lxml/includes/etree_defs.h b/src/lxml/includes/etree_defs.h index e671fa85d..d42f2158c 100644 --- a/src/lxml/includes/etree_defs.h +++ b/src/lxml/includes/etree_defs.h @@ -120,6 +120,16 @@ static PyObject* PyBytes_FromFormat(const char* format, ...) { # define _lx_PySlice_GetIndicesEx(o, l, b, e, s, sl) PySlice_GetIndicesEx(((PySliceObject*)o), l, b, e, s, sl) #endif +#if PY_VERSION_HEX >= 0x030B00A1 +/* Python 3.12 doesn't have wstr Unicode strings any more. */ +#undef PyUnicode_GET_DATA_SIZE +#define PyUnicode_GET_DATA_SIZE(ustr) (0) +#undef PyUnicode_AS_DATA +#define PyUnicode_AS_DATA(ustr) (NULL) +#undef PyUnicode_IS_READY +#define PyUnicode_IS_READY(ustr) (1) +#endif + #ifdef WITHOUT_THREADING # undef PyEval_SaveThread # define PyEval_SaveThread() (NULL) From 59ccc8cdd7ca2de7cc832d1bd2089a0fe4189dbd Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 12 May 2023 09:56:05 +0200 Subject: [PATCH 0094/1154] Remove redundant and long deprecated methods. --- CHANGES.txt | 5 +++++ src/lxml/parser.pxi | 4 ---- src/lxml/tests/test_xslt.py | 2 +- src/lxml/xpath.pxi | 15 --------------- src/lxml/xslt.pxi | 6 ------ 5 files changed, 6 insertions(+), 26 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 99aad582f..396a610ac 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,11 @@ Bugs fixed Other changes ------------- +* Some redundant and long deprecated methods were removed: + ``parser.setElementClassLookup()``, + ``xslt_transform.apply()``, + ``xpath.evaluate()``. + * Binary wheels use the latest library versions libxml2 2.10.3 and libxslt 1.1.37. * Built with Cython 0.29.34 to adapt to changes in Python 3.11 and 3.12. diff --git a/src/lxml/parser.pxi b/src/lxml/parser.pxi index e1e9da9f0..003407eac 100644 --- a/src/lxml/parser.pxi +++ b/src/lxml/parser.pxi @@ -975,10 +975,6 @@ cdef class _BaseParser: """The version of the underlying XML parser.""" return u"libxml2 %d.%d.%d" % LIBXML_VERSION - def setElementClassLookup(self, ElementClassLookup lookup = None): - u":deprecated: use ``parser.set_element_class_lookup(lookup)`` instead." - self.set_element_class_lookup(lookup) - def set_element_class_lookup(self, ElementClassLookup lookup = None): u"""set_element_class_lookup(self, lookup = None) diff --git a/src/lxml/tests/test_xslt.py b/src/lxml/tests/test_xslt.py index 0ef076694..84c1983fd 100644 --- a/src/lxml/tests/test_xslt.py +++ b/src/lxml/tests/test_xslt.py @@ -434,7 +434,7 @@ def test_xslt_parameter_missing(self): st = etree.XSLT(style) # at least libxslt 1.1.28 produces this error, earlier ones (e.g. 1.1.18) might not ... - self.assertRaises(etree.XSLTApplyError, st.apply, tree) + self.assertRaises(etree.XSLTApplyError, st, tree) def test_xslt_multiple_parameters(self): tree = self.parse('BC') diff --git a/src/lxml/xpath.pxi b/src/lxml/xpath.pxi index 704338e89..d7454c33f 100644 --- a/src/lxml/xpath.pxi +++ b/src/lxml/xpath.pxi @@ -147,21 +147,6 @@ cdef class _XPathEvaluatorBase: self._xpathCtxt = xpathCtxt self._context.set_context(xpathCtxt) - def evaluate(self, _eval_arg, **_variables): - u"""evaluate(self, _eval_arg, **_variables) - - Evaluate an XPath expression. - - Instead of calling this method, you can also call the evaluator object - itself. - - Variables may be provided as keyword arguments. Note that namespaces - are currently not supported for variables. - - :deprecated: call the object, not its method. - """ - return self(_eval_arg, **_variables) - cdef bint _checkAbsolutePath(self, char* path): cdef char c if path is NULL: diff --git a/src/lxml/xslt.pxi b/src/lxml/xslt.pxi index 559b277bc..78b20ce62 100644 --- a/src/lxml/xslt.pxi +++ b/src/lxml/xslt.pxi @@ -469,12 +469,6 @@ cdef class XSLT: raise ValueError("cannot set a maximum stylesheet traversal depth < 0") xslt.xsltMaxDepth = max_depth - def apply(self, _input, *, profile_run=False, **kw): - u"""apply(self, _input, profile_run=False, **kw) - - :deprecated: call the object, not this method.""" - return self(_input, profile_run=profile_run, **kw) - def tostring(self, _ElementTree result_tree): u"""tostring(self, result_tree) From 6733d06cbe3754bcb2341f4668083ca72dd30b7a Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 25 May 2023 08:45:59 +0200 Subject: [PATCH 0095/1154] Increase CI job timeout to avoid failures on macos. --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca3561b4b..7e9e615ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,9 +123,8 @@ jobs: env: { STATIC_DEPS: false } # always static # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. - # From testing, the runs tend to take ~3 minutes, so a limit of 20 minutes should be enough. This can always be - # changed in the future if needed. - timeout-minutes: 20 + # From testing, the runs tend to take 3-8 minutes, so a limit of 30 minutes should be enough. + timeout-minutes: 30 runs-on: ${{ matrix.os }} env: From a8c8c42653bc505762a61d81017cb8efc61af0e2 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 25 May 2023 08:58:25 +0200 Subject: [PATCH 0096/1154] Use latest Cython 0.29.35. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 907611706..b3a8bdb97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython>=0.29.34 +Cython>=0.29.35 From ff94502aca934403237f805aa710b4d28d9e1090 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 25 May 2023 08:58:41 +0200 Subject: [PATCH 0097/1154] Update changelog. --- CHANGES.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 396a610ac..16766b2ee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,7 +12,10 @@ Bugs fixed * The internal exception handling in C callbacks was improved for Cython 3.0. -* A memory leak in ``lxml.html.clean`` was resolved by switching to Cython 0.29.34. +* A memory leak in ``lxml.html.clean`` was resolved by switching to Cython 0.29.34+. + +* GH#370: A crash with recent libxml2 2.11.x versions was resolved. + Patch by Michael Schlenker. Other changes ------------- @@ -24,7 +27,7 @@ Other changes * Binary wheels use the latest library versions libxml2 2.10.3 and libxslt 1.1.37. -* Built with Cython 0.29.34 to adapt to changes in Python 3.11 and 3.12. +* Built with Cython 0.29.35 to adapt to changes in Python 3.11 and 3.12. 4.9.2 (2022-12-13) From e51b4ee52c9b782e308d3a27bf8e88d2f2797b23 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 25 May 2023 10:12:24 +0200 Subject: [PATCH 0098/1154] Work around the removal of the "--install-option" option in pip 23. --- tools/ci-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci-run.sh b/tools/ci-run.sh index ff256f9d5..e975f2c7c 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -52,7 +52,7 @@ ccache -s || true echo "Installing requirements [python]" python -m pip install -U pip setuptools wheel if [ -z "${PYTHON_VERSION##*-dev}" ]; - then python -m pip install --install-option=--cython-compile-minimal https://github.com/cython/cython/archive/master.zip; + then CYTHON_COMPILE_MINIMAL=true python -m pip install https://github.com/cython/cython/archive/master.zip; else python -m pip install -r requirements.txt; fi if [ -z "${PYTHON_VERSION##2*}" ]; then From 0268b8eb3287655303869e7b4e617ff0734fdfc4 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Thu, 25 May 2023 11:17:17 +0200 Subject: [PATCH 0099/1154] Try latest libxml2/libxslt in CI. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e9e615ad..5a3924770 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,8 +131,8 @@ jobs: OS_NAME: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} MACOSX_DEPLOYMENT_TARGET: 11.0 - LIBXML2_VERSION: 2.10.3 - LIBXSLT_VERSION: 1.1.37 + LIBXML2_VERSION: 2.11.4 + LIBXSLT_VERSION: 1.1.38 COVERAGE: false GCC_VERSION: 9 USE_CCACHE: 1 From 94681908507547c9b62d64cfd803bfbb71d2a14a Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Fri, 16 Jun 2023 09:24:21 +0200 Subject: [PATCH 0100/1154] Make regexp string raw to correct its escape sequence usage (GH-371) Fixes: $ python3 -Wd setup.py setup.py:117: DeprecationWarning: invalid escape sequence \. ... --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 756af767a..06137d74a 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ def static_env_list(name, separator=None): def setup_extra_options(): is_interesting_package = re.compile('^(libxml|libxslt|libexslt)$').match - is_interesting_header = re.compile('^(zconf|zlib|.*charset)\.h$').match + is_interesting_header = re.compile(r'^(zconf|zlib|.*charset)\.h$').match def extract_files(directories, pattern='*'): def get_files(root, dir_path, files): From ea11793fa91a13d1cf60e0fd203878ba659ea0c5 Mon Sep 17 00:00:00 2001 From: Pedro Nacht Date: Fri, 16 Jun 2023 04:26:19 -0300 Subject: [PATCH 0101/1154] Add security policy document (GH-372) --- SECURITY.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ac9e8fbf3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security Policy + +If you have discovered a security vulnerability in this project, please report it +privately. **Do not disclose it as a public issue.** This gives us time to work with you +to fix the issue before public exposure, reducing the chance that the exploit will be +used before a patch is released. + +Please submit the report through the +[Launchpad bug-tracker](https://bugs.launchpad.net/lxml/+filebug) (you may need to +create an account and log in). Make sure to mark the "🔒 This bug is a security +vulnerability" checkbox before submitting the report. This ensures the bug can only be +seen by the security group. + +Please provide the following information in your report: + +- A description of the vulnerability and its impact +- How to reproduce the issue + +This project is maintained by a few maintainers on a reasonable-effort basis. As such, +we ask that you give us 90 days to work on a fix before public exposure. From 089d4366af8c6f56aa36fe888001965e1f03e921 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 16 Jun 2023 03:33:26 -0400 Subject: [PATCH 0102/1154] Don't parse hostname from netloc manually; rely on urlsplit's result (GH-348) This manual parsing of netloc can be fooled by use of a userinfo component. SplitResult already has a hostname property. New test `test_host_whitelist_sneaky_userinfo` fails on master. --- src/lxml/html/clean.py | 7 +++---- src/lxml/html/tests/test_clean.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/lxml/html/clean.py b/src/lxml/html/clean.py index e6b0543cd..fcd32a10d 100644 --- a/src/lxml/html/clean.py +++ b/src/lxml/html/clean.py @@ -490,11 +490,10 @@ def allow_embedded_url(self, el, url): """ if self.whitelist_tags is not None and el.tag not in self.whitelist_tags: return False - scheme, netloc, path, query, fragment = urlsplit(url) - netloc = netloc.lower().split(':', 1)[0] - if scheme not in ('http', 'https'): + parts = urlsplit(url) + if parts.scheme not in ('http', 'https'): return False - if netloc in self.host_whitelist: + if parts.hostname in self.host_whitelist: return True return False diff --git a/src/lxml/html/tests/test_clean.py b/src/lxml/html/tests/test_clean.py index 2c785f563..ada0969bc 100644 --- a/src/lxml/html/tests/test_clean.py +++ b/src/lxml/html/tests/test_clean.py @@ -271,6 +271,26 @@ def test_formaction_attribute_in_button_input(self): expected, cleaner.clean_html(html)) + def test_host_whitelist_valid(self): + # Frame with valid hostname in src is allowed. + html = '
' + cleaner = Cleaner(frames=False, host_whitelist=["example.com"]) + self.assertEqual(expected, cleaner.clean_html(html)) + + def test_host_whitelist_invalid(self): + html = '
' + cleaner = Cleaner(frames=False, host_whitelist=["example.com"]) + self.assertEqual(expected, cleaner.clean_html(html)) + + def test_host_whitelist_invalid(self): + html = '
' - cleaner = Cleaner(frames=False, host_whitelist=["example.com"]) - self.assertEqual(expected, cleaner.clean_html(html)) - - def test_host_whitelist_invalid(self): - html = '
-...
-... Password: -...
-... spam spam SPAM! -... -... Text -... -... -... ''' - ->>> print(re.sub('[\x00-\x07\x0E]', '', doc)) - - - - - - - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - -
- Password: -
- spam spam SPAM! - - Text - - - - ->>> print(tostring(fromstring(doc)).decode("utf-8")) - - - - - - - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - -
- Password: -
- spam spam SPAM! - - Text - - - - ->>> print(Cleaner(page_structure=False, comments=False).clean_html(doc)) - - - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - spam spam SPAM! - - Text - - - - ->>> print(Cleaner(page_structure=False, safe_attrs_only=False).clean_html(doc)) - - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - spam spam SPAM! - - Text - - - - ->>> print(Cleaner(style=True, inline_style=True, links=True, add_nofollow=True, page_structure=False, safe_attrs_only=False).clean_html(doc)) - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - spam spam SPAM! - Author - Text - - - - ->>> print(Cleaner(style=True, inline_style=False, links=True, add_nofollow=True, page_structure=False, safe_attrs_only=False).clean_html(doc)) - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - spam spam SPAM! - Author - Text - - - - ->>> print(Cleaner(links=False, page_structure=False, javascript=True, host_whitelist=['example.com'], whitelist_tags=None).clean_html(doc)) - - - - - - - - - a link - a control char link - data - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - spam spam SPAM! - - Text - - - diff --git a/src/lxml/html/tests/test_clean_embed.txt b/src/lxml/html/tests/test_clean_embed.txt deleted file mode 100644 index 59a40551d..000000000 --- a/src/lxml/html/tests/test_clean_embed.txt +++ /dev/null @@ -1,39 +0,0 @@ -THIS FAILS IN libxml2 2.6.29 AND 2.6.30 !! - - ->>> from lxml.html import fromstring, tostring ->>> from lxml.html.clean import clean, clean_html, Cleaner ->>> from lxml.html import usedoctest - ->>> def tostring(el): # work-around for Py3 'bytes' type -... from lxml.html import tostring -... s = tostring(el) -... if not isinstance(s, str): -... s = s.decode('UTF-8') -... return s - ->>> doc_embed = '''
-... -... -... -... -...
''' ->>> print(tostring(fromstring(doc_embed))) -
- - - - -
->>> print(Cleaner().clean_html(doc_embed)) -
-
->>> print(Cleaner(host_whitelist=['www.youtube.com']).clean_html(doc_embed)) -
- -
->>> print(Cleaner(host_whitelist=['www.youtube.com'], whitelist_tags=None).clean_html(doc_embed)) -
- - -
diff --git a/tools/ci-run.sh b/tools/ci-run.sh index 86c3530d3..e3ff44340 100644 --- a/tools/ci-run.sh +++ b/tools/ci-run.sh @@ -78,7 +78,7 @@ fi if [ -z "${PYTHON_VERSION##2*}" ] || [ -z "${PYTHON_VERSION##pypy-2*}" ]; then python -m pip install -U beautifulsoup4==4.9.3 cssselect==1.1.0 html5lib==1.1 rnc2rng==2.6.5 ${EXTRA_DEPS} || exit 1 else - python -m pip install -U beautifulsoup4 cssselect html5lib rnc2rng ${EXTRA_DEPS} || exit 1 + python -m pip install -U beautifulsoup4 cssselect html5lib lxml_html_clean rnc2rng ${EXTRA_DEPS} || exit 1 fi if [[ "$COVERAGE" == "true" ]]; then python -m pip install "coverage<5" || exit 1 diff --git a/tox.ini b/tox.ini index 9c5a3a28f..1a2d68a09 100644 --- a/tox.ini +++ b/tox.ini @@ -18,3 +18,5 @@ install_command = pip install {opts} {packages} deps = -r{toxinidir}/requirements.txt html5lib + lxml_html_clean + setuptools;python_version >= '3.12' From a970538530cb97476acadcf746357b7ff6ecc303 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Fri, 29 Mar 2024 21:12:59 +0100 Subject: [PATCH 0453/1154] Remove Cleaner related documentation after removing the code. --- doc/lxmlhtml.txt | 184 ----------------------------------------------- 1 file changed, 184 deletions(-) diff --git a/doc/lxmlhtml.txt b/doc/lxmlhtml.txt index 8f32da6c1..5800c2bba 100644 --- a/doc/lxmlhtml.txt +++ b/doc/lxmlhtml.txt @@ -482,190 +482,6 @@ Example: >>> [a.attrib['href'] for a in result.xpath("//a[@target='_blank']")] ['http://tinyurl.com/2xae8s', 'http://preview.tinyurl.com/2xae8s'] -Cleaning up HTML -================ - -The module ``lxml.html.clean`` provides a ``Cleaner`` class for cleaning up -HTML pages. It supports removing embedded or script content, special tags, -CSS style annotations and much more. - -Note: the HTML Cleaner in ``lxml.html.clean`` is **not** considered -appropriate **for security sensitive environments**. -See e.g. `bleach `_ or -`nh3 `_ for alternatives. - -Note: owing to the increased number of security vulnerabilities that have been -reported concerning the blocklist-based nature of lxml.html.clean, it has been -determined that this specific component of the project will be extracted -and transitioned into a separate project. This strategic decision is aimed -at enhancing the suitability of the lxml library for deployment -in security-sensitive environments, thereby addressing and mitigating potential -risks more effectively. - -Say, you have an overburdened web page from a hideous source which contains -lots of content that upsets browsers and tries to run unnecessary code on the -client side: - -.. sourcecode:: pycon - - >>> html = '''\ - ... - ... - ... - ... - ... - ... - ... - ... - ... a link - ... another link - ...

a paragraph

- ...
secret EVIL!
- ... of EVIL! - ... - ...
- ... Password: - ...
- ... annoying EVIL! - ... spam spam SPAM! - ... - ... - ... ''' - -To remove the all superfluous content from this unparsed document, use the -``clean_html`` function: - -.. sourcecode:: pycon - - >>> from lxml.html.clean import clean_html - >>> print clean_html(html) -
- - a link - another link -

a paragraph

-
secret EVIL!
- of EVIL! - - - Password: - annoying EVIL!spam spam SPAM! -
- -The ``Cleaner`` class supports several keyword arguments to control exactly -which content is removed: - -.. sourcecode:: pycon - - >>> from lxml.html.clean import Cleaner - - >>> cleaner = Cleaner(page_structure=False, links=False) - >>> print cleaner.clean_html(html) - - - - - - - a link - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - annoying EVIL! - spam spam SPAM! - - - - - >>> cleaner = Cleaner(style=True, links=True, add_nofollow=True, - ... page_structure=False, safe_attrs_only=False) - - >>> print cleaner.clean_html(html) - - - - - a link - another link -

a paragraph

-
secret EVIL!
- of EVIL! - Password: - annoying EVIL! - spam spam SPAM! - - - - -You can also whitelist some otherwise dangerous content with -``Cleaner(host_whitelist=['www.youtube.com'])``, which would allow -embedded media from YouTube, while still filtering out embedded media -from other sites. - -See the docstring of ``Cleaner`` for the details of what can be -cleaned. - - -autolink --------- - -In addition to cleaning up malicious HTML, ``lxml.html.clean`` -contains functions to do other things to your HTML. This includes -autolinking:: - - autolink(doc, ...) - - autolink_html(html, ...) - -This finds anything that looks like a link (e.g., -``http://example.com``) in the *text* of an HTML document, and -turns it into an anchor. It avoids making bad links. - -Links in the elements ``