From 21ce91648b8e09dd9d5c60acab01e849196ad699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sun, 21 Aug 2022 00:18:29 +0200 Subject: [PATCH 01/69] allow building docs with python 3.8 (#227) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- doc/source/conf.py | 18 ++++++++++-------- doc/source/index.rst | 3 --- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 7ff2b1f8..01c2b895 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- -import sys +from __future__ import annotations + import urllib.request +import importlib.metadata import lxml from docutils.nodes import Text, reference @@ -11,11 +13,6 @@ from sphinx.environment import BuildEnvironment from sphinx.errors import ExtensionError -if sys.version_info >= (3, 8): - from importlib import metadata as importlib_metadata -else: - import importlib_metadata - extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] @@ -28,11 +25,10 @@ project = u'python-xmlsec' copyright = u'2020, Oleg Hoefling ' # noqa: A001 author = u'Bulat Gaifullin ' -release = importlib_metadata.version('xmlsec') +release = importlib.metadata.version('xmlsec') parsed: Version = parse(release) version = '{}.{}'.format(parsed.major, parsed.minor) -language = None exclude_patterns: list[str] = [] pygments_style = 'sphinx' todo_include_todos = False @@ -69,6 +65,12 @@ autodoc_member_order = 'groupwise' autodoc_docstring_signature = True + +rst_prolog = ''' +.. role:: xml(code) + :language: xml +''' + # LXML crossref'ing stuff: # LXML doesn't have an intersphinx docs, # so we link to lxml.etree._Element explicitly diff --git a/doc/source/index.rst b/doc/source/index.rst index 5cc758b9..e08f47d9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -3,9 +3,6 @@ You can adapt this file completely to your liking, but it should at least contain the root ``toctree`` directive. -.. role:: xml(code) - :language: xml - Welcome to python-xmlsec's documentation! ========================================= From 00759a33f6cbc2aa62536a1898d8107f88a49e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sun, 21 Aug 2022 00:20:05 +0200 Subject: [PATCH 02/69] add proper param types in register_callbacks (#228) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- src/main.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main.c b/src/main.c index 41f5e5fc..ffcae14f 100644 --- a/src/main.c +++ b/src/main.c @@ -275,18 +275,23 @@ static PyObject* PyXmlSec_PyIORegisterDefaultCallbacks(PyObject *self) { } static char PyXmlSec_PyIORegisterCallbacks__doc__[] = \ + "register_callbacks(input_match_callback, input_open_callback, input_read_callback, input_close_callback) -> None\n" "Register globally a custom set of IO callbacks with xmlsec.\n\n" - ":param callable input_match_callback: A callable that takes a filename `bytestring` and " + ":param input_match_callback: A callable that takes a filename `bytestring` and " "returns a boolean as to whether the other callbacks in this set can handle that name.\n" - ":param callable input_open_callback: A callable that takes a filename and returns some " + ":type input_match_callback: ~collections.abc.Callable[[bytes], bool]\n" + ":param input_open_callback: A callable that takes a filename and returns some " "context object (e.g. a file object) that the remaining callables in this set will be passed " "during handling.\n" + ":type input_open_callback: ~collections.abc.Callable[[bytes], Any]\n" // FIXME: How do we handle failures in ^^ (e.g. can't find the file)? - ":param callable input_read_callback: A callable that that takes the context object from the " + ":param input_read_callback: A callable that that takes the context object from the " "open callback and a buffer, and should fill the buffer with data (e.g. BytesIO.readinto()). " "xmlsec will call this function several times until there is no more data returned.\n" - ":param callable input_close_callback: A callable that takes the context object from the " + ":type input_read_callback: ~collections.abc.Callable[[Any, memoryview], int]\n" + ":param input_close_callback: A callable that takes the context object from the " "open callback and can do any resource cleanup necessary.\n" + ":type input_close_callback: ~collections.abc.Callable[[Any], None]\n" ; static PyObject* PyXmlSec_PyIORegisterCallbacks(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = { From 824084311638b7a6d07c0e6fa9414f14e84ee713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sun, 21 Aug 2022 09:10:09 +0200 Subject: [PATCH 03/69] adjust doc examples to #212 (#229) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- doc/source/examples/encrypt.py | 11 +++++++---- doc/source/examples/sign.py | 3 ++- doc/source/examples/verify.py | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/source/examples/encrypt.py b/doc/source/examples/encrypt.py index f69d4613..98f63b6f 100644 --- a/doc/source/examples/encrypt.py +++ b/doc/source/examples/encrypt.py @@ -2,10 +2,9 @@ import xmlsec -manager = xmlsec.KeysManager() -key = xmlsec.Key.from_file('rsacert.pem', xmlsec.constants.KeyDataFormatCertPem, None) -manager.add_key(key) -template = etree.parse('enc1-doc.xml').getroot() +with open('enc1-doc.xml') as fp: + template = etree.parse(fp).getroot() + enc_data = xmlsec.template.encrypted_data_create( template, xmlsec.constants.TransformAes128Cbc, @@ -20,6 +19,10 @@ data = template.find('./Data') # Encryption +manager = xmlsec.KeysManager() +key = xmlsec.Key.from_file('rsacert.pem', xmlsec.constants.KeyDataFormatCertPem, None) +manager.add_key(key) + enc_ctx = xmlsec.EncryptionContext(manager) enc_ctx.key = xmlsec.Key.generate( xmlsec.constants.KeyDataAes, 128, xmlsec.constants.KeyDataTypeSession diff --git a/doc/source/examples/sign.py b/doc/source/examples/sign.py index 4529bc8a..519c13a0 100644 --- a/doc/source/examples/sign.py +++ b/doc/source/examples/sign.py @@ -2,7 +2,8 @@ import xmlsec -template = etree.parse('sign1-tmpl.xml').getroot() +with open('sign1-tmpl.xml') as fp: + template = etree.parse(fp).getroot() signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) ctx = xmlsec.SignatureContext() diff --git a/doc/source/examples/verify.py b/doc/source/examples/verify.py index 8629c550..808a53c2 100644 --- a/doc/source/examples/verify.py +++ b/doc/source/examples/verify.py @@ -2,7 +2,9 @@ import xmlsec -template = etree.parse('sign1-res.xml').getroot() +with open('sign1-res.xml') as fp: + template = etree.parse(fp).getroot() + xmlsec.tree.add_ids(template, ["ID"]) signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) # Create a digital signature context (no key manager is needed). From c39eeff382ed119913463832f2f44d89b90dcc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Wed, 24 Aug 2022 20:26:22 +0200 Subject: [PATCH 04/69] Add integration with pre-commit.ci (#230) * add pre-commit.ci badge Signed-off-by: oleg.hoefling * fix pre-commit warnings Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- .pre-commit-config.yaml | 2 +- README.rst | 5 +-- doc/source/conf.py | 5 +-- doc/source/examples/sign_binary.py | 2 -- doc/source/examples/verify_binary.py | 2 -- doc/source/sphinx-pr-6916.diff | 46 ---------------------------- setup.cfg | 3 ++ tests/base.py | 21 +++---------- tests/conftest.py | 5 +-- tests/test_ds.py | 3 +- tests/test_enc.py | 9 +++--- tests/test_main.py | 4 +-- tests/test_xmlsec.py | 6 ++-- 13 files changed, 26 insertions(+), 87 deletions(-) delete mode 100644 doc/source/sphinx-pr-6916.diff diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3080b068..3c89b250 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,5 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: ".*.diff" # exclude patches repos: - repo: https://github.com/psf/black rev: 22.6.0 @@ -21,6 +20,7 @@ repos: - id: check-merge-conflict - id: check-json - id: detect-private-key + exclude: ^.*/rsakey.pem$ - id: mixed-line-ending - id: pretty-format-json args: [--autofix] diff --git a/README.rst b/README.rst index b2b3ab11..03312504 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,9 @@ python-xmlsec .. image:: https://img.shields.io/pypi/v/xmlsec.svg?logo=python&logoColor=white :target: https://pypi.python.org/pypi/xmlsec -.. image:: https://img.shields.io/travis/com/mehcode/python-xmlsec/master.svg?logo=travis&logoColor=white&label=Travis%20CI - :target: https://travis-ci.org/mehcode/python-xmlsec +.. image:: https://results.pre-commit.ci/badge/github/xmlsec/python-xmlsec/master.svg + :target: https://results.pre-commit.ci/latest/github/xmlsec/python-xmlsec/master + :alt: pre-commit.ci status .. image:: https://img.shields.io/appveyor/ci/hoefling/xmlsec/master.svg?logo=appveyor&logoColor=white&label=AppVeyor :target: https://ci.appveyor.com/project/hoefling/xmlsec .. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml/badge.svg diff --git a/doc/source/conf.py b/doc/source/conf.py index 01c2b895..35329050 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -import urllib.request import importlib.metadata +import urllib.request import lxml from docutils.nodes import Text, reference @@ -13,7 +11,6 @@ from sphinx.environment import BuildEnvironment from sphinx.errors import ExtensionError - extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] intersphinx_mapping = {'python': ('https://docs.python.org/3/', None)} diff --git a/doc/source/examples/sign_binary.py b/doc/source/examples/sign_binary.py index 4e6c0e00..275c6e40 100644 --- a/doc/source/examples/sign_binary.py +++ b/doc/source/examples/sign_binary.py @@ -1,5 +1,3 @@ -from lxml import etree - import xmlsec ctx = xmlsec.SignatureContext() diff --git a/doc/source/examples/verify_binary.py b/doc/source/examples/verify_binary.py index 06c2b727..1f888b99 100644 --- a/doc/source/examples/verify_binary.py +++ b/doc/source/examples/verify_binary.py @@ -1,5 +1,3 @@ -from lxml import etree - import xmlsec ctx = xmlsec.SignatureContext() diff --git a/doc/source/sphinx-pr-6916.diff b/doc/source/sphinx-pr-6916.diff deleted file mode 100644 index e7040a0f..00000000 --- a/doc/source/sphinx-pr-6916.diff +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py -index bc9bf49a74..4804c89c52 100644 ---- a/sphinx/environment/__init__.py -+++ b/sphinx/environment/__init__.py -@@ -46,6 +46,7 @@ - default_settings = { - 'embed_stylesheet': False, - 'cloak_email_addresses': True, -+ 'syntax_highlight': 'short', - 'pep_base_url': 'https://www.python.org/dev/peps/', - 'pep_references': None, - 'rfc_base_url': 'https://tools.ietf.org/html/', -diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py -index 85eeb43..80f1eea 100644 ---- a/sphinx/writers/html.py -+++ b/sphinx/writers/html.py -@@ -494,8 +494,11 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator): - self.body.append(self.starttag(node, 'kbd', '', - CLASS='docutils literal notranslate')) - else: -+ classes = 'docutils literal notranslate' -+ if 'code' in node['classes']: -+ classes += ' highlight' - self.body.append(self.starttag(node, 'code', '', -- CLASS='docutils literal notranslate')) -+ CLASS=classes)) - self.protect_literal_text += 1 - - def depart_literal(self, node: Element) -> None: -diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py -index 80cedd3..470f559 100644 ---- a/sphinx/writers/html5.py -+++ b/sphinx/writers/html5.py -@@ -446,8 +446,11 @@ class HTML5Translator(SphinxTranslator, BaseTranslator): - self.body.append(self.starttag(node, 'kbd', '', - CLASS='docutils literal notranslate')) - else: -+ classes = 'docutils literal notranslate' -+ if 'code' in node['classes']: -+ classes += ' highlight' - self.body.append(self.starttag(node, 'code', '', -- CLASS='docutils literal notranslate')) -+ CLASS=classes)) - self.protect_literal_text += 1 - - def depart_literal(self, node: Element) -> None: diff --git a/setup.cfg b/setup.cfg index 88d815b0..c090b4e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,5 +18,8 @@ upload_dir = doc/build/html [flake8] per-file-ignores = *.pyi: E301, E302, E305, E501, E701, F401, F822 + doc/source/conf.py: D1 + doc/source/examples/*.py: D1, E501 + tests/*.py: D1 exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* max-line-length = 130 diff --git a/tests/base.py b/tests/base.py index 06acf413..29605ce7 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,5 +1,6 @@ import gc import os +import resource import sys import unittest @@ -12,20 +13,8 @@ ns = {'dsig': xmlsec.constants.DSigNs, 'enc': xmlsec.constants.EncNs} -try: - import resource - - def get_memory_usage(): - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - -except ImportError: - resource = None - - def get_memory_usage(): - return 0 - - def get_iterations(): + """Parse iterations amount.""" if sys.platform in ('win32',): return 0 @@ -114,9 +103,9 @@ def assertXmlEqual(self, first, second, msg=None): # noqa: N802 for name in second.attrib.keys(): if name not in first.attrib: self.fail('x2 has an attribute x1 is missing: {}. {}'.format(name, msg)) - if not xml_text_compare(first.text, second.text): + if not _xml_text_compare(first.text, second.text): self.fail('text: {!r} != {!r}. {}'.format(first.text, second.text, msg)) - if not xml_text_compare(first.tail, second.tail): + if not _xml_text_compare(first.tail, second.tail): self.fail('tail: {!r} != {!r}. {}'.format(first.tail, second.tail, msg)) cl1 = sorted(first.getchildren(), key=lambda x: x.tag) cl2 = sorted(second.getchildren(), key=lambda x: x.tag) @@ -128,7 +117,7 @@ def assertXmlEqual(self, first, second, msg=None): # noqa: N802 self.assertXmlEqual(c1, c2) -def xml_text_compare(t1, t2): +def _xml_text_compare(t1, t2): if not t1 and not t2: return True if t1 == '*' or t2 == '*': diff --git a/tests/conftest.py b/tests/conftest.py index b51c4d45..675258c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ def pytest_collection_modifyitems(items): """ - Put the module init test first to implicitly check whether - any subsequent test fails because of module reinitialization. + Put the module init test first. + + This way, we implicitly check whether any subsequent test fails because of module reinitialization. """ def module_init_tests_first(item): diff --git a/tests/test_ds.py b/tests/test_ds.py index 9417fedb..694ad431 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -132,7 +132,6 @@ def test_sign_case3(self): def test_sign_case4(self): """Should sign a file using a dynamically created template, key from PEM and an X509 cert with custom ns.""" - root = self.load_xml("sign4-in.xml") xmlsec.tree.add_ids(root, ["ID"]) elem_id = root.get('ID', None) @@ -249,7 +248,7 @@ def test_verify_case_5(self): self.check_verify(5) def check_verify(self, i): - root = self.load_xml("sign%d-out.xml" % i) + root = self.load_xml("sign{}-out.xml".format(i)) xmlsec.tree.add_ids(root, ["ID"]) sign = xmlsec.tree.find_node(root, consts.NodeSignature) self.assertIsNotNone(sign) diff --git a/tests/test_enc.py b/tests/test_enc.py index 7add848c..63e00169 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -1,4 +1,3 @@ -import os import tempfile from lxml import etree @@ -125,7 +124,7 @@ def test_encrypt_binary(self): encrypted = ctx.encrypt_binary(enc_data, b'test') self.assertIsNotNone(encrypted) - self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) @@ -170,7 +169,7 @@ def test_encrypt_uri(self): encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name) self.assertIsNotNone(encrypted) - self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) @@ -219,7 +218,7 @@ def test_decrypt_key(self): self.assertEqual(self.load_xml("enc3-in.xml"), decrypted) def check_decrypt(self, i): - root = self.load_xml('enc%d-out.xml' % i) + root = self.load_xml('enc{}-out.xml'.format(i)) enc_data = xmlsec.tree.find_child(root, consts.NodeEncryptedData, consts.EncNs) self.assertIsNotNone(enc_data) @@ -228,7 +227,7 @@ def check_decrypt(self, i): ctx = xmlsec.EncryptionContext(manager) decrypted = ctx.decrypt(enc_data) self.assertIsNotNone(decrypted) - self.assertEqual(self.load_xml("enc%d-in.xml" % i), root) + self.assertEqual(self.load_xml("enc{}-in.xml".format(i)), root) def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() diff --git a/tests/test_main.py b/tests/test_main.py index 9fb71eaf..3db18582 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -112,7 +112,7 @@ def test_sign_data_not_first_callback(self): def match_cb(filename): nonlocal bad_match_calls bad_match_calls += 1 - False + return False for _ in range(2): self._register_mismatch_callbacks(match_cb) @@ -132,7 +132,7 @@ def test_failed_sign_because_default_callbacks(self): def mismatch_cb(filename): nonlocal mismatch_calls mismatch_calls += 1 - False + return False # NB: These first two sets of callbacks should never get called, # because the default callbacks always match beforehand: diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py index 32fac69a..303d7f8f 100644 --- a/tests/test_xmlsec.py +++ b/tests/test_xmlsec.py @@ -5,9 +5,9 @@ class TestModule(base.TestMemoryLeaks): def test_reinitialize_module(self): """ - This doesn't explicitly test anything, but will - be invoked first in the suite, so if the subsequent - tests don't fail, we know that the ``init()``/``shutdown()`` + This test doesn't explicitly verify anything, but will be invoked first in the suite. + + So if the subsequent tests don't fail, we know that the ``init()``/``shutdown()`` function pair doesn't break anything. """ xmlsec.shutdown() From 91592052b671d0ccb34a51ff84534cd684ea1473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Wed, 24 Aug 2022 20:53:17 +0200 Subject: [PATCH 05/69] fix test run on appveyor (#231) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- tests/base.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/base.py b/tests/base.py index 29605ce7..48aef81d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,6 +1,5 @@ import gc import os -import resource import sys import unittest @@ -8,26 +7,23 @@ import xmlsec -etype = type(etree.Element("test")) +etype = type(etree.Element('test')) ns = {'dsig': xmlsec.constants.DSigNs, 'enc': xmlsec.constants.EncNs} -def get_iterations(): - """Parse iterations amount.""" - if sys.platform in ('win32',): - return 0 +try: + import resource - try: - return int(os.getenv("PYXMLSEC_TEST_ITERATIONS", "10")) - except ValueError: - return 0 + test_iterations = int(os.environ.get('PYXMLSEC_TEST_ITERATIONS', '10')) +except (ImportError, ValueError): + test_iterations = 0 class TestMemoryLeaks(unittest.TestCase): maxDiff = None - iterations = get_iterations() + iterations = test_iterations data_dir = os.path.join(os.path.dirname(__file__), "data") From e83a4576f0a879acc5594e204bf6a485781d6427 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Sep 2022 11:09:21 +0200 Subject: [PATCH 06/69] [pre-commit.ci] pre-commit autoupdate (#233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 22.6.0 → 22.8.0](https://github.com/psf/black/compare/22.6.0...22.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c89b250..75e4b7ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.8.0 hooks: - id: black types: [] From c4658f37c9df3d9ecd90fa42878b5ae92afa79a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Thu, 29 Sep 2022 17:47:18 +0200 Subject: [PATCH 07/69] Correct codecov badge URL in readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 03312504..34bdd377 100644 --- a/README.rst +++ b/README.rst @@ -16,8 +16,8 @@ python-xmlsec :target: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml .. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml/badge.svg :target: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml -.. image:: https://codecov.io/gh/mehcode/python-xmlsec/branch/master/graph/badge.svg - :target: https://codecov.io/gh/mehcode/python-xmlsec +.. image:: https://codecov.io/gh/xmlsec/python-xmlsec/branch/master/graph/badge.svg + :target: https://codecov.io/gh/xmlsec/python-xmlsec .. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs :target: https://xmlsec.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status From cc8632f4f3181cdee6f325c47b2dde08117b5f55 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Oct 2022 08:06:32 +0200 Subject: [PATCH 08/69] [pre-commit.ci] pre-commit autoupdate (#238) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v0.971 → v0.981](https://github.com/pre-commit/mirrors-mypy/compare/v0.971...v0.981) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75e4b7ab..0545b12f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 + rev: v0.981 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From 92882809a0c380a8689ab0e7a995803decb2cc59 Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 13:58:25 +0100 Subject: [PATCH 09/69] Added changes to enable 3.11 builds --- .appveyor.yml | 4 ++++ .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 6 +++--- .github/workflows/opensuse-tumbleweed.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- .travis.yml | 3 +++ setup.py | 1 + 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index de17f8ba..b580525f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,10 @@ environment: python_version: 3.10.6 - python: 310-x64 python_version: 3.10.6 + - python: 311 + python_version: 3.11.2 + - python: 311-x64 + python_version: 3.10.6 install: - ps: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 4db5e306..24fa6ddd 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Setup Python diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 867c17ba..520e5ba6 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -5,16 +5,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2010_x86_64 - manylinux_2_24_x86_64 - musllinux_1_1_x86_64 exclude: - image: manylinux2010_x86_64 - python-abi: cp310-cp310 + python-abi: cp311-cp311 - image: manylinux2010_i686 - python-abi: cp310-cp310 + python-abi: cp311-cp311 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index d8bb8113..273f7f7f 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -6,7 +6,7 @@ jobs: container: opensuse/tumbleweed strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index fb13377a..d7959208 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.travis.yml b/.travis.yml index 9106805a..9e6ca540 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ matrix: - python: 3.9 dist: xenial sudo: required + - python: 3.11 + dist: xenial + sudo: required env: global: - CFLAGS=-coverage diff --git a/setup.py b/setup.py index 9a3c9277..5c7e0da5 100644 --- a/setup.py +++ b/setup.py @@ -533,6 +533,7 @@ def prepare_static_build_linux(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From d16f9853e9c85b97428d467b82fcf2fa5c766abf Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:00:32 +0100 Subject: [PATCH 10/69] Added changes to enable 3.11 builds --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b580525f..ce025819 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ environment: - python: 311 python_version: 3.11.2 - python: 311-x64 - python_version: 3.10.6 + python_version: 3.11.2 install: - ps: | From 519feff7d36389363122472d6a68aa285ed3405d Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:52:40 +0100 Subject: [PATCH 11/69] bumped isort to 5.11.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..6b8fdf67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.5 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 156394743a0c712e6638fe6e7e300c2f24b4fb12 Mon Sep 17 00:00:00 2001 From: tdivis Date: Fri, 24 Mar 2023 08:03:26 +0100 Subject: [PATCH 12/69] Fix #247 - Fix missing import of `template` in `__init__.pyi` stub. (#248) --- src/xmlsec/__init__.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index be0bd659..56356e55 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -5,6 +5,7 @@ from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element from xmlsec import constants as constants +from xmlsec import template as template from xmlsec import tree as tree from xmlsec.constants import __KeyData as KeyData from xmlsec.constants import __Transform as Transform From a7f95d55cd660d1a212fa76a527063b3b7dbe8bb Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 13:58:25 +0100 Subject: [PATCH 13/69] Added changes to enable 3.11 builds --- .appveyor.yml | 4 ++++ .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 6 +++--- .github/workflows/opensuse-tumbleweed.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- .travis.yml | 3 +++ setup.py | 1 + 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index de17f8ba..b580525f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,10 @@ environment: python_version: 3.10.6 - python: 310-x64 python_version: 3.10.6 + - python: 311 + python_version: 3.11.2 + - python: 311-x64 + python_version: 3.10.6 install: - ps: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 4db5e306..24fa6ddd 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Setup Python diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 867c17ba..520e5ba6 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -5,16 +5,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2010_x86_64 - manylinux_2_24_x86_64 - musllinux_1_1_x86_64 exclude: - image: manylinux2010_x86_64 - python-abi: cp310-cp310 + python-abi: cp311-cp311 - image: manylinux2010_i686 - python-abi: cp310-cp310 + python-abi: cp311-cp311 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index d8bb8113..273f7f7f 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -6,7 +6,7 @@ jobs: container: opensuse/tumbleweed strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index fb13377a..d7959208 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.travis.yml b/.travis.yml index 9106805a..9e6ca540 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ matrix: - python: 3.9 dist: xenial sudo: required + - python: 3.11 + dist: xenial + sudo: required env: global: - CFLAGS=-coverage diff --git a/setup.py b/setup.py index 9a3c9277..5c7e0da5 100644 --- a/setup.py +++ b/setup.py @@ -533,6 +533,7 @@ def prepare_static_build_linux(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From b7683774f747c7aed6a0b30e6045da679bc68760 Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:00:32 +0100 Subject: [PATCH 14/69] Added changes to enable 3.11 builds --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b580525f..ce025819 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ environment: - python: 311 python_version: 3.11.2 - python: 311-x64 - python_version: 3.10.6 + python_version: 3.11.2 install: - ps: | From bddf28e68a2509a287f9889aaeadc3adab80ccbc Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:52:40 +0100 Subject: [PATCH 15/69] bumped isort to 5.11.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..6b8fdf67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.5 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 2c58d43eedf72590e3e201252f5fc5ddae36f8c6 Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Wed, 22 Mar 2023 20:36:45 +0100 Subject: [PATCH 16/69] Fix #244 - Fix failing test with libxmlsec-1.2.36, also make libxmlsec version available from Python. --- src/main.c | 13 ++++ tests/data/sign5-out-xmlsec_1_2_36_to_37.xml | 67 ++++++++++++++++++++ tests/test_ds.py | 6 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/data/sign5-out-xmlsec_1_2_36_to_37.xml diff --git a/src/main.c b/src/main.c index ffcae14f..5773db3b 100644 --- a/src/main.c +++ b/src/main.c @@ -119,6 +119,13 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) { Py_RETURN_NONE; } +static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \ + "get_libxmlsec_version() -> tuple\n" + "Returns Version tuple of wrapped libxml library."; +static PyObject* PyXmlSec_GetLibXmlSecVersion() { + return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR); +} + static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ "enable_debug_trace(enabled) -> None\n" "Enables or disables calling LibXML2 callback from the default errors callback.\n\n" @@ -386,6 +393,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_PyShutdown__doc__ }, + { + "get_libxmlsec_version", + (PyCFunction)PyXmlSec_GetLibXmlSecVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlSecVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml new file mode 100644 index 00000000..f359b138 --- /dev/null +++ b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml @@ -0,0 +1,67 @@ + + + + + Hello, World! + + + + + + + + + + +HjY8ilZAIEM2tBbPn5mYO1ieIX4= + + +SIaj/6KY3C1SmDXU2++Gm31U1xTadFp04WhBgfsJFbxrL+q7GKSKN9kfQ+UpN9+i +D5fWmuavXEHe4Gw6RMaMEkq2URQo7F68+d5J/ajq8/l4n+xE6/reGScVwT6L4dEP +XXVJcAi2ZnQ3O7GTNvNGCPibL9mUcyCWBFZ92Uemtc/vJFCQ7ZyKMdMfACgxOwyN +T/9971oog241/2doudhonc0I/3mgPYWkZdX6yvr62mEjnG+oUZkhWYJ4ewZJ4hM4 +JjbFqZO+OEzDRSbw3DkmuBA/mtlx+3t13SESfEub5hqoMdVmtth/eTb64dsPdl9r +3k1ACVX9f8aHfQQdJOmLFQ== + + + + + + +Test Issuer +1 + +MIIE3zCCBEigAwIBAgIBBTANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tMB4X +DTAzMDMzMTA0MDIyMloXDTEzMDMyODA0MDIyMlowgb8xCzAJBgNVBAYTAlVTMRMw +EQYDVQQIEwpDYWxpZm9ybmlhMT0wOwYDVQQKEzRYTUwgU2VjdXJpdHkgTGlicmFy +eSAoaHR0cDovL3d3dy5hbGVrc2V5LmNvbS94bWxzZWMpMSEwHwYDVQQLExhFeGFt +cGxlcyBSU0EgQ2VydGlmaWNhdGUxFjAUBgNVBAMTDUFsZWtzZXkgU2FuaW4xITAf +BgkqhkiG9w0BCQEWEnhtbHNlY0BhbGVrc2V5LmNvbTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAJe4/rQ/gzV4FokE7CthjL/EXwCBSkXm2c3p4jyXO0Wt +quaNC3dxBwFPfPl94hmq3ZFZ9PHPPbp4RpYRnLZbRjlzVSOq954AXOXpSew7nD+E +mTqQrd9+ZIbGJnLOMQh5fhMVuOW/1lYCjWAhTCcYZPv7VXD2M70vVXDVXn6ZrqTg +qkVHE6gw1aCKncwg7OSOUclUxX8+Zi10v6N6+PPslFc5tKwAdWJhVLTQ4FKG+F53 +7FBDnNK6p4xiWryy/vPMYn4jYGvHUUk3eH4lFTCr+rSuJY8i/KNIf/IKim7g/o3w +Ae3GM8xrof2mgO8GjK/2QDqOQhQgYRIf4/wFsQXVZcMCAwEAAaOCAVcwggFTMAkG +A1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRp +ZmljYXRlMB0GA1UdDgQWBBQkhCzy1FkgYosuXIaQo6owuicanDCB+AYDVR0jBIHw +MIHtgBS0ue+a5pcOaGUemM76VQ2JBttMfKGB0aSBzjCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tggEA +MA0GCSqGSIb3DQEBBAUAA4GBALU/mzIxSv8vhDuomxFcplzwdlLZbvSQrfoNkMGY +1UoS3YJrN+jZLWKSyWE3mIaPpElqXiXQGGkwD5iPQ1iJMbI7BeLvx6ZxX/f+c8Wn +ss0uc1NxfahMaBoyG15IL4+beqO182fosaKJTrJNG3mc//ANGU9OsQM9mfBEt4oL +NJ2D + + + + + diff --git a/tests/test_ds.py b/tests/test_ds.py index 694ad431..38f0b25c 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -182,7 +182,11 @@ def test_sign_case5(self): self.assertEqual("rsakey.pem", ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign5-out.xml"), root) + if (1, 2, 36) <= xmlsec.get_libxmlsec_version() <= (1, 2, 37): + expected_xml_file = 'sign5-out-xmlsec_1_2_36_to_37.xml' + else: + expected_xml_file = 'sign5-out.xml' + self.assertEqual(self.load_xml(expected_xml_file), root) def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() From 1cf6785b3e06c2b8f6cac1266cf8f1934650bc4b Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Wed, 21 Dec 2022 14:01:14 +0100 Subject: [PATCH 17/69] Fix #164 - Add support for loading keys from engine (e.g. pkcs11). --- README.rst | 2 +- src/keys.c | 47 +++++++++++++++++++++++++++++++++++++++++ src/xmlsec/__init__.pyi | 2 ++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 34bdd377..e1924652 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Check the `examples `_ se Requirements ************ - ``libxml2 >= 2.9.1`` -- ``libxmlsec1 >= 1.2.18`` +- ``libxmlsec1 >= 1.2.33`` Install ******* diff --git a/src/keys.c b/src/keys.c index 1362b128..1440331c 100644 --- a/src/keys.c +++ b/src/keys.c @@ -185,6 +185,47 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* return NULL; } +static const char PyXmlSec_KeyFromEngine__doc__[] = \ + "from_engine(engine_and_key_id) -> xmlsec.Key\n" + "Loads PKI key from an engine.\n\n" + ":param engine_and_key_id: engine and key id, i.e. 'pkcs11;pkcs11:token=XmlsecToken;object=XmlsecKey;pin-value=password'\n" + ":type engine_and_key_id: :class:`str`, " + ":return: pointer to newly created key\n" + ":rtype: :class:`~xmlsec.Key`"; +static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject* kwargs) { + static char *kwlist[] = {"engine_and_key_id", NULL}; + + const char* engine_and_key_id = NULL; + PyXmlSec_Key* key = NULL; + + PYXMLSEC_DEBUG("load key from engine - start"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s:from_engine", kwlist, &engine_and_key_id)) { + goto ON_FAIL; + } + + if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; + + Py_BEGIN_ALLOW_THREADS; + key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), + (void*)engine_and_key_id); + Py_END_ALLOW_THREADS; + + if (key->handle == NULL) { + PyXmlSec_SetLastError("cannot read key"); + goto ON_FAIL; + } + + key->is_own = 1; + + PYXMLSEC_DEBUG("load key from engine - ok"); + return (PyObject*)key; + +ON_FAIL: + PYXMLSEC_DEBUG("load key from engine - fail"); + Py_XDECREF(key); + return NULL; +} + static const char PyXmlSec_KeyGenerate__doc__[] = \ "generate(klass, size, type) -> xmlsec.Key\n" "Generates key of kind ``klass`` with ``size`` and ``type``.\n\n" @@ -494,6 +535,12 @@ static PyMethodDef PyXmlSec_KeyMethods[] = { METH_CLASS|METH_VARARGS|METH_KEYWORDS, PyXmlSec_KeyFromFile__doc__ }, + { + "from_engine", + (PyCFunction)PyXmlSec_KeyFromEngine, + METH_CLASS|METH_VARARGS|METH_KEYWORDS, + PyXmlSec_KeyFromEngine__doc__ + }, { "generate", (PyCFunction)PyXmlSec_KeyGenerate, diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 56356e55..6c326f56 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -49,6 +49,8 @@ class Key: @classmethod def from_file(cls: type[Self], file: GenericPath[AnyStr] | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... @classmethod + def from_engine(cls: type[Self], engine_and_key_id: AnyStr) -> Self: ... + @classmethod def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ... @classmethod def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ... From 0b5939fc65e98cebb669d311b4fb58844bf887e5 Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Thu, 13 Apr 2023 14:28:02 +0200 Subject: [PATCH 18/69] Fix #164 - Add tests for pkcs11 (softhsm) key. --- .github/workflows/sdist.yml | 3 +- tests/softhsm_setup.py | 265 ++++++++++++++++++++++++++++++++++++ tests/test_pkcs11.py | 57 ++++++++ 3 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 tests/softhsm_setup.py create mode 100644 tests/test_pkcs11.py diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index d7959208..6ca7a657 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -16,9 +16,8 @@ jobs: run: | python setup.py sdist - name: Install test dependencies - env: - PYXMLSEC_STATIC_DEPS: true run: | + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl pip install --upgrade -r requirements-test.txt pip install black # for stub generation tests pip install dist/xmlsec-$(python setup.py --version).tar.gz diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py new file mode 100644 index 00000000..0a7c37de --- /dev/null +++ b/tests/softhsm_setup.py @@ -0,0 +1,265 @@ +""" +Testing the PKCS#11 shim layer. +Heavily inspired by from https://github.com/IdentityPython/pyXMLSecurity by leifj +under licence "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt +""" + +import logging +import os +import shutil +import subprocess +import tempfile +import traceback +import unittest +from typing import Dict, List, Optional, Tuple + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") + + +def paths_for_component(component: str, default_paths: List[str]): + env_path = os.environ.get(component) + return [env_path] if env_path else default_paths + + +def find_alts(component_name, alts: List[str]) -> str: + for a in alts: + if os.path.exists(a): + return a + raise unittest.SkipTest("Required component is missing: {}".format(component_name)) + + +def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: + env = {} + if softhsm_conf is not None: + env['SOFTHSM_CONF'] = softhsm_conf + env['SOFTHSM2_CONF'] = softhsm_conf + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + out, err = proc.communicate() + if err is not None and len(err) > 0: + logging.error(err) + if out is not None and len(out) > 0: + logging.debug(out) + rv = proc.wait() + if rv: + with open(softhsm_conf) as f: + conf = f.read() + msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' + msg = msg.format( + cmd=" ".join(args), code=rv, out=out.strip(), err=err.strip(), conf=conf, + ) + raise RuntimeError(msg) + return out, err + + +component_default_paths: Dict[str, List[str]] = { + 'P11_MODULE': [ + '/usr/lib/softhsm/libsofthsm2.so', + '/usr/lib/x86_64-linux-gnu/softhsm/libsofthsm2.so', + '/usr/lib/softhsm/libsofthsm.so', + '/usr/lib64/softhsm/libsofthsm2.so', + ], + 'P11_ENGINE': [ + '/usr/lib/ssl/engines/libpkcs11.so', + '/usr/lib/engines/engine_pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/libpkcs11.so', + '/usr/lib64/engines-3/pkcs11.so', + '/usr/lib64/engines-3/libpkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/libpkcs11.so', + ], + 'PKCS11_TOOL': [ + '/usr/bin/pkcs11-tool', + ], + 'SOFTHSM': [ + '/usr/bin/softhsm2-util', + '/usr/bin/softhsm', + ], + 'OPENSSL': [ + '/usr/bin/openssl', + ], +} + +component_path: Dict[str, str] = { + component_name: find_alts(component_name, paths_for_component(component_name, default_paths)) + for component_name, default_paths in component_default_paths.items() +} + +softhsm_version = 1 +if component_path['SOFTHSM'].endswith('softhsm2-util'): + softhsm_version = 2 + +openssl_version = subprocess.check_output([component_path['OPENSSL'], + 'version'] + )[8:11].decode() + +p11_test_files: List[str] = [] +softhsm_conf: Optional[str] = None +softhsm_db: Optional[str] = None + + +def _temp_file() -> str: + f = tempfile.NamedTemporaryFile(delete=False) + p11_test_files.append(f.name) + return f.name + + +def _temp_dir() -> str: + d = tempfile.mkdtemp() + p11_test_files.append(d) + return d + + +@unittest.skipIf(component_path['P11_MODULE'] is None, "SoftHSM PKCS11 module not installed") +def setup() -> None: + logging.debug("Creating test pkcs11 token using softhsm") + try: + global softhsm_conf + softhsm_conf = _temp_file() + logging.debug("Generating softhsm.conf") + with open(softhsm_conf, "w") as f: + if softhsm_version == 2: + softhsm_db = _temp_dir() + f.write(""" +# Generated by test +directories.tokendir = %s +objectstore.backend = file +log.level = DEBUG +""" % softhsm_db) + else: + softhsm_db = _temp_file() + f.write(""" +# Generated by test +0:%s +""" % softhsm_db) + + logging.debug("Initializing the token") + out, err = run_cmd([component_path['SOFTHSM'], + '--slot', '0', + '--label', 'test', + '--init-token', + '--pin', 'secret1', + '--so-pin', 'secret2'], + softhsm_conf=softhsm_conf) + + # logging.debug("Generating 1024 bit RSA key in token") + # run_cmd([component_path['PKCS11_TOOL'], + # '--module', component_path['P11_MODULE'], + # '-l', + # '-k', + # '--key-type', 'rsa:1024', + # '--id', 'a1b2', + # '--label', 'test', + # '--pin', 'secret1'], softhsm_conf=softhsm_conf) + + hash_priv_key = _temp_file() + logging.debug("Converting test private key to format for softhsm") + run_cmd([component_path['OPENSSL'], 'pkcs8', + '-topk8', + '-inform', 'PEM', + '-outform', 'PEM', + '-nocrypt', + '-in', os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', hash_priv_key], softhsm_conf=softhsm_conf) + + logging.debug("Importing the test key to softhsm") + run_cmd([component_path['SOFTHSM'], + '--import', hash_priv_key, + '--token', 'test', + '--id', 'a1b2', + '--label', 'test', + '--pin', 'secret1'], + softhsm_conf=softhsm_conf) + run_cmd([component_path['PKCS11_TOOL'], + '--module', component_path['P11_MODULE'], + '-l', + '--pin', 'secret1', '-O'], softhsm_conf=softhsm_conf) + signer_cert_pem = _temp_file() + openssl_conf = _temp_file() + logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) + with open(openssl_conf, "w") as f: + # Might be needed with some versions of openssl, but in more recent versions dynamic_path breaks it. + # dynamic_path = ( + # "dynamic_path = %s" % component_path['P11_ENGINE'] + # if openssl_version.startswith(b'1.') + # else "" + # ) + f.write("\n".join([ + "openssl_conf = openssl_def", + "[openssl_def]", + "engines = engine_section", + "[engine_section]", + "pkcs11 = pkcs11_section", + "[req]", + "distinguished_name = req_distinguished_name", + "[req_distinguished_name]", + "[pkcs11_section]", + "engine_id = pkcs11", + # dynamic_path, + "MODULE_PATH = %s" % component_path['P11_MODULE'], + "init = 0", + ])) + + with open(openssl_conf, "r") as f: + logging.debug('-------- START DEBUG openssl_conf --------') + logging.debug(f.readlines()) + logging.debug('-------- END DEBUG openssl_conf --------') + logging.debug('-------- START DEBUG paths --------') + logging.debug(run_cmd(['ls', '-ld', component_path['P11_ENGINE']])) + logging.debug(run_cmd(['ls', '-ld', component_path['P11_MODULE']])) + logging.debug('-------- END DEBUG paths --------') + + signer_cert_der = _temp_file() + + logging.debug("Generating self-signed certificate") + run_cmd([component_path['OPENSSL'], 'req', + '-new', + '-x509', + '-subj', "/CN=Test Signer", + '-engine', 'pkcs11', + '-config', openssl_conf, + '-keyform', 'engine', + '-key', 'label_test', + '-passin', 'pass:secret1', + '-out', signer_cert_pem], softhsm_conf=softhsm_conf) + + run_cmd([component_path['OPENSSL'], 'x509', + '-inform', 'PEM', + '-outform', 'DER', + '-in', signer_cert_pem, + '-out', signer_cert_der], softhsm_conf=softhsm_conf) + + logging.debug("Importing certificate into token") + + run_cmd([component_path['PKCS11_TOOL'], + '--module', component_path['P11_MODULE'], + '-l', + '--slot-index', '0', + '--id', 'a1b2', + '--label', 'test', + '-y', 'cert', + '-w', signer_cert_der, + '--pin', 'secret1'], softhsm_conf=softhsm_conf) + + # TODO: Should be teardowned in teardown: + os.environ['SOFTHSM_CONF'] = softhsm_conf + os.environ['SOFTHSM2_CONF'] = softhsm_conf + + except Exception as ex: + print("-" * 64) + traceback.print_exc() + print("-" * 64) + logging.error("PKCS11 tests disabled: unable to initialize test token: %s" % ex) + raise ex + + +def teardown() -> None: + global p11_test_files + for o in p11_test_files: + if os.path.exists(o): + if os.path.isdir(o): + shutil.rmtree(o) + else: + os.unlink(o) + p11_test_files = [] diff --git a/tests/test_pkcs11.py b/tests/test_pkcs11.py new file mode 100644 index 00000000..accd29ae --- /dev/null +++ b/tests/test_pkcs11.py @@ -0,0 +1,57 @@ +import xmlsec +from tests import base +from xmlsec import constants as consts + +KEY_URL = "pkcs11;pkcs11:token=test;object=test;pin-value=secret1" + + +def setUpModule(): + from tests import softhsm_setup + + softhsm_setup.setup() + + +def tearDownModule(): + from tests import softhsm_setup + + softhsm_setup.teardown() + + +class TestKeys(base.TestMemoryLeaks): + def test_del_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + del ctx.key + self.assertIsNone(ctx.key) + + def test_set_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + + def test_sign_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaises(TypeError): + ctx.sign('') + + def test_sign_fail(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): + ctx.sign(self.load_xml('sign1-in.xml')) + + def test_sign_case1(self): + """Should sign a pre-constructed template file using a key from a pkcs11 engine.""" + root = self.load_xml("sign1-in.xml") + sign = xmlsec.tree.find_node(root, consts.NodeSignature) + self.assertIsNotNone(sign) + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + ctx.key.name = 'rsakey.pem' + self.assertEqual("rsakey.pem", ctx.key.name) + + ctx.sign(sign) + self.assertEqual(self.load_xml("sign1-out.xml"), root) From 4f5daea286df89c64fbfc615f422be62b2cdf114 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 11:55:28 +0000 Subject: [PATCH 19/69] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/softhsm_setup.py | 220 +++++++++++++++++++++++++++-------------- 1 file changed, 147 insertions(+), 73 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 0a7c37de..432d4b1b 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -45,7 +45,11 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: conf = f.read() msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' msg = msg.format( - cmd=" ".join(args), code=rv, out=out.strip(), err=err.strip(), conf=conf, + cmd=" ".join(args), + code=rv, + out=out.strip(), + err=err.strip(), + conf=conf, ) raise RuntimeError(msg) return out, err @@ -90,9 +94,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: if component_path['SOFTHSM'].endswith('softhsm2-util'): softhsm_version = 2 -openssl_version = subprocess.check_output([component_path['OPENSSL'], - 'version'] - )[8:11].decode() +openssl_version = subprocess.check_output([component_path['OPENSSL'], 'version'])[8:11].decode() p11_test_files: List[str] = [] softhsm_conf: Optional[str] = None @@ -121,27 +123,41 @@ def setup() -> None: with open(softhsm_conf, "w") as f: if softhsm_version == 2: softhsm_db = _temp_dir() - f.write(""" + f.write( + """ # Generated by test directories.tokendir = %s objectstore.backend = file log.level = DEBUG -""" % softhsm_db) +""" + % softhsm_db + ) else: softhsm_db = _temp_file() - f.write(""" + f.write( + """ # Generated by test 0:%s -""" % softhsm_db) +""" + % softhsm_db + ) logging.debug("Initializing the token") - out, err = run_cmd([component_path['SOFTHSM'], - '--slot', '0', - '--label', 'test', - '--init-token', - '--pin', 'secret1', - '--so-pin', 'secret2'], - softhsm_conf=softhsm_conf) + out, err = run_cmd( + [ + component_path['SOFTHSM'], + '--slot', + '0', + '--label', + 'test', + '--init-token', + '--pin', + 'secret1', + '--so-pin', + 'secret2', + ], + softhsm_conf=softhsm_conf, + ) # logging.debug("Generating 1024 bit RSA key in token") # run_cmd([component_path['PKCS11_TOOL'], @@ -155,26 +171,45 @@ def setup() -> None: hash_priv_key = _temp_file() logging.debug("Converting test private key to format for softhsm") - run_cmd([component_path['OPENSSL'], 'pkcs8', - '-topk8', - '-inform', 'PEM', - '-outform', 'PEM', - '-nocrypt', - '-in', os.path.join(DATA_DIR, 'rsakey.pem'), - '-out', hash_priv_key], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['OPENSSL'], + 'pkcs8', + '-topk8', + '-inform', + 'PEM', + '-outform', + 'PEM', + '-nocrypt', + '-in', + os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', + hash_priv_key, + ], + softhsm_conf=softhsm_conf, + ) logging.debug("Importing the test key to softhsm") - run_cmd([component_path['SOFTHSM'], - '--import', hash_priv_key, - '--token', 'test', - '--id', 'a1b2', - '--label', 'test', - '--pin', 'secret1'], - softhsm_conf=softhsm_conf) - run_cmd([component_path['PKCS11_TOOL'], - '--module', component_path['P11_MODULE'], - '-l', - '--pin', 'secret1', '-O'], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['SOFTHSM'], + '--import', + hash_priv_key, + '--token', + 'test', + '--id', + 'a1b2', + '--label', + 'test', + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) + run_cmd( + [component_path['PKCS11_TOOL'], '--module', component_path['P11_MODULE'], '-l', '--pin', 'secret1', '-O'], + softhsm_conf=softhsm_conf, + ) signer_cert_pem = _temp_file() openssl_conf = _temp_file() logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) @@ -185,21 +220,25 @@ def setup() -> None: # if openssl_version.startswith(b'1.') # else "" # ) - f.write("\n".join([ - "openssl_conf = openssl_def", - "[openssl_def]", - "engines = engine_section", - "[engine_section]", - "pkcs11 = pkcs11_section", - "[req]", - "distinguished_name = req_distinguished_name", - "[req_distinguished_name]", - "[pkcs11_section]", - "engine_id = pkcs11", - # dynamic_path, - "MODULE_PATH = %s" % component_path['P11_MODULE'], - "init = 0", - ])) + f.write( + "\n".join( + [ + "openssl_conf = openssl_def", + "[openssl_def]", + "engines = engine_section", + "[engine_section]", + "pkcs11 = pkcs11_section", + "[req]", + "distinguished_name = req_distinguished_name", + "[req_distinguished_name]", + "[pkcs11_section]", + "engine_id = pkcs11", + # dynamic_path, + "MODULE_PATH = %s" % component_path['P11_MODULE'], + "init = 0", + ] + ) + ) with open(openssl_conf, "r") as f: logging.debug('-------- START DEBUG openssl_conf --------') @@ -213,34 +252,69 @@ def setup() -> None: signer_cert_der = _temp_file() logging.debug("Generating self-signed certificate") - run_cmd([component_path['OPENSSL'], 'req', - '-new', - '-x509', - '-subj', "/CN=Test Signer", - '-engine', 'pkcs11', - '-config', openssl_conf, - '-keyform', 'engine', - '-key', 'label_test', - '-passin', 'pass:secret1', - '-out', signer_cert_pem], softhsm_conf=softhsm_conf) - - run_cmd([component_path['OPENSSL'], 'x509', - '-inform', 'PEM', - '-outform', 'DER', - '-in', signer_cert_pem, - '-out', signer_cert_der], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['OPENSSL'], + 'req', + '-new', + '-x509', + '-subj', + "/CN=Test Signer", + '-engine', + 'pkcs11', + '-config', + openssl_conf, + '-keyform', + 'engine', + '-key', + 'label_test', + '-passin', + 'pass:secret1', + '-out', + signer_cert_pem, + ], + softhsm_conf=softhsm_conf, + ) + + run_cmd( + [ + component_path['OPENSSL'], + 'x509', + '-inform', + 'PEM', + '-outform', + 'DER', + '-in', + signer_cert_pem, + '-out', + signer_cert_der, + ], + softhsm_conf=softhsm_conf, + ) logging.debug("Importing certificate into token") - run_cmd([component_path['PKCS11_TOOL'], - '--module', component_path['P11_MODULE'], - '-l', - '--slot-index', '0', - '--id', 'a1b2', - '--label', 'test', - '-y', 'cert', - '-w', signer_cert_der, - '--pin', 'secret1'], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--slot-index', + '0', + '--id', + 'a1b2', + '--label', + 'test', + '-y', + 'cert', + '-w', + signer_cert_der, + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) # TODO: Should be teardowned in teardown: os.environ['SOFTHSM_CONF'] = softhsm_conf From 70753591a9b56542961cb1e3b4cd05c92aea7028 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 12 Jun 2023 14:26:51 +0200 Subject: [PATCH 20/69] Remove SOAP The SOAP support fully removed from xmlsec1 library. --- doc/source/modules/constants.rst | 6 - src/constants.c | 2 - src/xmlsec/constants.pyi | 2 - tests/data/enc-bad-in.xml | 208 ------------------------------- tests/test_enc.py | 19 --- 5 files changed, 237 deletions(-) delete mode 100644 tests/data/enc-bad-in.xml diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 4a63fcd7..8127590a 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -166,12 +166,6 @@ Namespaces .. data:: xmlsec.constants.XPointerNs :annotation: = 'http://www.w3.org/2001/04/xmldsig-more/xptr' -.. data:: xmlsec.constants.Soap11Ns - :annotation: = 'http://schemas.xmlsoap.org/soap/envelope/' - -.. data:: xmlsec.constants.Soap12Ns - :annotation: = 'http://www.w3.org/2002/06/soap-envelope' - .. data:: xmlsec.constants.NsExcC14N :annotation: = 'http://www.w3.org/2001/10/xml-exc-c14n#' diff --git a/src/constants.c b/src/constants.c index 34c81b29..72ae217f 100644 --- a/src/constants.c +++ b/src/constants.c @@ -316,8 +316,6 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_NS_CONSTANT(XPathNs, "XPATH"); PYXMLSEC_ADD_NS_CONSTANT(XPath2Ns, "XPATH2"); PYXMLSEC_ADD_NS_CONSTANT(XPointerNs, "XPOINTER"); - PYXMLSEC_ADD_NS_CONSTANT(Soap11Ns, "SOAP11"); - PYXMLSEC_ADD_NS_CONSTANT(Soap12Ns, "SOAP12"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14N, "EXC_C14N"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14NWithComments, "EXC_C14N_WITH_COMMENT"); diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 3430a027..9fd24e53 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -85,8 +85,6 @@ NodeX509Data: Final[str] Ns: Final[str] NsExcC14N: Final[str] NsExcC14NWithComments: Final[str] -Soap11Ns: Final[str] -Soap12Ns: Final[str] TransformAes128Cbc: Final[__Transform] TransformAes128Gcm: Final[__Transform] TransformAes192Cbc: Final[__Transform] diff --git a/tests/data/enc-bad-in.xml b/tests/data/enc-bad-in.xml deleted file mode 100644 index 460738fc..00000000 --- a/tests/data/enc-bad-in.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - - - - - - MyNextCar - CreditApplication - MYNEXTCAR - VW - 409D03 - MyNextCar - - 2018-11-20T09:37:45Z - 7f0842cc-8d47-4955-be31-c61d07ee490b - - VW - - - - - - -
- VCI_MNA_0000070250 - - - Car Chantilly -
- 14839 Stonecroft Center Ct - Chantilly - VA - US - 20151 -
- - - MyNextCar - MNA - - 7039562100 - - CAR - - N -
- - - 2017 - Q7 - CAR - New - 0 - Prestige - - 64300.0 - MSRP - - - 64300.0 - Selling Price - - - - - 113456789 - NationalId - - - John - Q - Public - -
- 999 Washington Ave - Apt #332 - Front Royal - VA - US - 22630 - 01 - 10 - Own -
-
- 21 E 9th Ave - Boulder - CO - US - 80301-7577 - 07 - 11 - Own -
- - 3032852402 - 3032852405 - 7203554444 - JohnQPublic@anydomain.org - - - 1967-07-31 - - 0 - - UPS -
- 1775 Wiehle Ave. - Reston - VA - US - 20190 -
- 9500.0 - Driver - 01 - 05 - Current - - - FedEx - 4000.00 - Driver - 04 - 09 - Previous - - 1252.52 - - 1500.00 - - - 1 - Consents to Credit Check - - - - - 123435325 - NationalId - - - Lisa - C - Public - -
- 999 Lewis Street - Front Royal - VA - US - 22630 - 5 - 0 - Own -
- - 5401110000 - 5401110073 - public@test.com - - - 1963-04-20 - - - Christendom College -
- 999 Christendom Dr - Front Royal - VA - US - 22630 -
- 6200.00 - Professor - 5 - 0 - Current - - 1252.52 - - 1 - Consents to Credit Check - - - - R - 0.00 - 66 - 5000.00 - INDIVCOAPP - 2000.00 - MyNextCar - - 1978 - Bonneville - Pontiac - Coupe - - - - - - - - diff --git a/tests/test_enc.py b/tests/test_enc.py index 63e00169..1788b4d6 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -233,22 +233,3 @@ def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() with self.assertRaises(TypeError): ctx.decrypt('') - - def check_no_segfault(self): - namespaces = {'soap': 'http://schemas.xmlsoap.org/soap/envelope/'} - - manager = xmlsec.KeysManager() - key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) - manager.add_key(key) - template = self.load_xml('enc-bad-in.xml') - enc_data = xmlsec.template.encrypted_data_create( - template, xmlsec.Transform.AES128, type=xmlsec.EncryptionType.CONTENT, ns='xenc' - ) - xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) - key_info = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') - enc_key = xmlsec.template.add_encrypted_key(key_info, xmlsec.Transform.RSA_PKCS1) - xmlsec.template.encrypted_data_ensure_cipher_value(enc_key) - data = template.find('soap:Body', namespaces=namespaces) - enc_ctx = xmlsec.EncryptionContext(manager) - enc_ctx.key = xmlsec.Key.generate(xmlsec.KeyData.AES, 192, xmlsec.KeyDataType.SESSION) - self.assertRaises(Exception, enc_ctx.encrypt_xml(enc_data, data)) From 2c26131e2d965346b538d86349fe121027afd9f4 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 12 Jun 2023 15:12:57 +0200 Subject: [PATCH 21/69] Upgrade isort because of poetry dependency issue Fix https://github.com/PyCQA/isort/issues/2077 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..bf877f39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From b865569c84ff00f116b321527a2bd381321c9402 Mon Sep 17 00:00:00 2001 From: Konstantin Demin Date: Thu, 24 Aug 2023 00:29:17 +0300 Subject: [PATCH 22/69] setup: better deal with compiler flags - honor environment variable PYXMLSEC_OPTIMIZE_SIZE to switch between speed and size optimization - enabled by default for backward compatibility - better deal with compiler flags for different platforms Signed-off-by: Konstantin Demin --- setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9a3c9277..c0fd0f7d 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def run(self): ext = self.ext_map['xmlsec'] self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) + self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) if self.static or sys.platform == 'win32': self.info('starting static build on {}'.format(sys.platform)) @@ -153,11 +154,18 @@ def run(self): ) if self.debug: - ext.extra_compile_args.append('-Wall') - ext.extra_compile_args.append('-O0') ext.define_macros.append(('PYXMLSEC_ENABLE_DEBUG', '1')) + if sys.platform == 'win32': + ext.extra_compile_args.append('/Od') + else: + ext.extra_compile_args.append('-Wall') + ext.extra_compile_args.append('-O0') else: - ext.extra_compile_args.append('-Os') + if self.size_opt: + if sys.platform == 'win32': + ext.extra_compile_args.append('/Os') + else: + ext.extra_compile_args.append('-Os') super(build_ext, self).run() From 5f57f2eac756df213e83a2fdb452057909e2dd21 Mon Sep 17 00:00:00 2001 From: David Adamec Date: Fri, 15 Sep 2023 17:49:15 -0700 Subject: [PATCH 23/69] Add compatibility for xmlsec 1.3 --- doc/source/modules/constants.rst | 6 ------ src/constants.c | 2 -- src/enc.c | 7 +++++++ src/xmlsec/constants.pyi | 2 -- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 4a63fcd7..8127590a 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -166,12 +166,6 @@ Namespaces .. data:: xmlsec.constants.XPointerNs :annotation: = 'http://www.w3.org/2001/04/xmldsig-more/xptr' -.. data:: xmlsec.constants.Soap11Ns - :annotation: = 'http://schemas.xmlsoap.org/soap/envelope/' - -.. data:: xmlsec.constants.Soap12Ns - :annotation: = 'http://www.w3.org/2002/06/soap-envelope' - .. data:: xmlsec.constants.NsExcC14N :annotation: = 'http://www.w3.org/2001/10/xml-exc-c14n#' diff --git a/src/constants.c b/src/constants.c index 34c81b29..72ae217f 100644 --- a/src/constants.c +++ b/src/constants.c @@ -316,8 +316,6 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_NS_CONSTANT(XPathNs, "XPATH"); PYXMLSEC_ADD_NS_CONSTANT(XPath2Ns, "XPATH2"); PYXMLSEC_ADD_NS_CONSTANT(XPointerNs, "XPOINTER"); - PYXMLSEC_ADD_NS_CONSTANT(Soap11Ns, "SOAP11"); - PYXMLSEC_ADD_NS_CONSTANT(Soap12Ns, "SOAP12"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14N, "EXC_C14N"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14NWithComments, "EXC_C14N_WITH_COMMENT"); diff --git a/src/enc.c b/src/enc.c index aaf35ae5..475cd2d4 100644 --- a/src/enc.c +++ b/src/enc.c @@ -50,6 +50,13 @@ static int PyXmlSec_EncryptionContext__init__(PyObject* self, PyObject* args, Py } ctx->manager = manager; PYXMLSEC_DEBUGF("%p: init enc context - ok, manager - %p", self, manager); + + // xmlsec 1.3 changed the key search to strict mode, causing various examples + // in the docs to fail. For backwards compatibility, this changes it back to + // lax mode for now. + ctx->handle->keyInfoReadCtx.flags = XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH; + ctx->handle->keyInfoWriteCtx.flags = XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH; + return 0; ON_FAIL: PYXMLSEC_DEBUGF("%p: init enc context - failed", self); diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 3430a027..9fd24e53 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -85,8 +85,6 @@ NodeX509Data: Final[str] Ns: Final[str] NsExcC14N: Final[str] NsExcC14NWithComments: Final[str] -Soap11Ns: Final[str] -Soap12Ns: Final[str] TransformAes128Cbc: Final[__Transform] TransformAes128Gcm: Final[__Transform] TransformAes192Cbc: Final[__Transform] From 0e7c5e718e8d221331b0a5f851094a5b4181eeae Mon Sep 17 00:00:00 2001 From: David Adamec Date: Fri, 15 Sep 2023 18:07:07 -0700 Subject: [PATCH 24/69] fix 1.2 compatibility --- src/enc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/enc.c b/src/enc.c index 475cd2d4..5453ef99 100644 --- a/src/enc.c +++ b/src/enc.c @@ -17,6 +17,11 @@ #include #include +// Backwards compatibility with xmlsec 1.2 +#ifndef XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH +#define XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH 0x00008000 +#endif + typedef struct { PyObject_HEAD xmlSecEncCtxPtr handle; From 40c14c45a352f2812e796a071ef7d12f5a70be58 Mon Sep 17 00:00:00 2001 From: David Adamec Date: Fri, 15 Sep 2023 19:40:04 -0700 Subject: [PATCH 25/69] update isort to fix poetry error in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..bf877f39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From f2617835b9c0dc8740caa89e7aa765e3d5afe445 Mon Sep 17 00:00:00 2001 From: Chris Novakovic Date: Thu, 12 Oct 2023 18:09:38 +0100 Subject: [PATCH 26/69] Don't define `MODULE_NAME` as a string literal The `MODULE_NAME` macro is used in contexts where a string literal is not valid, but the fallback value set in `src/common.h` defines it as such; this differs from how it is defined in `setup.py`. Define `MODULE_NAME` in `src/common.h` as it is defined in `setup.py`. Fixes #267. --- src/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common.h b/src/common.h index 243ed651..a6176551 100644 --- a/src/common.h +++ b/src/common.h @@ -13,7 +13,7 @@ #include "debug.h" #ifndef MODULE_NAME -#define MODULE_NAME "xmlsec" +#define MODULE_NAME xmlsec #endif #define JOIN(X,Y) DO_JOIN1(X,Y) From c97f2b8ddc7b30de73411bdfbeb9903899ee1495 Mon Sep 17 00:00:00 2001 From: Chris Novakovic Date: Fri, 13 Oct 2023 10:53:48 +0100 Subject: [PATCH 27/69] Make DES/3DES/KW-3DES support conditional on DES availability in XMLSec Some TLS libraries (e.g. OpenSSL) can be built without support for DES and DES-derived algorithms. Rather than assuming that xmlsec always supports them, guard the declaration of the DES, 3DES and KW-3DES constants based on the value of `XMLSEC_NO_DES`. Fixes #269. --- src/constants.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/constants.c b/src/constants.c index 34c81b29..d797086d 100644 --- a/src/constants.c +++ b/src/constants.c @@ -441,7 +441,9 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataRetrievalMethod, "RETRIEVALMETHOD") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEncryptedKey, "ENCRYPTEDKEY") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataAes, "AES") +#ifndef XMLSEC_NO_DES PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDes, "DES") +#endif #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDsa, "DSA") #endif @@ -489,8 +491,10 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWAes192, "KW_AES192"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWAes256, "KW_AES256"); +#ifndef XMLSEC_NO_DES PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDes3Cbc, "DES3"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWDes3, "KW_DES3"); +#endif #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDsaSha1, "DSA_SHA1"); #endif From 31bea7c0d2694150818f9d901682aa1fe84449ac Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Sat, 13 Jan 2024 19:40:05 -0500 Subject: [PATCH 28/69] Use KeyDataEc rather than deprecated KeyDataEcdsa --- doc/source/modules/constants.rst | 2 +- src/constants.c | 2 +- src/xmlsec/constants.pyi | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 8127590a..8fdac118 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -47,7 +47,7 @@ KeyData The DSA key klass. -.. data:: xmlsec.constants.KeyDataEcdsa +.. data:: xmlsec.constants.KeyDataEc The ECDSA key klass. diff --git a/src/constants.c b/src/constants.c index e3b64527..f2c5a636 100644 --- a/src/constants.c +++ b/src/constants.c @@ -447,7 +447,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #endif #if XMLSEC_VERSION_HEX > 0x10212 // from version 1.2.19 - PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEcdsa, "ECDSA") + PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEc, "ECDSA") #endif PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataHmac, "HMAC") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataRsa, "RSA") diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 9fd24e53..80afdd22 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -29,7 +29,7 @@ EncNs: Final[str] KeyDataAes: Final[__KeyData] KeyDataDes: Final[__KeyData] KeyDataDsa: Final[__KeyData] -KeyDataEcdsa: Final[__KeyData] +KeyDataEc: Final[__KeyData] KeyDataEncryptedKey: Final[__KeyData] KeyDataFormatBinary: Final[int] KeyDataFormatCertDer: Final[int] From 7891e715a7c343a1fd52c86a5085a1d1cc62e711 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Sat, 13 Jan 2024 19:40:28 -0500 Subject: [PATCH 29/69] Use xmlSecCryptoAppKeyLoadEx instead of deprecated xmlSecCryptoAppKeyLoad --- src/keys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys.c b/src/keys.c index 1362b128..47678a53 100644 --- a/src/keys.c +++ b/src/keys.c @@ -163,7 +163,7 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* if (is_content) { key->handle = xmlSecCryptoAppKeyLoadMemory((const xmlSecByte*)data, (xmlSecSize)data_size, format, password, NULL, NULL); } else { - key->handle = xmlSecCryptoAppKeyLoad(data, format, password, NULL, NULL); + key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); } Py_END_ALLOW_THREADS; From c9c660b8f336a8501ab655b9f59a993378b5f38b Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 12 Mar 2024 19:51:08 -0400 Subject: [PATCH 30/69] Use xmlSecCryptoAppKeyLoadEx instead of xmlSecCryptoAppKeyLoad for pkcs11 support --- src/keys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys.c b/src/keys.c index a8ffde25..8b84c343 100644 --- a/src/keys.c +++ b/src/keys.c @@ -206,7 +206,7 @@ static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; Py_BEGIN_ALLOW_THREADS; - key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), + key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); Py_END_ALLOW_THREADS; From 595aeaa2651ba0a1f80565ceac7550fd1abd38fe Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Wed, 13 Mar 2024 16:46:55 +0100 Subject: [PATCH 31/69] Make the library backward compatible Starting from version xmlsec1-1.3.3, some of the constant and keys that were deprecated, removed from the library entirely. This change would add version awareness and backward compatible. --- doc/source/modules/constants.rst | 6 +++++- src/constants.c | 7 +++++-- src/keys.c | 15 ++++++++++++--- src/xmlsec/constants.pyi | 1 + 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 8fdac118..3df6b50f 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -47,9 +47,13 @@ KeyData The DSA key klass. +.. data:: xmlsec.constants.KeyDataEcdsa + + (Deprecated. The EC key klass) The ECDSA key klass. + .. data:: xmlsec.constants.KeyDataEc - The ECDSA key klass. + The EC key klass. .. data:: xmlsec.constants.KeyDataHmac diff --git a/src/constants.c b/src/constants.c index f2c5a636..bd1fa5e0 100644 --- a/src/constants.c +++ b/src/constants.c @@ -445,8 +445,11 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDsa, "DSA") #endif -#if XMLSEC_VERSION_HEX > 0x10212 - // from version 1.2.19 +#if XMLSEC_VERSION_HEX > 0x10212 && XMLSEC_VERSION_HEX < 0x10303 + // from version 1.2.19 to version 1.3.2 (inclusive) + PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEcdsa, "ECDSA") +#elif XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEc, "ECDSA") #endif PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataHmac, "HMAC") diff --git a/src/keys.c b/src/keys.c index 8b84c343..5ff04aae 100644 --- a/src/keys.c +++ b/src/keys.c @@ -163,7 +163,12 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* if (is_content) { key->handle = xmlSecCryptoAppKeyLoadMemory((const xmlSecByte*)data, (xmlSecSize)data_size, format, password, NULL, NULL); } else { - key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); + #if XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); + #else + key->handle = xmlSecCryptoAppKeyLoad(data, format, password, NULL, NULL); + #endif } Py_END_ALLOW_THREADS; @@ -206,8 +211,12 @@ static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; Py_BEGIN_ALLOW_THREADS; - key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), - (void*)engine_and_key_id); + #if XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); + #else + key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); + #endif Py_END_ALLOW_THREADS; if (key->handle == NULL) { diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 80afdd22..3c3ea94f 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -30,6 +30,7 @@ KeyDataAes: Final[__KeyData] KeyDataDes: Final[__KeyData] KeyDataDsa: Final[__KeyData] KeyDataEc: Final[__KeyData] +KeyDataEcdsa: Final[__KeyData] KeyDataEncryptedKey: Final[__KeyData] KeyDataFormatBinary: Final[int] KeyDataFormatCertDer: Final[int] From 802dff28340d9031d694ad77825a7333dc98ef93 Mon Sep 17 00:00:00 2001 From: Jim Jagielski Date: Wed, 13 Mar 2024 16:17:34 -0400 Subject: [PATCH 32/69] Update correct URL: No func change --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e1924652..60fb3ea1 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ Building from source .. code-block:: bash - git clone https://github.com/mehcode/python-xmlsec.git + git clone https://github.com/xmlsec/python-xmlsec.git #. Change into the ``python-xmlsec`` root directory. From 0bad2e462da4ea912c7a258334d0bcb3e0c5aa86 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 17 Mar 2024 13:25:15 -0300 Subject: [PATCH 33/69] Add ability to query libxml version number --- src/main.c | 15 ++++++++++++++- src/platform.h | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main.c b/src/main.c index 5773db3b..0ee5e2cd 100644 --- a/src/main.c +++ b/src/main.c @@ -121,11 +121,18 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) { static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \ "get_libxmlsec_version() -> tuple\n" - "Returns Version tuple of wrapped libxml library."; + "Returns Version tuple of wrapped libxmlsec library."; static PyObject* PyXmlSec_GetLibXmlSecVersion() { return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR); } +static char PyXmlSec_GetLibXmlVersion__doc__[] = \ + "get_libxml_version() -> tuple\n" + "Returns Version tuple of wrapped libxml library."; +static PyObject* PyXmlSec_GetLibXmlVersion() { + return Py_BuildValue("(iii)", XMLSEC_LIBXML_VERSION_MAJOR, XMLSEC_LIBXML_VERSION_MINOR, XMLSEC_LIBXML_VERSION_PATCH); +} + static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ "enable_debug_trace(enabled) -> None\n" "Enables or disables calling LibXML2 callback from the default errors callback.\n\n" @@ -399,6 +406,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_GetLibXmlSecVersion__doc__ }, + { + "get_libxml_version", + (PyCFunction)PyXmlSec_GetLibXmlVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/src/platform.h b/src/platform.h index 35163e88..1fc82b7b 100644 --- a/src/platform.h +++ b/src/platform.h @@ -12,6 +12,7 @@ #define PY_SSIZE_T_CLEAN 1 +#include #include #include @@ -19,6 +20,12 @@ #include #endif /* MS_WIN32 */ +#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) + +#define XMLSEC_LIBXML_VERSION_MAJOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100 * 100) +#define XMLSEC_LIBXML_VERSION_MINOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100) +#define XMLSEC_LIBXML_VERSION_PATCH XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 1) + #define XMLSEC_VERSION_HEX ((XMLSEC_VERSION_MAJOR << 16) | (XMLSEC_VERSION_MINOR << 8) | (XMLSEC_VERSION_SUBMINOR)) // XKMS support was removed in version 1.2.21 From 01b18c9a09ee54a3d78c8da039777de8512d7380 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 20:35:34 -0300 Subject: [PATCH 34/69] Set enable-md5 when doing static build for libxmlsec1 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 489844c8..277a8892 100644 --- a/setup.py +++ b/setup.py @@ -437,6 +437,7 @@ def prepare_static_build_linux(self): prefix_arg, '--disable-shared', '--disable-gost', + '--enable-md5', '--disable-crypto-dl', '--enable-static=yes', '--enable-shared=no', From 2a717dd3987ba9538617874aba36e4f8326f514e Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:25:01 -0300 Subject: [PATCH 35/69] Fix sdist workflow --- .github/workflows/sdist.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 6ca7a657..e7c0f39d 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -18,8 +18,7 @@ jobs: - name: Install test dependencies run: | sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl - pip install --upgrade -r requirements-test.txt - pip install black # for stub generation tests + pip install --upgrade -r requirements-test.txt --no-binary lxml pip install dist/xmlsec-$(python setup.py --version).tar.gz - name: Run tests run: | From e8ff43b50faf940637d0627b7de2f4566f0dfe3d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 22:05:04 -0300 Subject: [PATCH 36/69] Fix macos workflow --- .github/workflows/macosx.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 24fa6ddd..2c974831 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -19,8 +19,9 @@ jobs: - name: Build macosx_x86_64 wheel env: CC: clang - CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -I/usr/local/opt/libxml2/include" + LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping -L/usr/local/opt/libxml2/lib" + PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" run: | python -m build rm -rf build/ @@ -31,9 +32,13 @@ jobs: echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - name: Install test dependencies run: | - pip install coverage --upgrade -r requirements-test.txt + pip install coverage --upgrade -r requirements-test.txt --no-binary lxml pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV + env: + CFLAGS: "-I/usr/local/opt/libxml2/include" + LDFLAGS: "-L/usr/local/opt/libxml2/lib" + PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" - name: Run tests run: | coverage run -m pytest -v --color=yes From 5c2cbd59b936bc5fe8563c4cfa9a1b350b5fae3f Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 09:08:36 -0300 Subject: [PATCH 37/69] Add arch flag --- .github/workflows/macosx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 2c974831..39fc37de 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -36,6 +36,7 @@ jobs: pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV env: + CC: clang CFLAGS: "-I/usr/local/opt/libxml2/include" LDFLAGS: "-L/usr/local/opt/libxml2/lib" PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" @@ -45,5 +46,5 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov -arch x86_64 show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt From ad0b27d3ac3f1d89b4641dfdc1482d54d9dd20a1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 09:13:53 -0300 Subject: [PATCH 38/69] No need to install lxml from source, brew libxml2 version matches --- .github/workflows/macosx.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 39fc37de..129c4e89 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -32,14 +32,9 @@ jobs: echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - name: Install test dependencies run: | - pip install coverage --upgrade -r requirements-test.txt --no-binary lxml + pip install coverage --upgrade -r requirements-test.txt pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV - env: - CC: clang - CFLAGS: "-I/usr/local/opt/libxml2/include" - LDFLAGS: "-L/usr/local/opt/libxml2/lib" - PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" - name: Run tests run: | coverage run -m pytest -v --color=yes From 4bd490143932efaaf41f25ea3b3f3866cbe50554 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 09:16:54 -0300 Subject: [PATCH 39/69] fix arch flag --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 129c4e89..2bc44c1e 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -41,5 +41,5 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov -arch x86_64 show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=x86_64 --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt From 869beeb281cc1366a63e29f7145c62082fbacbef Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:28:25 -0300 Subject: [PATCH 40/69] Fix manylinux build --- .github/workflows/manylinux.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 520e5ba6..6d2dbb1a 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -7,14 +7,7 @@ jobs: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - - manylinux2010_x86_64 - - manylinux_2_24_x86_64 - - musllinux_1_1_x86_64 - exclude: - - image: manylinux2010_x86_64 - python-abi: cp311-cp311 - - image: manylinux2010_i686 - python-abi: cp311-cp311 + - manylinux2014_x86_64 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 From 5794266cff235c0788842c49815408ed35f04e06 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 11:19:59 -0300 Subject: [PATCH 41/69] Try just setting pkg_config_path. --- .github/workflows/macosx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 2bc44c1e..e2a04057 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -19,8 +19,8 @@ jobs: - name: Build macosx_x86_64 wheel env: CC: clang - CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -I/usr/local/opt/libxml2/include" - LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping -L/usr/local/opt/libxml2/lib" + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" run: | python -m build From ee9fbd03ac4c466447f0f67e3032866657dcb19d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 20:52:49 -0300 Subject: [PATCH 42/69] Version check when setting up lxml - Expose both the compiled version and the linked version of libxml2 - Do a check that the versions match when initializing the module. Otherwise raise an exception. This seems like a better result then a difficult to diagnose segfault. --- src/lxml.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lxml.h | 9 +++++ src/main.c | 30 ++++++++++++++-- src/platform.h | 7 ---- 4 files changed, 131 insertions(+), 10 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index aa1abae0..bb7e6a96 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -9,6 +9,7 @@ #include "common.h" #include "lxml.h" +#include "exception.h" #include #include @@ -17,8 +18,102 @@ #include #include +#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) + +#define XMLSEC_EXTRACT_MAJOR(x) XMLSEC_EXTRACT_VERSION(x, 100 * 100) +#define XMLSEC_EXTRACT_MINOR(x) XMLSEC_EXTRACT_VERSION(x, 100) +#define XMLSEC_EXTRACT_PATCH(x) XMLSEC_EXTRACT_VERSION(x, 1) + +static long PyXmlSec_GetLibXmlVersionLong() { + return PyOS_strtol(xmlParserVersion, NULL, 10); +} +long PyXmlSec_GetLibXmlVersionMajor() { + return XMLSEC_EXTRACT_MAJOR(PyXmlSec_GetLibXmlVersionLong()); +} +long PyXmlSec_GetLibXmlVersionMinor() { + return XMLSEC_EXTRACT_MINOR(PyXmlSec_GetLibXmlVersionLong()); +} +long PyXmlSec_GetLibXmlVersionPatch() { + return XMLSEC_EXTRACT_PATCH(PyXmlSec_GetLibXmlVersionLong()); +} + +long PyXmlSec_GetLibXmlCompiledVersionMajor() { + return XMLSEC_EXTRACT_MAJOR(LIBXML_VERSION); +} +long PyXmlSec_GetLibXmlCompiledVersionMinor() { + return XMLSEC_EXTRACT_MINOR(LIBXML_VERSION); +} +long PyXmlSec_GetLibXmlCompiledVersionPatch() { + return XMLSEC_EXTRACT_PATCH(LIBXML_VERSION); +} + +static int PyXmlSec_CheckLibXmlLibraryVersion(void) { + // Make sure that the version of libxml2 that we were compiled against is the same as the one + // that is loaded. If there is a version mismatch, we could run into segfaults. + + if (PyXmlSec_GetLibXmlVersionMajor() != PyXmlSec_GetLibXmlCompiledVersionMajor() || + PyXmlSec_GetLibXmlVersionMinor() != PyXmlSec_GetLibXmlCompiledVersionMinor()) { + return -1; + } + + return 0; +} + +static int PyXmlSec_CheckLxmlLibraryVersion(void) { + // Make sure that the version of libxml2 lxml is using is the same as the one we are using. Because + // we pass trees between the two libraries, we need to make sure that they are using the same version + // of libxml2, or we could run into difficult to debug segfaults. + // See: https://github.com/xmlsec/python-xmlsec/issues/283 + + PyObject* lxml = NULL; + PyObject* version = NULL; + + // Default to failure + int result = -1; + + lxml = PyImport_ImportModule("lxml.etree"); + if (lxml == NULL) { + goto FINALIZE; + } + version = PyObject_GetAttrString(lxml, "LIBXML_VERSION"); + if (version == NULL) { + goto FINALIZE; + } + if (!PyTuple_Check(version) || PyTuple_Size(version) != 3) { + goto FINALIZE; + } + + PyObject* major = PyTuple_GetItem(version, 0); + PyObject* minor = PyTuple_GetItem(version, 1); + + if (!PyLong_Check(major) || !PyLong_Check(minor)) { + goto FINALIZE; + } + + if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { + goto FINALIZE; + } + + result = 0; + +FINALIZE: + // Cleanup our references, and return the result + Py_XDECREF(lxml); + Py_XDECREF(version); + return result; +} int PyXmlSec_InitLxmlModule(void) { + if (PyXmlSec_CheckLibXmlLibraryVersion() < 0) { + PyXmlSec_SetLastError("xmlsec libxml2 library compiled version vs runtime version mismatch"); + return -1; + } + + if (PyXmlSec_CheckLxmlLibraryVersion() < 0) { + PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); + return -1; + } + return import_lxml__etree(); } diff --git a/src/lxml.h b/src/lxml.h index 6824076b..72050efe 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -29,4 +29,13 @@ PyXmlSec_LxmlElementPtr PyXmlSec_elementFactory(PyXmlSec_LxmlDocumentPtr doc, xm // converts o to PyObject, None object is not allowed, does not increment ref_counts int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); +// get version numbers for libxml2 both compiled and loaded +long PyXmlSec_GetLibXmlVersionMajor(); +long PyXmlSec_GetLibXmlVersionMinor(); +long PyXmlSec_GetLibXmlVersionPatch(); + +long PyXmlSec_GetLibXmlCompiledVersionMajor(); +long PyXmlSec_GetLibXmlCompiledVersionMinor(); +long PyXmlSec_GetLibXmlCompiledVersionPatch(); + #endif // __PYXMLSEC_LXML_H__ diff --git a/src/main.c b/src/main.c index 0ee5e2cd..61eac139 100644 --- a/src/main.c +++ b/src/main.c @@ -10,6 +10,7 @@ #include "common.h" #include "platform.h" #include "exception.h" +#include "lxml.h" #include #include @@ -127,10 +128,27 @@ static PyObject* PyXmlSec_GetLibXmlSecVersion() { } static char PyXmlSec_GetLibXmlVersion__doc__[] = \ - "get_libxml_version() -> tuple\n" - "Returns Version tuple of wrapped libxml library."; + "get_libxml_version() -> tuple[int, int, int]\n" + "Returns version tuple of libxml2 library xmlsec is using."; static PyObject* PyXmlSec_GetLibXmlVersion() { - return Py_BuildValue("(iii)", XMLSEC_LIBXML_VERSION_MAJOR, XMLSEC_LIBXML_VERSION_MINOR, XMLSEC_LIBXML_VERSION_PATCH); + return Py_BuildValue( + "(iii)", + PyXmlSec_GetLibXmlVersionMajor(), + PyXmlSec_GetLibXmlVersionMinor(), + PyXmlSec_GetLibXmlVersionPatch() + ); +} + +static char PyXmlSec_GetLibXmlCompiledVersion__doc__[] = \ + "get_libxml_compiled_version() -> tuple[int, int, int]\n" + "Returns version tuple of libxml2 library xmlsec was compiled with."; +static PyObject* PyXmlSec_GetLibXmlCompiledVersion() { + return Py_BuildValue( + "(iii)", + PyXmlSec_GetLibXmlCompiledVersionMajor(), + PyXmlSec_GetLibXmlCompiledVersionMinor(), + PyXmlSec_GetLibXmlCompiledVersionPatch() + ); } static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ @@ -412,6 +430,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_GetLibXmlVersion__doc__ }, + { + "get_libxml_compiled_version", + (PyCFunction)PyXmlSec_GetLibXmlCompiledVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlCompiledVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/src/platform.h b/src/platform.h index 1fc82b7b..35163e88 100644 --- a/src/platform.h +++ b/src/platform.h @@ -12,7 +12,6 @@ #define PY_SSIZE_T_CLEAN 1 -#include #include #include @@ -20,12 +19,6 @@ #include #endif /* MS_WIN32 */ -#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) - -#define XMLSEC_LIBXML_VERSION_MAJOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100 * 100) -#define XMLSEC_LIBXML_VERSION_MINOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100) -#define XMLSEC_LIBXML_VERSION_PATCH XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 1) - #define XMLSEC_VERSION_HEX ((XMLSEC_VERSION_MAJOR << 16) | (XMLSEC_VERSION_MINOR << 8) | (XMLSEC_VERSION_SUBMINOR)) // XKMS support was removed in version 1.2.21 From d8d6c6266a1a9c3deed4a8b809325c6549c84530 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 24 Mar 2024 19:45:23 -0300 Subject: [PATCH 43/69] Update .github/workflows/macosx.yml Co-authored-by: Stu Tomlinson --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index e2a04057..f1bfb7d0 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -21,7 +21,7 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" + PKG_CONFIG_PATH: "$(brew --prefix)/opt/libxml2/lib/pkgconfig" run: | python -m build rm -rf build/ From d84f015f6aa4a6b058c4b0ef5ca9044f39b512f8 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 24 Mar 2024 19:45:32 -0300 Subject: [PATCH 44/69] Update .github/workflows/macosx.yml Co-authored-by: Stu Tomlinson --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index f1bfb7d0..9b5149c1 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -41,5 +41,5 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=x86_64 --instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=$(uname -m) --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt From 47cb969eeb99d9435fb9e3eb063f5dd2490f220b Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 24 Mar 2024 19:56:06 -0300 Subject: [PATCH 45/69] Move env var definition --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 9b5149c1..c477157c 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -21,8 +21,8 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - PKG_CONFIG_PATH: "$(brew --prefix)/opt/libxml2/lib/pkgconfig" run: | + export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build rm -rf build/ - name: Set environment variables From 497971cfe6d0f79cbc54f59f6c4b93f740c27c53 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 25 Mar 2024 07:17:16 -0300 Subject: [PATCH 46/69] Remove PyXmlSec_CheckLibXmlLibraryVersion check --- src/lxml.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index bb7e6a96..6286cacb 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -47,18 +47,6 @@ long PyXmlSec_GetLibXmlCompiledVersionPatch() { return XMLSEC_EXTRACT_PATCH(LIBXML_VERSION); } -static int PyXmlSec_CheckLibXmlLibraryVersion(void) { - // Make sure that the version of libxml2 that we were compiled against is the same as the one - // that is loaded. If there is a version mismatch, we could run into segfaults. - - if (PyXmlSec_GetLibXmlVersionMajor() != PyXmlSec_GetLibXmlCompiledVersionMajor() || - PyXmlSec_GetLibXmlVersionMinor() != PyXmlSec_GetLibXmlCompiledVersionMinor()) { - return -1; - } - - return 0; -} - static int PyXmlSec_CheckLxmlLibraryVersion(void) { // Make sure that the version of libxml2 lxml is using is the same as the one we are using. Because // we pass trees between the two libraries, we need to make sure that they are using the same version @@ -104,11 +92,6 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { } int PyXmlSec_InitLxmlModule(void) { - if (PyXmlSec_CheckLibXmlLibraryVersion() < 0) { - PyXmlSec_SetLastError("xmlsec libxml2 library compiled version vs runtime version mismatch"); - return -1; - } - if (PyXmlSec_CheckLxmlLibraryVersion() < 0) { PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); return -1; From c7e3208822245f5a3cc1dd04cdeb11125be3945f Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 25 Mar 2024 11:08:36 -0300 Subject: [PATCH 47/69] Add type hints --- src/xmlsec/__init__.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 6c326f56..9cfc8cc6 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -13,6 +13,8 @@ from xmlsec.constants import __Transform as Transform _E = TypeVar('_E', bound=_Element) def enable_debug_trace(enabled: bool = ...) -> None: ... +def get_libxml_version() -> tuple[int, int, int]: ... +def get_libxml_compiled_version() -> tuple[int, int, int]: ... def init() -> None: ... def shutdown() -> None: ... def cleanup_callbacks() -> None: ... From e7cbb7c8c40698a7b75c57cd59c47d356c2485e7 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 27 Mar 2024 09:36:17 -0300 Subject: [PATCH 48/69] Update checks based on some code review feedback. --- src/lxml.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 6286cacb..7be62dad 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -67,18 +67,31 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { if (version == NULL) { goto FINALIZE; } - if (!PyTuple_Check(version) || PyTuple_Size(version) != 3) { + if (!PyTuple_Check(version) || PyTuple_Size(version) < 2) { goto FINALIZE; } PyObject* major = PyTuple_GetItem(version, 0); PyObject* minor = PyTuple_GetItem(version, 1); + if (PyErr_Occurred()) { + goto FINALIZE; + } + if (!PyLong_Check(major) || !PyLong_Check(minor)) { goto FINALIZE; } - if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { + long lxml_major = PyLong_AsLong(major); + long lxml_minor = PyLong_AsLong(minor); + long xmlsec_major = PyXmlSec_GetLibXmlVersionMajor(); + long xmlsec_minor = PyXmlSec_GetLibXmlVersionMinor(); + + if (PyErr_Occurred()) { + goto FINALIZE; + } + + if (lxml_major != xmlsec_major || lxml_minor != xmlsec_minor) { goto FINALIZE; } @@ -88,6 +101,10 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { // Cleanup our references, and return the result Py_XDECREF(lxml); Py_XDECREF(version); + + // Clear any errors that may have occurred + PyErr_Clear(); + return result; } From 802904edd0edb09cbfab854a527622c4d3019ff1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 28 Mar 2024 09:36:20 -0300 Subject: [PATCH 49/69] Make sure exception is handled when getting tuple item Co-authored-by: scoder --- src/lxml.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 7be62dad..f5b74b6a 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -72,9 +72,11 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { } PyObject* major = PyTuple_GetItem(version, 0); + if (major == NULL) { + goto FINALIZE; + } PyObject* minor = PyTuple_GetItem(version, 1); - - if (PyErr_Occurred()) { + if (minor == NULL) { goto FINALIZE; } From 8ca3bdca7b825a92bbdcbffec643429266891b34 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 28 Mar 2024 09:37:24 -0300 Subject: [PATCH 50/69] Revert changes to version comparison Co-authored-by: scoder --- src/lxml.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index f5b74b6a..02211a3a 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -84,16 +84,7 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { goto FINALIZE; } - long lxml_major = PyLong_AsLong(major); - long lxml_minor = PyLong_AsLong(minor); - long xmlsec_major = PyXmlSec_GetLibXmlVersionMajor(); - long xmlsec_minor = PyXmlSec_GetLibXmlVersionMinor(); - - if (PyErr_Occurred()) { - goto FINALIZE; - } - - if (lxml_major != xmlsec_major || lxml_minor != xmlsec_minor) { + if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { goto FINALIZE; } From b184cfe8f6b1a9592ff3e58aa2a93067d49bd5d4 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 28 Mar 2024 09:37:59 -0300 Subject: [PATCH 51/69] Clear any potential exceptions before Py_XDECREF Co-authored-by: scoder --- src/lxml.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 02211a3a..c98e933b 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -91,13 +91,13 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { result = 0; FINALIZE: + // Clear any errors that may have occurred + PyErr_Clear(); + // Cleanup our references, and return the result Py_XDECREF(lxml); Py_XDECREF(version); - // Clear any errors that may have occurred - PyErr_Clear(); - return result; } From 06f6e46ff1600bae93965bad3ca8f75c52ab41d5 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 15:11:05 +0200 Subject: [PATCH 52/69] Add more manylinux distros to workflow (#301) --- .github/workflows/manylinux.yml | 9 ++++++++- setup.py | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 6d2dbb1a..f620ba44 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -1,13 +1,20 @@ name: manylinux on: [push, pull_request] jobs: - pep513: + manylinux: runs-on: ubuntu-latest + env: + # python-xmlsec is not compatible with xmlsec1 v1.3.3. + # So we need to use the rc until the next release. + # TODO: Remove it when xmlsec1 v1.3.4 is fully released. + PYXMLSEC_XMLSEC1_VERSION: "1.3.4-rc1" strategy: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2014_x86_64 + - manylinux_2_28_x86_64 + - musllinux_1_1_x86_64 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/setup.py b/setup.py index 277a8892..2d8347f7 100644 --- a/setup.py +++ b/setup.py @@ -238,7 +238,7 @@ def prepare_static_build_win(self): ext.include_dirs = [str(p.absolute()) for p in includes] def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1q') + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1t') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') @@ -391,7 +391,6 @@ def prepare_static_build_linux(self): prefix_arg, '--disable-dependency-tracking', '--disable-shared', - '--enable-rebuild-docs=no', '--without-lzma', '--without-python', '--with-iconv={}'.format(self.prefix_dir), From d48d165f1555b81615f3e43e51ff2e5cd8e7d831 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 16:25:38 +0200 Subject: [PATCH 53/69] Fix pre-commit check (#302) --- tests/softhsm_setup.py | 108 +++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 59 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 432d4b1b..c2a805db 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -1,7 +1,7 @@ -""" -Testing the PKCS#11 shim layer. +"""Testing the PKCS#11 shim layer. + Heavily inspired by from https://github.com/IdentityPython/pyXMLSecurity by leifj -under licence "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt +under license "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt """ import logging @@ -13,7 +13,7 @@ import unittest from typing import Dict, List, Optional, Tuple -DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') def paths_for_component(component: str, default_paths: List[str]): @@ -25,7 +25,7 @@ def find_alts(component_name, alts: List[str]) -> str: for a in alts: if os.path.exists(a): return a - raise unittest.SkipTest("Required component is missing: {}".format(component_name)) + raise unittest.SkipTest('Required component is missing: {}'.format(component_name)) def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: @@ -45,7 +45,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: conf = f.read() msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' msg = msg.format( - cmd=" ".join(args), + cmd=' '.join(args), code=rv, out=out.strip(), err=err.strip(), @@ -113,36 +113,34 @@ def _temp_dir() -> str: return d -@unittest.skipIf(component_path['P11_MODULE'] is None, "SoftHSM PKCS11 module not installed") +@unittest.skipIf(component_path['P11_MODULE'] is None, 'SoftHSM PKCS11 module not installed') def setup() -> None: - logging.debug("Creating test pkcs11 token using softhsm") + logging.debug('Creating test pkcs11 token using softhsm') try: global softhsm_conf softhsm_conf = _temp_file() - logging.debug("Generating softhsm.conf") - with open(softhsm_conf, "w") as f: + logging.debug('Generating softhsm.conf') + with open(softhsm_conf, 'w') as f: if softhsm_version == 2: softhsm_db = _temp_dir() f.write( - """ + f""" # Generated by test -directories.tokendir = %s +directories.tokendir = {softhsm_db} objectstore.backend = file log.level = DEBUG """ - % softhsm_db ) else: softhsm_db = _temp_file() f.write( - """ + f""" # Generated by test -0:%s +0:{softhsm_db} """ - % softhsm_db ) - logging.debug("Initializing the token") + logging.debug('Initializing the token') out, err = run_cmd( [ component_path['SOFTHSM'], @@ -159,18 +157,8 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - # logging.debug("Generating 1024 bit RSA key in token") - # run_cmd([component_path['PKCS11_TOOL'], - # '--module', component_path['P11_MODULE'], - # '-l', - # '-k', - # '--key-type', 'rsa:1024', - # '--id', 'a1b2', - # '--label', 'test', - # '--pin', 'secret1'], softhsm_conf=softhsm_conf) - hash_priv_key = _temp_file() - logging.debug("Converting test private key to format for softhsm") + logging.debug('Converting test private key to format for softhsm') run_cmd( [ component_path['OPENSSL'], @@ -189,7 +177,7 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - logging.debug("Importing the test key to softhsm") + logging.debug('Importing the test key to softhsm') run_cmd( [ component_path['SOFTHSM'], @@ -207,40 +195,42 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) run_cmd( - [component_path['PKCS11_TOOL'], '--module', component_path['P11_MODULE'], '-l', '--pin', 'secret1', '-O'], + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--pin', + 'secret1', + '-O', + ], softhsm_conf=softhsm_conf, ) signer_cert_pem = _temp_file() openssl_conf = _temp_file() - logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) - with open(openssl_conf, "w") as f: - # Might be needed with some versions of openssl, but in more recent versions dynamic_path breaks it. - # dynamic_path = ( - # "dynamic_path = %s" % component_path['P11_ENGINE'] - # if openssl_version.startswith(b'1.') - # else "" - # ) + logging.debug('Generating OpenSSL config for version %s', openssl_version) + with open(openssl_conf, 'w') as f: f.write( - "\n".join( + '\n'.join( [ - "openssl_conf = openssl_def", - "[openssl_def]", - "engines = engine_section", - "[engine_section]", - "pkcs11 = pkcs11_section", - "[req]", - "distinguished_name = req_distinguished_name", - "[req_distinguished_name]", - "[pkcs11_section]", - "engine_id = pkcs11", + 'openssl_conf = openssl_def', + '[openssl_def]', + 'engines = engine_section', + '[engine_section]', + 'pkcs11 = pkcs11_section', + '[req]', + 'distinguished_name = req_distinguished_name', + '[req_distinguished_name]', + '[pkcs11_section]', + 'engine_id = pkcs11', # dynamic_path, - "MODULE_PATH = %s" % component_path['P11_MODULE'], - "init = 0", + f"MODULE_PATH = {component_path['P11_MODULE']}", + 'init = 0', ] ) ) - with open(openssl_conf, "r") as f: + with open(openssl_conf, 'r') as f: logging.debug('-------- START DEBUG openssl_conf --------') logging.debug(f.readlines()) logging.debug('-------- END DEBUG openssl_conf --------') @@ -251,7 +241,7 @@ def setup() -> None: signer_cert_der = _temp_file() - logging.debug("Generating self-signed certificate") + logging.debug('Generating self-signed certificate') run_cmd( [ component_path['OPENSSL'], @@ -259,7 +249,7 @@ def setup() -> None: '-new', '-x509', '-subj', - "/CN=Test Signer", + '/CN=Test Signer', '-engine', 'pkcs11', '-config', @@ -292,7 +282,7 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - logging.debug("Importing certificate into token") + logging.debug('Importing certificate into token') run_cmd( [ @@ -316,15 +306,15 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - # TODO: Should be teardowned in teardown: + # TODO: Should be teardowned in teardown # noqa: T101 os.environ['SOFTHSM_CONF'] = softhsm_conf os.environ['SOFTHSM2_CONF'] = softhsm_conf except Exception as ex: - print("-" * 64) + print('-' * 64) traceback.print_exc() - print("-" * 64) - logging.error("PKCS11 tests disabled: unable to initialize test token: %s" % ex) + print('-' * 64) + logging.exception('PKCS11 tests disabled: unable to initialize test token') raise ex From 64d9bc1e2b23cffbf1dbcde23162b1ec3734e6a0 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 17:14:15 +0200 Subject: [PATCH 54/69] Remove unsupported Python 3.5 type hints (#303) Tuple, Dict, List and Optional not supported in Python 3.5, so we can't use them. Using them breaks MacOS workflows. --- tests/softhsm_setup.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index c2a805db..ef34b6c0 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -11,24 +11,23 @@ import tempfile import traceback import unittest -from typing import Dict, List, Optional, Tuple DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -def paths_for_component(component: str, default_paths: List[str]): +def paths_for_component(component: str, default_paths): env_path = os.environ.get(component) return [env_path] if env_path else default_paths -def find_alts(component_name, alts: List[str]) -> str: +def find_alts(component_name, alts) -> str: for a in alts: if os.path.exists(a): return a raise unittest.SkipTest('Required component is missing: {}'.format(component_name)) -def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: +def run_cmd(args, softhsm_conf=None): env = {} if softhsm_conf is not None: env['SOFTHSM_CONF'] = softhsm_conf @@ -55,7 +54,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: return out, err -component_default_paths: Dict[str, List[str]] = { +component_default_paths = { 'P11_MODULE': [ '/usr/lib/softhsm/libsofthsm2.so', '/usr/lib/x86_64-linux-gnu/softhsm/libsofthsm2.so', @@ -85,7 +84,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: ], } -component_path: Dict[str, str] = { +component_path = { component_name: find_alts(component_name, paths_for_component(component_name, default_paths)) for component_name, default_paths in component_default_paths.items() } @@ -96,9 +95,9 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: openssl_version = subprocess.check_output([component_path['OPENSSL'], 'version'])[8:11].decode() -p11_test_files: List[str] = [] -softhsm_conf: Optional[str] = None -softhsm_db: Optional[str] = None +p11_test_files = [] +softhsm_conf = None +softhsm_db = None def _temp_file() -> str: From 2a0438684f90979a7dc7c36fe4a7b31e30e688b7 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 17:45:41 +0200 Subject: [PATCH 55/69] Use old formatting to keep it compatible with Python 3.5 (#304) --- tests/softhsm_setup.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index ef34b6c0..247f1b18 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -123,20 +123,24 @@ def setup() -> None: if softhsm_version == 2: softhsm_db = _temp_dir() f.write( - f""" + """ # Generated by test -directories.tokendir = {softhsm_db} +directories.tokendir = {} objectstore.backend = file log.level = DEBUG -""" +""".format( + softhsm_db + ) ) else: softhsm_db = _temp_file() f.write( - f""" + """ # Generated by test -0:{softhsm_db} -""" +0:{} +""".format( + softhsm_db + ) ) logging.debug('Initializing the token') @@ -223,7 +227,7 @@ def setup() -> None: '[pkcs11_section]', 'engine_id = pkcs11', # dynamic_path, - f"MODULE_PATH = {component_path['P11_MODULE']}", + "MODULE_PATH = {}".format(component_path['P11_MODULE']), 'init = 0', ] ) From df37761539d0dd35ab428fbb4776e19591b2cc78 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 20:04:23 +0000 Subject: [PATCH 56/69] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 22.8.0 → 24.3.0](https://github.com/psf/black/compare/22.8.0...24.3.0) - [github.com/pre-commit/pre-commit-hooks: v4.3.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.3.0...v4.5.0) - [github.com/PyCQA/flake8: 5.0.4 → 7.0.0](https://github.com/PyCQA/flake8/compare/5.0.4...7.0.0) - [github.com/PyCQA/isort: 5.12.0 → 5.13.2](https://github.com/PyCQA/isort/compare/5.12.0...5.13.2) - [github.com/pre-commit/mirrors-mypy: v0.981 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v0.981...v1.9.0) - [github.com/pre-commit/pygrep-hooks: v1.9.0 → v1.10.0](https://github.com/pre-commit/pygrep-hooks/compare/v1.9.0...v1.10.0) --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf877f39..9678ec68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/psf/black - rev: 22.8.0 + rev: 24.3.0 hooks: - id: black types: [] files: ^.*.pyi?$ exclude: ^doc/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace @@ -25,17 +25,17 @@ repos: - id: pretty-format-json args: [--autofix] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.0.0 hooks: - id: flake8 exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.981 + rev: v1.9.0 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) @@ -43,6 +43,6 @@ repos: files: ^.*.pyi?$ additional_dependencies: [lxml-stubs,types-docutils] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 + rev: v1.10.0 hooks: - id: rst-backticks From d5367145229dd316cc46f0df9e944dd28627bd56 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 20:04:38 +0000 Subject: [PATCH 57/69] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d8347f7..f0da1a2d 100644 --- a/setup.py +++ b/setup.py @@ -133,7 +133,7 @@ def run(self): [('MODULE_NAME', self.distribution.metadata.name), ('MODULE_VERSION', self.distribution.metadata.version)] ) # escape the XMLSEC_CRYPTO macro value, see mehcode/python-xmlsec#141 - for (key, value) in ext.define_macros: + for key, value in ext.define_macros: if key == 'XMLSEC_CRYPTO' and not (value.startswith('"') and value.endswith('"')): ext.define_macros.remove((key, value)) ext.define_macros.append((key, '"{0}"'.format(value))) From 18f325019d4ac2ecfe32dba6b6815a58f6fd7988 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 23:51:22 +0200 Subject: [PATCH 58/69] Fix opensuse-tumbleweed workflow (#305) --- .github/workflows/opensuse-tumbleweed.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index 273f7f7f..2f4caf49 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -11,9 +11,13 @@ jobs: - uses: actions/checkout@v1 - name: Install build dependencies run: | - zypper -n install -t pattern devel_basis + zypper refresh + zypper update + # The follwoing installs "devel_basis" pattern since installing the pattern fails because of few + # incompatibilty issues among packages + zypper -n install autoconf automake binutils bison cpp cpp13 flex gawk gcc gcc13 gdbm-devel gettext-runtime gettext-tools glibc-devel info kbd kbd-legacy libapparmor1 libasan8 libatomic1 libctf-nobfd0 libctf0 libdb-4_8 libfl-devel libfl2 libgdbm6 libgdbm_compat4 libgomp1 libhwasan0 libisl23 libitm1 libkmod2 liblsan0 libltdl7 libmpc3 libmpfr6 libseccomp2 libtextstyle0 libtool libtsan2 libubsan1 libxcrypt-devel libzio1 linux-glibc-devel m4 make makeinfo ncurses-devel pam-config patch perl perl-Text-Unidecode perl-base purge-kernels-service system-user-nobody systemd systemd-default-settings systemd-default-settings-branding-openSUSE systemd-presets-branding-openSUSE systemd-presets-common-SUSE tack update-alternatives zlib-devel PKGVER_NO_DOT=$(tr -d '.' <<< ${{ matrix.python-version }}) - zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${PKGVER_NO_DOT}-pip + zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${{ matrix.python-version }} -m venv .venv .venv/bin/python -m pip install --upgrade pip setuptools wheel - name: Build linux_x86_64 wheel @@ -22,7 +26,7 @@ jobs: rm -rf build/ - name: Install test dependencies run: | - .venv/bin/python -m pip install --upgrade -r requirements-test.txt + .venv/bin/python -m pip install --upgrade --no-binary=lxml -r requirements-test.txt .venv/bin/python -m pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ - name: Run tests run: | From 83ee6324c6e1aaeafef96a43a6584e3991fa2272 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 20:18:14 -0300 Subject: [PATCH 59/69] Fetch latest openssl version, when doing static build. --- .github/workflows/manylinux.yml | 6 ++- setup.py | 83 +++++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index f620ba44..ea05780b 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -18,11 +18,15 @@ jobs: container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 - - name: Install build dependencies + - name: Install python build dependencies run: | # https://github.com/actions/runner/issues/2033 chown -R $(id -u):$(id -g) $PWD /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build + - name: Install system build dependencies (manylinux) + run: | + yum install -y perl-core + if: contains(matrix.image, 'manylinux') - name: Set environment variables shell: bash run: | diff --git a/setup.py b/setup.py index f0da1a2d..32c4d134 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,8 @@ from distutils.errors import DistutilsError from distutils.version import StrictVersion as Version from pathlib import Path -from urllib.request import urlcleanup, urljoin, urlopen, urlretrieve +from urllib.parse import urljoin +from urllib.request import urlcleanup, urlopen, urlretrieve, Request from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig @@ -31,31 +32,60 @@ def handle_starttag(self, tag, attrs): self.hrefs.append(value) +def make_request(url, github_token=None, json_response=False): + headers = {'User-Agent': 'https://github.com/xmlsec/python-xmlsec'} + if github_token: + headers['authorization'] = "Bearer " + github_token + request = Request(url, headers=headers) + with contextlib.closing(urlopen(request)) as r: + if json_response: + return json.load(r) + else: + charset = r.headers.get_content_charset() or 'utf-8' + content = r.read().decode(charset) + return content + + def latest_release_from_html(url, matcher): - with contextlib.closing(urlopen(url)) as r: - charset = r.headers.get_content_charset() or 'utf-8' - content = r.read().decode(charset) - collector = HrefCollector() - collector.feed(content) - hrefs = collector.hrefs - - def comp(text): - try: - return Version(matcher.match(text).groupdict()['version']) - except (AttributeError, ValueError): - return Version('0.0') + content = make_request(url) + collector = HrefCollector() + collector.feed(content) + hrefs = collector.hrefs + + def comp(text): + try: + return Version(matcher.match(text).groupdict()['version']) + except (AttributeError, ValueError): + return Version('0.0') - latest = max(hrefs, key=comp) - return '{}/{}'.format(url, latest) + latest = max(hrefs, key=comp) + return '{}/{}'.format(url, latest) def latest_release_from_gnome_org_cache(url, lib_name): cache_url = '{}/cache.json'.format(url) - with contextlib.closing(urlopen(cache_url)) as r: - cache = json.load(r) - latest_version = cache[2][lib_name][-1] - latest_source = cache[1][lib_name][latest_version]['tar.xz'] - return '{}/{}'.format(url, latest_source) + cache = make_request(cache_url, json_response=True) + latest_version = cache[2][lib_name][-1] + latest_source = cache[1][lib_name][latest_version]['tar.xz'] + return '{}/{}'.format(url, latest_source) + + +def latest_release_from_github_api(repo): + api_url = 'https://api.github.com/repos/{}/releases'.format(repo) + + # if we are running in CI, pass along the GH_TOKEN, so we don't get rate limited + token = os.environ.get("GH_TOKEN") + if token: + log.info("Using GitHub token to avoid rate limiting") + api_releases = make_request(api_url, token, json_response=True) + releases = [r['tarball_url'] for r in api_releases if r['prerelease'] is False and r['draft'] is False] + if not releases: + raise DistutilsError('No release found for {}'.format(repo)) + return releases[0] + + +def latest_openssl_release(): + return latest_release_from_github_api('openssl/openssl') def latest_zlib_release(): @@ -238,7 +268,7 @@ def prepare_static_build_win(self): ext.include_dirs = [str(p.absolute()) for p in includes] def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1t') + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') @@ -250,8 +280,13 @@ def prepare_static_build_linux(self): if openssl_tar is None: self.info('{:10}: {}'.format('OpenSSL', 'source tar not found, downloading ...')) openssl_tar = self.libs_dir / 'openssl.tar.gz' - self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) - urlretrieve('https://www.openssl.org/source/openssl-{}.tar.gz'.format(self.openssl_version), str(openssl_tar)) + if self.openssl_version is None: + url = latest_openssl_release() + self.info('{:10}: {}'.format('OpenSSL', 'PYXMLSEC_OPENSSL_VERSION unset, downloading latest from {}'.format(url))) + else: + url = 'https://api.github.com/repos/openssl/openssl/tarball/openssl-{}'.format(self.openssl_version) + self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) + urlretrieve(url, str(openssl_tar)) # fetch zlib zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) @@ -361,7 +396,7 @@ def prepare_static_build_linux(self): self.info('Building OpenSSL') openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC'], cwd=str(openssl_dir), env=env) + subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'], cwd=str(openssl_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) subprocess.check_output( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env From 73cca3cabf203b6660ce3396621047789bf77d94 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:23:29 +0000 Subject: [PATCH 60/69] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 32c4d134..cdf95c41 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from distutils.version import StrictVersion as Version from pathlib import Path from urllib.parse import urljoin -from urllib.request import urlcleanup, urlopen, urlretrieve, Request +from urllib.request import Request, urlcleanup, urlopen, urlretrieve from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig From a51d1ff1ef0b1bb98df97c91bfc0f4504392ff9c Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:02:39 -0300 Subject: [PATCH 61/69] Allow static build from MacOS --- setup.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index cdf95c41..7b5d43d6 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,17 @@ def latest_xmlsec_release(): return latest_release_from_html('https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz')) +class CrossCompileInfo: + def __init__(self, host, arch, compiler): + self.host = host + self.arch = arch + self.compiler = compiler + + @property + def triplet(self): + return "{}-{}-{}".format(self.host, self.arch, self.compiler) + + class build_ext(build_ext_orig): def info(self, message): self.announce(message, level=log.INFO) @@ -136,7 +147,9 @@ def run(self): if sys.platform == 'win32': self.prepare_static_build_win() elif 'linux' in sys.platform: - self.prepare_static_build_linux() + self.prepare_static_build(sys.platform) + elif 'darwin' in sys.platform: + self.prepare_static_build(sys.platform) else: import pkgconfig @@ -267,7 +280,7 @@ def prepare_static_build_win(self): includes.append(next(p / 'xmlsec' for p in includes if (p / 'xmlsec').is_dir())) ext.include_dirs = [str(p.absolute()) for p in includes] - def prepare_static_build_linux(self): + def prepare_static_build(self, build_platform): self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') @@ -387,16 +400,42 @@ def prepare_static_build_linux(self): prefix_arg = '--prefix={}'.format(self.prefix_dir) - cflags = ['-fPIC'] env = os.environ.copy() - if 'CFLAGS' in env: - env['CFLAGS'].append(' '.join(cflags)) - else: - env['CFLAGS'] = ' '.join(cflags) + cflags = [] + if env.get('CFLAGS'): + cflags.append(env['CFLAGS']) + cflags.append('-fPIC') + ldflags = [] + if env.get('LDFLAGS'): + ldflags.append(env['LDFLAGS']) + + cross_compiling = False + if build_platform == 'darwin': + import platform + + arch = self.plat_name.rsplit('-', 1)[1] + if arch != platform.machine() and arch in ('x86_64', 'arm64'): + self.info('Cross-compiling for {}'.format(arch)) + cflags.append('-arch {}'.format(arch)) + ldflags.append('-arch {}'.format(arch)) + cross_compiling = CrossCompileInfo('darwin64', arch, 'cc') + major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) + if major_version >= 11: + if 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = "11.0" + + env['CFLAGS'] = ' '.join(cflags) + env['LDFLAGS'] = ' '.join(ldflags) self.info('Building OpenSSL') openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'], cwd=str(openssl_dir), env=env) + openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] + if cross_compiling: + openssl_config_cmd.insert(0, './Configure') + openssl_config_cmd.append(cross_compiling.triplet) + else: + openssl_config_cmd.insert(0, './config') + subprocess.check_output(openssl_config_cmd, cwd=str(openssl_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) subprocess.check_output( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env @@ -408,10 +447,22 @@ def prepare_static_build_linux(self): subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) + host_arg = "" + if cross_compiling: + host_arg = '--host={}'.format(cross_compiling.arch) + self.info('Building libiconv') libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) subprocess.check_output( - ['./configure', prefix_arg, '--disable-dependency-tracking', '--disable-shared'], cwd=str(libiconv_dir), env=env + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + host_arg, + ], + cwd=str(libiconv_dir), + env=env, ) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) subprocess.check_output( @@ -430,6 +481,7 @@ def prepare_static_build_linux(self): '--without-python', '--with-iconv={}'.format(self.prefix_dir), '--with-zlib={}'.format(self.prefix_dir), + host_arg, ], cwd=str(libxml2_dir), env=env, @@ -450,6 +502,7 @@ def prepare_static_build_linux(self): '--without-python', '--without-crypto', '--with-libxml-prefix={}'.format(self.prefix_dir), + host_arg, ], cwd=str(libxslt_dir), env=env, @@ -460,10 +513,8 @@ def prepare_static_build_linux(self): ) self.info('Building xmlsec1') - if 'LDFLAGS' in env: - env['LDFLAGS'].append(' -lpthread') - else: - env['LDFLAGS'] = '-lpthread' + ldflags.append('-lpthread') + env['LDFLAGS'] = ' '.join(ldflags) xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) subprocess.check_output( [ @@ -480,6 +531,7 @@ def prepare_static_build_linux(self): '--with-openssl={}'.format(self.prefix_dir), '--with-libxml={}'.format(self.prefix_dir), '--with-libxslt={}'.format(self.prefix_dir), + host_arg, ], cwd=str(xmlsec1_dir), env=env, @@ -517,7 +569,8 @@ def prepare_static_build_linux(self): ext.include_dirs.extend([str(p.absolute()) for p in (self.prefix_dir / 'include').iterdir() if p.is_dir()]) ext.library_dirs = [] - ext.libraries = ['m', 'rt'] + if build_platform == 'linux': + ext.libraries = ['m', 'rt'] extra_objects = [ 'libxmlsec1.a', 'libxslt.a', From 47e711b0317955f0f363985606f0f9fdf6939bea Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 7 Apr 2024 19:13:33 -0300 Subject: [PATCH 62/69] Add CI action for OSX static build --- .github/workflows/macosx.yml | 3 +++ setup.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index c477157c..5baa1e44 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -6,6 +6,7 @@ jobs: strategy: matrix: python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + static_deps: ["static", ""] steps: - uses: actions/checkout@v3 - name: Setup Python @@ -21,6 +22,7 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + PYXMLSEC_STATIC_DEPS: ${{ matrix.static_deps }} run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build @@ -43,3 +45,4 @@ jobs: /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=$(uname -m) --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt + if: matrix.static_deps != 'static' diff --git a/setup.py b/setup.py index 7b5d43d6..7e9e8ba1 100644 --- a/setup.py +++ b/setup.py @@ -38,11 +38,11 @@ def make_request(url, github_token=None, json_response=False): headers['authorization'] = "Bearer " + github_token request = Request(url, headers=headers) with contextlib.closing(urlopen(request)) as r: + charset = r.headers.get_content_charset() or 'utf-8' + content = r.read().decode(charset) if json_response: - return json.load(r) + return json.loads(content) else: - charset = r.headers.get_content_charset() or 'utf-8' - content = r.read().decode(charset) return content From 6922ccba43f544b90aaa782dfd7f8ff4f51a1103 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 20:18:07 +0000 Subject: [PATCH 63/69] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9678ec68..820778ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: files: ^.*.pyi?$ exclude: ^doc/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace From e9251d34b735caac7160735872d5deeb7910285d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 7 Apr 2024 21:56:55 -0300 Subject: [PATCH 64/69] Fix linux brew workflow --- .github/workflows/linuxbrew.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 191e2001..886bd8c9 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -3,26 +3,28 @@ on: [push, pull_request] jobs: linuxbrew: runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Install build dependencies + - name: Install brew run: | sudo apt install -y build-essential procps curl file git /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - test -d ~/.linuxbrew && eval $(~/.linuxbrew/bin/brew shellenv) - test -r ~/.bash_profile && echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.bash_profile - echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.profile + echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH + - name: Install build dependencies + run: | brew update - brew install python gcc libxml2 libxmlsec1 pkg-config + brew install python@${{ matrix.python }} gcc libxml2 libxmlsec1 pkg-config + echo "/home/linuxbrew/.linuxbrew/opt/python@${{ matrix.python }}/libexec/bin" >> $GITHUB_PATH + - name: Install python dependencies + run: | pip3 install --upgrade setuptools wheel build - ln -s $(brew --prefix)/bin/gcc-12 $(brew --prefix)/bin/gcc-5 - ls -l $(brew --prefix)/bin/gcc* - name: Build linux_x86_64 wheel run: | + export CFLAGS="-I$(brew --prefix)/include" + export LDFLAGS="-L$(brew --prefix)/lib" python3 -m build rm -rf build/ - name: Install test dependencies From e789da55100ab1ce96d2f9d0d798d5a67b16b74d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 8 Apr 2024 21:59:40 -0300 Subject: [PATCH 65/69] Add gh_token to avoid getting rate limited --- .github/workflows/macosx.yml | 1 + .github/workflows/manylinux.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 5baa1e44..6d0548e8 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -23,6 +23,7 @@ jobs: CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" PYXMLSEC_STATIC_DEPS: ${{ matrix.static_deps }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index ea05780b..a80ad67b 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -34,6 +34,7 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build - name: Label manylinux wheel From b864c31c2c887e7ddf4fdc6c9faa2bfd20ef6d64 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:12:14 -0300 Subject: [PATCH 66/69] Use ci build wheel to build staticly linked wheels for linux and OSX --- .github/workflows/wheels.yml | 122 +++++++++++++++++++++++++++++++++++ pyproject.toml | 28 ++++++++ 2 files changed, 150 insertions(+) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000..0d83f59b --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,122 @@ +name: Wheel build + +on: + release: + types: [created] + schedule: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + - cron: "42 3 * * 4" + push: + pull_request: + workflow_dispatch: + +permissions: {} + +jobs: + sdist: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: "3.x" + + - name: Install build dependencies + run: | + pip install --upgrade pip setuptools wheel + + - name: Package source dist + run: python setup.py sdist + + - name: Install test dependencies + run: | + sudo apt-get update -y -q + sudo apt-get install -y -q libxml2-dev libxslt1-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl + pip install --upgrade -r requirements-test.txt --no-binary lxml + pip install dist/xmlsec-$(python setup.py --version).tar.gz + + - name: Run tests + run: pytest -v --color=yes + + - name: Upload sdist + uses: actions/upload-artifact@v4.3.1 + with: + name: sdist + path: dist/*.tar.gz + + generate-wheels-matrix: + # Create a matrix of all architectures & versions to build. + # This enables the next step to run cibuildwheel in parallel. + # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 + name: Generate wheels matrix + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v4 + - name: Install cibuildwheel + # Nb. keep cibuildwheel version pin consistent with job below + run: pipx install cibuildwheel==2.16.5 + - id: set-matrix + # Once we have the windows build figured out, it can be added here + # by updating the matrix to include windows builds as well. + # See example here: + # https://github.com/lxml/lxml/blob/3ccc7d583e325ceb0ebdf8fc295bbb7fc8cd404d/.github/workflows/wheels.yml#L95C1-L106C51 + run: | + MATRIX=$( + { + cibuildwheel --print-build-identifiers --platform linux \ + | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos \ + | jq -nRc '{"only": inputs, "os": "macos-latest"}' + } | jq -sc + ) + echo "include=$MATRIX" + echo "include=$MATRIX" >> $GITHUB_OUTPUT + + build_wheels: + name: Build for ${{ matrix.only }} + needs: generate-wheels-matrix + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} + + steps: + - name: Check out the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + with: + only: ${{ matrix.only }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/upload-artifact@v4.3.1 + with: + path: ./wheelhouse/*.whl + name: xmlsec-wheel-${{ matrix.only }} diff --git a/pyproject.toml b/pyproject.toml index e28878e3..f3ddbdbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,3 +45,31 @@ known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] [build-system] requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] + +[tool.cibuildwheel] +build-verbosity = 1 +build-frontend = "build" +skip = ["pp*", "*-musllinux_i686"] +test-command = "pytest -v --color=yes {package}/tests" +before-test = "pip install -r requirements-test.txt" +test-skip = "*-macosx_arm64" + +[tool.cibuildwheel.environment] +PYXMLSEC_STATIC_DEPS = "true" + +[tool.cibuildwheel.linux] +archs = ["x86_64", "aarch64", "i686"] +environment-pass = [ + "PYXMLSEC_LIBXML2_VERSION", + "PYXMLSEC_LIBXSLT_VERSION", + "PYXMLSEC_STATIC_DEPS", + "GH_TOKEN" +] + +[tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] +before-all = "brew install perl" + +[[tool.cibuildwheel.overrides]] +select = "*-manylinux*" +before-all = "yum install -y perl-core" From 6741ff80cf0df8c8e9e611baaf77f4c2c68321a1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Tue, 9 Apr 2024 16:35:03 -0300 Subject: [PATCH 67/69] Make skipped wheel builds match upstream lxml. --- pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f3ddbdbb..99927b4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,16 @@ requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>= [tool.cibuildwheel] build-verbosity = 1 build-frontend = "build" -skip = ["pp*", "*-musllinux_i686"] +skip = [ + "pp*", + "*-musllinux_i686", + # LXML doesn't publish wheels for these platforms, which makes it + # difficult for us to build wheels, so we exclude them. + "cp36-manylinux_aarch64", + "cp37-manylinux_aarch64", + "cp36-musllinux_aarch64", + "cp37-musllinux_aarch64", +] test-command = "pytest -v --color=yes {package}/tests" before-test = "pip install -r requirements-test.txt" test-skip = "*-macosx_arm64" From 3edf947c05c5e853dc77cc66c8dcc0b19735c033 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Tue, 9 Apr 2024 16:47:00 -0300 Subject: [PATCH 68/69] Remove PYXMLSEC_XMLSEC1_VERSION override. --- .github/workflows/manylinux.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index a80ad67b..a44776b3 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -3,11 +3,6 @@ on: [push, pull_request] jobs: manylinux: runs-on: ubuntu-latest - env: - # python-xmlsec is not compatible with xmlsec1 v1.3.3. - # So we need to use the rc until the next release. - # TODO: Remove it when xmlsec1 v1.3.4 is fully released. - PYXMLSEC_XMLSEC1_VERSION: "1.3.4-rc1" strategy: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] From 1b3b527fd27ea0f48ee3c69354249d90d56cec23 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Wed, 17 Apr 2024 18:16:58 +0200 Subject: [PATCH 69/69] Add windows wheel build (#313) --- .github/workflows/wheels.yml | 4 +++- pyproject.toml | 3 +++ setup.py | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0d83f59b..ccd62cf8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -81,7 +81,9 @@ jobs: cibuildwheel --print-build-identifiers --platform linux \ | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ && cibuildwheel --print-build-identifiers --platform macos \ - | jq -nRc '{"only": inputs, "os": "macos-latest"}' + | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows \ + | jq -nRc '{"only": inputs, "os": "windows-2019"}' } | jq -sc ) echo "include=$MATRIX" diff --git a/pyproject.toml b/pyproject.toml index 99927b4b..9b6469d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,9 @@ environment-pass = [ archs = ["x86_64", "arm64"] before-all = "brew install perl" +[tool.cibuildwheel.windows] +archs = ["AMD64", "x86"] + [[tool.cibuildwheel.overrides]] select = "*-manylinux*" before-all = "yum install -y perl-core" diff --git a/setup.py b/setup.py index 7e9e8ba1..92588ebc 100644 --- a/setup.py +++ b/setup.py @@ -213,19 +213,19 @@ def run(self): super(build_ext, self).run() def prepare_static_build_win(self): - release_url = 'https://github.com/bgaifullin/libxml2-win-binaries/releases/download/v2018.08/' - if sys.maxsize > 2147483647: + release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2024.04.17/' + if sys.maxsize > 2147483647: # 2.0 GiB suffix = 'win64' else: suffix = 'win32' libs = [ - 'libxml2-2.9.4.{}.zip'.format(suffix), - 'libxslt-1.1.29.{}.zip'.format(suffix), - 'zlib-1.2.8.{}.zip'.format(suffix), - 'iconv-1.14.{}.zip'.format(suffix), - 'openssl-1.0.1.{}.zip'.format(suffix), - 'xmlsec-1.2.24.{}.zip'.format(suffix), + 'libxml2-2.11.7.{}.zip'.format(suffix), + 'libxslt-1.1.37.{}.zip'.format(suffix), + 'zlib-1.2.12.{}.zip'.format(suffix), + 'iconv-1.16-1.{}.zip'.format(suffix), + 'openssl-3.0.8.{}.zip'.format(suffix), + 'xmlsec-1.3.4.{}.zip'.format(suffix), ] for libfile in libs: @@ -262,7 +262,7 @@ def prepare_static_build_win(self): ext.libraries = [ 'libxmlsec_a', 'libxmlsec-openssl_a', - 'libeay32', + 'libcrypto', 'iconv_a', 'libxslt_a', 'libexslt_a', @@ -599,6 +599,7 @@ def prepare_static_build(self, build_platform): use_scm_version=True, description='Python bindings for the XML Security Library', long_description=long_desc, + long_description_content_type='text/markdown', ext_modules=[pyxmlsec], cmdclass={'build_ext': build_ext}, python_requires='>=3.5',