From 413374412bece065af9f7c3f756744a0672beda1 Mon Sep 17 00:00:00 2001 From: Nikita Maruniak Date: Thu, 20 Jun 2024 18:01:09 +0300 Subject: [PATCH 1/6] Upgrade to Python 3.11 Remove `nose` because it is outdated and incompatible with newer versions of Python. Use `unittest` and `doctest` modules instead. --- Makefile | 7 +++--- setup.py | 1 - tests/test_catalog.py | 6 ++--- tests/test_exceptions.py | 12 +++++----- tests/test_schema.py | 26 +++++++++++----------- tests/test_transform.py | 48 ++++++++++++++++++++-------------------- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index 2fe943e..61ab98e 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,12 @@ check_prereqs: bash -c '[[ -n $$VIRTUAL_ENV ]]' - bash -c '[[ $$(python3 --version) == *3.5.2* ]]' + bash -c '[[ $$(python3 --version) == *3.11* ]]' install: check_prereqs python3 -m pip install -e '.[dev]' test: install - pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-except,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access - nosetests --with-doctest -v + pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-except,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access,broad-exception-raised,consider-using-f-string + python3 -m doctest -v singer/utils.py + python3 -m unittest -v diff --git a/setup.py b/setup.py index 91f3ed0..64e7aa0 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,6 @@ 'pylint', 'ipython', 'ipdb', - 'nose', 'singer-tools' ] }, diff --git a/tests/test_catalog.py b/tests/test_catalog.py index cd6dc50..8a72e1a 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -25,7 +25,7 @@ def test_one_selected_stream(self): CatalogEntry(tap_stream_id='c',schema=Schema(),metadata=[])]) state = {} selected_streams = catalog.get_selected_streams(state) - self.assertEquals([e for e in selected_streams],[selected_entry]) + self.assertEqual([e for e in selected_streams],[selected_entry]) def test_resumes_currently_syncing_stream(self): selected_entry_a = CatalogEntry(tap_stream_id='a', @@ -44,7 +44,7 @@ def test_resumes_currently_syncing_stream(self): selected_entry_c]) state = {'currently_syncing': 'c'} selected_streams = catalog.get_selected_streams(state) - self.assertEquals([e for e in selected_streams][0],selected_entry_c) + self.assertEqual([e for e in selected_streams][0],selected_entry_c) class TestToDictAndFromDict(unittest.TestCase): @@ -141,4 +141,4 @@ def test(self): CatalogEntry(tap_stream_id='b'), CatalogEntry(tap_stream_id='c')]) entry = catalog.get_stream('b') - self.assertEquals('b', entry.tap_stream_id) + self.assertEqual('b', entry.tap_stream_id) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 491595f..f58c6ef 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -14,7 +14,7 @@ def test_SingerError_prints_correctly(self): raise SingerError(error_text) expected_text = "SingerError\n" + error_text - self.assertEquals(expected_text, + self.assertEqual(expected_text, str(test_run.exception)) def test_SingerConfigurationError_prints_correctly(self): @@ -24,7 +24,7 @@ def test_SingerConfigurationError_prints_correctly(self): raise SingerConfigurationError(error_text) expected_text = "SingerConfigurationError\n" + error_text - self.assertEquals(expected_text, + self.assertEqual(expected_text, str(test_run.exception)) def test_SingerDiscoveryError_prints_correctly(self): @@ -34,7 +34,7 @@ def test_SingerDiscoveryError_prints_correctly(self): raise SingerDiscoveryError(error_text) expected_text = "SingerDiscoveryError\n" + error_text - self.assertEquals(expected_text, + self.assertEqual(expected_text, str(test_run.exception)) def test_SingerSyncError_prints_correctly(self): @@ -44,7 +44,7 @@ def test_SingerSyncError_prints_correctly(self): raise SingerSyncError(error_text) expected_text = "SingerSyncError\n" + error_text - self.assertEquals(expected_text, + self.assertEqual(expected_text, str(test_run.exception)) def test_SingerRetryableRequestError_prints_correctly(self): @@ -54,7 +54,7 @@ def test_SingerRetryableRequestError_prints_correctly(self): raise SingerRetryableRequestError(error_text) expected_text = "SingerRetryableRequestError\n" + error_text - self.assertEquals(expected_text, + self.assertEqual(expected_text, str(test_run.exception)) def test_SingerError_prints_multiple_lines_correctly(self): @@ -64,5 +64,5 @@ def test_SingerError_prints_multiple_lines_correctly(self): raise SingerError(error_text) expected_text = "SingerError\n" + error_text - self.assertEquals(expected_text, + self.assertEqual(expected_text, str(test_run.exception)) diff --git a/tests/test_schema.py b/tests/test_schema.py index fa28bac..908277e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -44,38 +44,38 @@ class TestSchema(unittest.TestCase): additionalProperties=True) def test_string_to_dict(self): - self.assertEquals(self.string_dict, self.string_obj.to_dict()) + self.assertEqual(self.string_dict, self.string_obj.to_dict()) def test_integer_to_dict(self): - self.assertEquals(self.integer_dict, self.integer_obj.to_dict()) + self.assertEqual(self.integer_dict, self.integer_obj.to_dict()) def test_array_to_dict(self): - self.assertEquals(self.array_dict, self.array_obj.to_dict()) + self.assertEqual(self.array_dict, self.array_obj.to_dict()) def test_object_to_dict(self): - self.assertEquals(self.object_dict, self.object_obj.to_dict()) + self.assertEqual(self.object_dict, self.object_obj.to_dict()) def test_string_from_dict(self): - self.assertEquals(self.string_obj, Schema.from_dict(self.string_dict)) + self.assertEqual(self.string_obj, Schema.from_dict(self.string_dict)) def test_integer_from_dict(self): - self.assertEquals(self.integer_obj, Schema.from_dict(self.integer_dict)) + self.assertEqual(self.integer_obj, Schema.from_dict(self.integer_dict)) def test_array_from_dict(self): - self.assertEquals(self.array_obj, Schema.from_dict(self.array_dict)) + self.assertEqual(self.array_obj, Schema.from_dict(self.array_dict)) def test_object_from_dict(self): - self.assertEquals(self.object_obj, Schema.from_dict(self.object_dict)) + self.assertEqual(self.object_obj, Schema.from_dict(self.object_dict)) def test_repr_atomic(self): - self.assertEquals(self.string_obj, eval(repr(self.string_obj))) + self.assertEqual(self.string_obj, eval(repr(self.string_obj))) def test_repr_recursive(self): - self.assertEquals(self.object_obj, eval(repr(self.object_obj))) + self.assertEqual(self.object_obj, eval(repr(self.object_obj))) def test_object_from_dict_with_defaults(self): schema = Schema.from_dict(self.object_dict, inclusion='automatic') - self.assertEquals('whatever', schema.inclusion, + self.assertEqual('whatever', schema.inclusion, msg='The schema value should override the default') - self.assertEquals('automatic', schema.properties['a_string'].inclusion) - self.assertEquals('automatic', schema.properties['an_array'].items.inclusion) + self.assertEqual('automatic', schema.properties['a_string'].inclusion) + self.assertEqual('automatic', schema.properties['an_array'].items.inclusion) diff --git a/tests/test_transform.py b/tests/test_transform.py index 959c4b8..96e0c3b 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -264,11 +264,11 @@ def test_decimal_types_transform(self): nan = {'percentage': decimal.Decimal('NaN')} snan = {'percentage': decimal.Decimal('sNaN')} - self.assertEquals(inf, transform(inf, schema)) - self.assertEquals(negative_inf, transform(negative_inf, schema)) - self.assertEquals({'percentage': '1.4142135623730951'}, transform(root2, schema)) - self.assertEquals({'percentage': 'NaN'}, transform(nan, schema)) - self.assertEquals({'percentage': 'NaN'}, transform(snan, schema)) + self.assertEqual(inf, transform(inf, schema)) + self.assertEqual(negative_inf, transform(negative_inf, schema)) + self.assertEqual({'percentage': '1.4142135623730951'}, transform(root2, schema)) + self.assertEqual({'percentage': 'NaN'}, transform(nan, schema)) + self.assertEqual({'percentage': 'NaN'}, transform(snan, schema)) str1 = {'percentage':'0.1'} @@ -276,11 +276,11 @@ def test_decimal_types_transform(self): str3 = {'percentage': '1E+13'} str4 = {'percentage': '100'} str5 = {'percentage': '-100'} - self.assertEquals(str1, transform(str1, schema)) - self.assertEquals({'percentage': '1E-13'}, transform(str2, schema)) - self.assertEquals({'percentage': '1E+13'}, transform(str3, schema)) - self.assertEquals({'percentage': '100'}, transform(str4, schema)) - self.assertEquals({'percentage': '-100'}, transform(str5, schema)) + self.assertEqual(str1, transform(str1, schema)) + self.assertEqual({'percentage': '1E-13'}, transform(str2, schema)) + self.assertEqual({'percentage': '1E+13'}, transform(str3, schema)) + self.assertEqual({'percentage': '100'}, transform(str4, schema)) + self.assertEqual({'percentage': '-100'}, transform(str5, schema)) float1 = {'percentage': 12.0000000000000000000000000001234556} float2 = {'percentage': 0.0123} @@ -288,28 +288,28 @@ def test_decimal_types_transform(self): float4 = {'percentage': -100.0123} float5 = {'percentage': 0.000001} float6 = {'percentage': 0.0000001} - self.assertEquals({'percentage':'12.0'}, transform(float1, schema)) - self.assertEquals({'percentage':'0.0123'}, transform(float2, schema)) - self.assertEquals({'percentage':'100.0123'}, transform(float3, schema)) - self.assertEquals({'percentage':'-100.0123'}, transform(float4, schema)) - self.assertEquals({'percentage':'0.000001'}, transform(float5, schema)) - self.assertEquals({'percentage':'1E-7'}, transform(float6, schema)) + self.assertEqual({'percentage':'12.0'}, transform(float1, schema)) + self.assertEqual({'percentage':'0.0123'}, transform(float2, schema)) + self.assertEqual({'percentage':'100.0123'}, transform(float3, schema)) + self.assertEqual({'percentage':'-100.0123'}, transform(float4, schema)) + self.assertEqual({'percentage':'0.000001'}, transform(float5, schema)) + self.assertEqual({'percentage':'1E-7'}, transform(float6, schema)) int1 = {'percentage': 123} int2 = {'percentage': 0} int3 = {'percentage': -1000} - self.assertEquals({'percentage':'123'}, transform(int1, schema)) - self.assertEquals({'percentage':'0'}, transform(int2, schema)) - self.assertEquals({'percentage':'-1000'}, transform(int3, schema)) + self.assertEqual({'percentage':'123'}, transform(int1, schema)) + self.assertEqual({'percentage':'0'}, transform(int2, schema)) + self.assertEqual({'percentage':'-1000'}, transform(int3, schema)) dec1 = {'percentage': decimal.Decimal('1.1010101')} dec2 = {'percentage': decimal.Decimal('.111111111111111111111111')} dec3 = {'percentage': decimal.Decimal('-.111111111111111111111111')} dec4 = {'percentage': decimal.Decimal('100')} - self.assertEquals({'percentage':'1.1010101'}, transform(dec1, schema)) - self.assertEquals({'percentage':'0.111111111111111111111111'}, transform(dec2, schema)) - self.assertEquals({'percentage':'-0.111111111111111111111111'}, transform(dec3, schema)) - self.assertEquals({'percentage':'100'}, transform(dec4, schema)) + self.assertEqual({'percentage':'1.1010101'}, transform(dec1, schema)) + self.assertEqual({'percentage':'0.111111111111111111111111'}, transform(dec2, schema)) + self.assertEqual({'percentage':'-0.111111111111111111111111'}, transform(dec3, schema)) + self.assertEqual({'percentage':'100'}, transform(dec4, schema)) bad1 = {'percentage': 'fsdkjl'} with self.assertRaises(SchemaMismatch): @@ -317,7 +317,7 @@ def test_decimal_types_transform(self): badnull = {'percentage': None} with self.assertRaises(SchemaMismatch): - self.assertEquals({'percentage':None}, transform(badnull, schema)) + self.assertEqual({'percentage':None}, transform(badnull, schema)) class TestTransformsWithMetadata(unittest.TestCase): From b304d4e82f9af006cbca09e5a6a4b5b132d4d9e4 Mon Sep 17 00:00:00 2001 From: Nikita Maruniak Date: Thu, 20 Jun 2024 18:52:19 +0300 Subject: [PATCH 2/6] Upgade to pytz 2024.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 64e7aa0..66a4b4a 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ classifiers=['Programming Language :: Python :: 3 :: Only'], url="http://singer.io", install_requires=[ - 'pytz>=2018.4', + 'pytz>=2024.1', 'jsonschema>=2.6.0,==2.*', 'simplejson>=3.13.2,==3.*', 'python-dateutil>=2.7.3,==2.*', From 7d900aec67ab2997518c7e163f3ae6dc57bb8fa0 Mon Sep 17 00:00:00 2001 From: Nikita Maruniak Date: Thu, 20 Jun 2024 18:53:00 +0300 Subject: [PATCH 3/6] Upgrade to simplejson 3.19.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 66a4b4a..d94a0dc 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ install_requires=[ 'pytz>=2024.1', 'jsonschema>=2.6.0,==2.*', - 'simplejson>=3.13.2,==3.*', + 'simplejson>=3.19.2,==3.*', 'python-dateutil>=2.7.3,==2.*', 'backoff>=2.2.1,==2.*', 'ciso8601>=2.3.1,==2.*', From 3907ec5b3a9b14cc5d15dff8e024c92c6bd9dd28 Mon Sep 17 00:00:00 2001 From: Nikita Maruniak Date: Thu, 20 Jun 2024 18:55:14 +0300 Subject: [PATCH 4/6] Upgrade to dateutil 2.8.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d94a0dc..8204121 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ 'pytz>=2024.1', 'jsonschema>=2.6.0,==2.*', 'simplejson>=3.19.2,==3.*', - 'python-dateutil>=2.7.3,==2.*', + 'python-dateutil>=2.8.2,==2.*', 'backoff>=2.2.1,==2.*', 'ciso8601>=2.3.1,==2.*', ], From ea0489dcb73fcb19195eea50eeff9a13d6914e9a Mon Sep 17 00:00:00 2001 From: Nikita Maruniak Date: Thu, 20 Jun 2024 19:01:32 +0300 Subject: [PATCH 5/6] Upgrade to jsonschema 4.22.0. Remove incompatible singer-tools --- Makefile | 2 +- setup.py | 3 +-- singer/transform.py | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 61ab98e..0b2a926 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,6 @@ install: check_prereqs python3 -m pip install -e '.[dev]' test: install - pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-except,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access,broad-exception-raised,consider-using-f-string + pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-except,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access,broad-exception-raised,consider-using-f-string,no-name-in-module python3 -m doctest -v singer/utils.py python3 -m unittest -v diff --git a/setup.py b/setup.py index 8204121..6f06e6c 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ url="http://singer.io", install_requires=[ 'pytz>=2024.1', - 'jsonschema>=2.6.0,==2.*', + 'jsonschema>=4.22.0,==4.*', 'simplejson>=3.19.2,==3.*', 'python-dateutil>=2.8.2,==2.*', 'backoff>=2.2.1,==2.*', @@ -22,7 +22,6 @@ 'pylint', 'ipython', 'ipdb', - 'singer-tools' ] }, packages=find_packages(), diff --git a/singer/transform.py b/singer/transform.py index 69f812a..3f4fa26 100644 --- a/singer/transform.py +++ b/singer/transform.py @@ -2,6 +2,9 @@ import decimal import logging import re + +# TODO: Deprecated jsonschema API. Migrate to new one as described in docs: +# https://python-jsonschema.readthedocs.io/en/v4.22.0/referencing/#migrating-from-refresolver from jsonschema import RefResolver import singer.metadata From 76e37496f824bc37b90b0500566f02d02b7feef2 Mon Sep 17 00:00:00 2001 From: Andrii Porokhnavets Date: Fri, 16 May 2025 11:53:47 +0300 Subject: [PATCH 6/6] [CPL-20326] Update jsonschema dependency to 4.17.3 to align with Airbyte CDK --- setup.py | 2 +- singer/transform.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6f06e6c..ab2a818 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ url="http://singer.io", install_requires=[ 'pytz>=2024.1', - 'jsonschema>=4.22.0,==4.*', + 'jsonschema~=4.17.3,<4.18.0', 'simplejson>=3.19.2,==3.*', 'python-dateutil>=2.8.2,==2.*', 'backoff>=2.2.1,==2.*', diff --git a/singer/transform.py b/singer/transform.py index 3f4fa26..2e7081f 100644 --- a/singer/transform.py +++ b/singer/transform.py @@ -3,8 +3,6 @@ import logging import re -# TODO: Deprecated jsonschema API. Migrate to new one as described in docs: -# https://python-jsonschema.readthedocs.io/en/v4.22.0/referencing/#migrating-from-refresolver from jsonschema import RefResolver import singer.metadata