From a45d6b94699f0b4a351e6422f43d086cfd0db33b Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 9 May 2022 16:01:31 +0000 Subject: [PATCH 001/335] Updated zim.Archive to libkiwix's bhavior - Added `Archive.tags` and `Archive.get_tags()` for a parsed list of Tags - `Archive.get_tags()` can return libkiwix's version which includes hints - `Article.article_counter` now returns libzim's `article_count` or the parsed count based on the file's version Also updated Changelog format --- CHANGELOG.md | 73 +++++++++++++++++++--------- src/zimscraperlib/zim/_libkiwix.py | 39 ++++++++++++++- src/zimscraperlib/zim/archive.py | 34 ++++++++++++- tests/conftest.py | 13 +++++ tests/zim/test_archive.py | 77 ++++++++++++++++++++++++++++-- 5 files changed, 207 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a013f81..616da1a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,40 @@ -# 1.4.3 +## Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). + +## [Unreleased] + +### Added + +- `zim.Archive.tags` and `zim.Archive.get_tags()` to retrieve parsed Tags + with optionnal `libkiwix` param to include libkiwix's hints +- [tests] Counter tests now also uses a libzim6 file. + +### Changed + +- `zim.Archive.article_counter` follows libkiwix's new bahavior of + returning libzim's `article_count` for libzim 7+ ZIMs and + returning previously returned (parsed) value for older ZIMs. + +### Removed + +- Unreachable code removed in `imaging` module. +- [tests] “Sanskrit” removed from tests as output not predicatble depending on plaftform. + + +## [1.4.3] * `zim.Archive.counters` wont fail on missing `Counter` metadata -# 1.4.2 +## [1.4.2] * Fixed leak in `zim.Archive`'s `.counters` * New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding -# 1.4.1 +## [1.4.1] * New `Counter` metadata based properties for Archive: * `.counters`: parsed dict of the Counter metadata @@ -17,7 +44,7 @@ * Added `uri` module with `rebuild_uri()` -# 1.4.0 +## [1.4.0] * Using new python-libzim based on libzim v7 * New Creator API @@ -37,7 +64,7 @@ * Fixed `image.save_image()` saving to disk even when using a bytes stream * Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst -# 1.3.6 (internal) +## [1.3.6 (internal)] Intermediate release using unreleased libzim to support development of libzim7. Don't use it. @@ -58,7 +85,7 @@ Don't use it. * Added delete_fpath to add_item_for() and fixed StaticItem's auto remove * Updated badges for new repo name -# 1.3.5 +## [1.3.5] * add `stream_file()` to stream content from a URL into a file or a `BytesIO` object * deprecated `save_file()` @@ -67,7 +94,7 @@ Don't use it. * Added support for in-memory optimization for PNG, JPEG, and WebP images * allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ -# 1.3.4 +## [1.3.4] * added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager * do not use extension for finding format in `ensure_matches()` in `image.optimization` module @@ -76,21 +103,21 @@ Don't use it. * `save_image` moved from `image` to `image.utils` * added `convert_image` `optimize_image` `resize_image` functions to `image` module -# 1.3.3 +## [1.3.3] * added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads -# 1.3.2 +## [1.3.2] * fixed rewriting of links with empty target * added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats * added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix -# 1.3.1 +## [1.3.1] * replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`) -# 1.3.0 +## [1.3.0] * detect mimetypes from filenames for all text files * fixed non-filename based StaticArticle @@ -107,7 +134,7 @@ Don't use it. * changed `get_colors()` param names (`image_path` -> `src`) * changed `resize_image()` param names (`fpath` -> `src`) -# 1.2.1 +## [1.2.1] * fixed URL rewriting when running from / * added support for link rewriting in `` element @@ -115,7 +142,7 @@ Don't use it. * use non greedy match for CSS URL links (shortest string matching `url()` format) * fix namespace of target only if link doesn't have a netloc -# 1.2.0 +## [1.2.0] * added UTF8 to constants * added mime_type discovery via magic (filesystem) @@ -128,11 +155,11 @@ Don't use it. * Added zim.rewriting: tools to rewrite links/urls in HTML/CSS * add timeout and retries to save_file() and make it return headers -# 1.1.2 +## [1.1.2] * fixed `convert_image()` which tried to use a closed file -# 1.1.1 +## [1.1.1] * exposed reencode, Config and get_media_info in zimscraperlib.video * added save_image() and convert_image() in zimscraperlib.imaging @@ -140,34 +167,34 @@ Don't use it. * resize_image() now supports params given by user and preservs image colorspace * fixed tests for zimscraperlib.imaging -# 1.1.0 +## [1.1.0] * added video module with reencode, presets, config builder and video file probing * `make_zim_file()` accepts extra kwargs for zimwriterfs -# 1.0.6 +## [1.0.6] * added translation support to i18n -# 1.0.5 +## [1.0.5] * added s3transfer to verbose dependencies list * changed default log format to include module name -# 1.0.4 +## [1.0.4] * verbose dependencies (urllib3, boto3) now logged at WARNING level by default * ability to set verbose dependencies log level and add modules to the list * zimscraperlib's logging level now aligned with scraper's requested one -# 1.0.3 +## [1.0.3] * fix_ogvjs_dist script more generic (#1) * updated zim to support other zimwriterfs params (#10) * more flexible requirements for requests dependency -# 1.0.2 +## [1.0.2] * fixed return value of `get_language_details` on non-existent language * fixed crash on `resize_image` with method `height` @@ -179,11 +206,11 @@ Don't use it. * added `create_favicon` to generate a squared favicon * added `handle_user_provided_file` to handle user file/URL from param -# 1.0.1 +## [1.0.1] * fixed fix_ogvjs_dist -# 1.0.0 +## [1.0.0] * initial version providing * download: save_file, save_large_file diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index 2701912f..2971f209 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -9,11 +9,12 @@ https://github.com/kiwix/libkiwix/blob/master/src/reader.cpp https://github.com/kiwix/libkiwix/blob/master/src/tools/archiveTools.cpp +https://github.com/kiwix/libkiwix/blob/master/src/tools/otherTools.cpp """ import io from collections import namedtuple -from typing import Dict, Optional, Tuple +from typing import Dict, List, Optional, Tuple MimetypeAndCounter = namedtuple("MimetypeAndCounter", ["mimetype", "value"]) CounterMap = Dict[type(MimetypeAndCounter.mimetype), type(MimetypeAndCounter.value)] @@ -105,3 +106,39 @@ def getMediaCount(counterMap: CounterMap) -> int: counter += count return counter + + +def convertTags(tags_str: str) -> List[str]: + """List of tags expanded with libkiwix's additional hints for pic/vid/det/index""" + tags = tags_str.split(";") + tagsList = [] + picSeen = vidSeen = detSeen = indexSeen = False + for tag in tags: + # not upstream + if not tag: + continue + picSeen |= tag == "nopic" or tag.startswith("_pictures:") + vidSeen |= tag == "novid" or tag.startswith("_videos:") + detSeen |= tag == "nodet" or tag.startswith("_details:") + indexSeen |= tag.startswith("_ftindex") + + if tag == "nopic": + tagsList.append("_pictures:no") + elif tag == "novid": + tagsList.append("_videos:no") + elif tag == "nodet": + tagsList.append("_details:no") + elif tag == "_ftindex": + tagsList.append("_ftindex:yes") + else: + tagsList.append(tag) + + if not indexSeen: + tagsList.append("_ftindex:no") + if not picSeen: + tagsList.append("_pictures:yes") + if not vidSeen: + tagsList.append("_videos:yes") + if not detSeen: + tagsList.append("_details:yes") + return tagsList diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 0d4134e1..5444c37a 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -10,13 +10,13 @@ - direct access to search results and number of results - public Entry access by Id""" -from typing import Dict, Iterable, Optional +from typing import Dict, Iterable, List, Optional import libzim.reader import libzim.search # Query, Searcher import libzim.suggestion # SuggestionSearcher -from ._libkiwix import getArticleCount, getMediaCount, parseMimetypeCounter +from ._libkiwix import convertTags, getArticleCount, getMediaCount, parseMimetypeCounter from .items import Item @@ -36,6 +36,22 @@ def metadata(self) -> Dict[str, str]: if not key.startswith("Illustration_") } + @property + def tags(self): + return self.get_tags() + + def get_tags(self, libkiwix: bool = False) -> List[str]: + """List of ZIM tags, optionnaly expanded with libkiwix's hints""" + try: + tags_meta = self.get_text_metadata("Tags") + except RuntimeError: # pragma: nocover + tags_meta = "" + + if libkiwix: + return convertTags(tags_meta) + + return tags_meta.split(";") + def get_text_metadata(self, name: str) -> str: """Decoded value of a text metadata""" return super().get_metadata(name).decode("UTF-8") @@ -94,6 +110,20 @@ def counters(self) -> Dict[str, int]: @property def article_counter(self) -> int: """Nb of *articles* in the ZIM, using counters (from libkiwix)""" + + # [libkiwix HACK] + # getArticleCount() returns different things depending on + # the "version" of the zim. + # On old zim (<=6), it returns the number of entry in `A` namespace + # On recent zim (>=7), it returns: + # - the number of entry in `C` namespace (==getEntryCount) + # if no frontArticleIndex is present + # - the number of front article if a frontArticleIndex is present + # The use case >=7 without frontArticleIndex is pretty rare so we don't care + # We can detect if we are reading a zim <= 6 + # by checking if we have a newNamespaceScheme. + if self.has_new_namespace_scheme: + return self.article_count return getArticleCount(self.counters) @property diff --git a/tests/conftest.py b/tests/conftest.py index 92d831b4..74a48ad9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -140,6 +140,19 @@ def small_zim_file(tmpdir_factory): return dst +@pytest.fixture(scope="session") +def ns_zim_file(tmpdir_factory): + from zimscraperlib.download import stream_file + + dst = tmpdir_factory.mktemp("data").join("ns.zim") + stream_file( + "https://github.com/openzim/zim-testing-suite/raw/v0.4/data/withns/" + "wikibooks_be_all_nopic_2017-02.zim", + dst, + ) + return dst + + @pytest.mark.slow @pytest.fixture(scope="session") def real_zim_file(tmpdir_factory): diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index b72e1607..2a9d05d6 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -5,6 +5,7 @@ import pytest from zimscraperlib.zim import Archive +from zimscraperlib.zim._libkiwix import convertTags def test_metadata(small_zim_file): @@ -66,11 +67,81 @@ def test_counters(small_zim_file): assert zim.counters == {"image/png": 1, "text/html": 1} -def test_article_counter(small_zim_file): +def test_article_counter(small_zim_file, ns_zim_file): with Archive(small_zim_file) as zim: - assert zim.article_counter == 1 + assert zim.article_counter == 0 + with Archive(ns_zim_file) as zim: + assert zim.article_counter == 66 -def test_media_counter(small_zim_file): + +def test_media_counter(small_zim_file, ns_zim_file): with Archive(small_zim_file) as zim: assert zim.media_counter == 1 + + with Archive(ns_zim_file) as zim: + assert zim.media_counter == 34 + + +def test_get_tags(small_zim_file, real_zim_file): + with Archive(small_zim_file) as zim: + assert zim.get_tags() == ["_ftindex:no"] + assert zim.get_tags(libkiwix=True) == [ + "_ftindex:no", + "_pictures:yes", + "_videos:yes", + "_details:yes", + ] + assert zim.tags == zim.get_tags() + + with Archive(real_zim_file) as zim: + assert zim.get_tags() == [ + "wikipedia", + "_category:wikipedia", + "_pictures:no", + "_videos:no", + "_details:yes", + "_ftindex:yes", + ] + assert zim.get_tags(libkiwix=True) == [ + "wikipedia", + "_category:wikipedia", + "_pictures:no", + "_videos:no", + "_details:yes", + "_ftindex:yes", + ] + assert zim.tags == zim.get_tags() + + +def test_libkiwix_convertTags(): + assert convertTags("") == [ + "_ftindex:no", + "_pictures:yes", + "_videos:yes", + "_details:yes", + ] + assert convertTags("nopic") == [ + "_pictures:no", + "_ftindex:no", + "_videos:yes", + "_details:yes", + ] + assert convertTags("novid") == [ + "_videos:no", + "_ftindex:no", + "_pictures:yes", + "_details:yes", + ] + assert convertTags("nodet") == [ + "_details:no", + "_ftindex:no", + "_pictures:yes", + "_videos:yes", + ] + assert convertTags("_ftindex") == [ + "_ftindex:yes", + "_pictures:yes", + "_videos:yes", + "_details:yes", + ] From 60f94b596c418832c647394b2572e20078484fb2 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 9 May 2022 16:05:47 +0000 Subject: [PATCH 002/335] released v1.5.0 --- CHANGELOG.md | 2 +- src/zimscraperlib/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616da1a8..651b8ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [1.5.0] - 2022-05-09 ### Added diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 428b770e..bc80560f 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.4.3 +1.5.0 From 7ec55c4781b53f9f3945b78de0ad93ae7e9fc7c9 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 9 May 2022 16:07:07 +0000 Subject: [PATCH 003/335] back to dev --- CHANGELOG.md | 4 ++++ src/zimscraperlib/VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 651b8ff3..700e726e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +- n/a + ## [1.5.0] - 2022-05-09 ### Added diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index bc80560f..9bdd20b1 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.5.0 +1.5.0.post0 From 504c99788450f6cff08fb5b57d55ba1bf4bcf9f0 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 23 May 2022 10:15:31 +0000 Subject: [PATCH 004/335] Releasing v1.6.0 - Using pylibzim 1.1.0 - Updated tests to match new behavior (duplicates, title index) --- CHANGELOG.md | 12 ++++++++++-- requirements.txt | 4 ++-- src/zimscraperlib/VERSION | 2 +- tests/zim/conftest.py | 2 +- tests/zim/test_fs.py | 8 ++++---- tests/zim/test_zim_creator.py | 13 ++++++++----- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 700e726e..48d2dbf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,17 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [1.6.0] - 2022-05-23 -- n/a +## Added + + - `Archive.get_metadata_item()` to retrieve full item instead of just value + +### Changed + +- Using pylibzim v1.1.0 (using libzim 7.2.1) + - Adding duplicate entries now raises RuntimeError + - filesize is fixed for larger ZIMs ## [1.5.0] - 2022-05-09 diff --git a/requirements.txt b/requirements.txt index e00dcdff..6a88a0e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -iso-639==0.4.5 + iso-639==0.4.5 requests>=2.25.1,<3.0 colorthief==0.2.1 python-resize-image>=1.1.19,<1.2 Babel>=2.9,<3.0 file-magic>=0.4.0,<0.5 -libzim>=1.0.0,<1.1 +libzim>=1.1.0,<1.2 beautifulsoup4>=4.9.3,<4.10 lxml>=4.6.3,<4.7 optimize-images>=1.3.6,<1.6 diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 9bdd20b1..dc1e644a 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.5.0.post0 +1.6.0 diff --git a/tests/zim/conftest.py b/tests/zim/conftest.py index a3c87493..5e10e58e 100644 --- a/tests/zim/conftest.py +++ b/tests/zim/conftest.py @@ -39,7 +39,7 @@ def build_data(tmp_path, png_image): redirects_file = tmp_path / "redirects.tsv" with open(redirects_file, "w") as fh: fh.write(" \tAccueil\tBienvenue !!\twelcome\n") - fh.write(" \tAccueil\t\tcommons.png\n") + fh.write(" \tAccueil2\t\tcommons.png\n") fh.write(" \timage\t\tcommons.png\n") build_dir = tmp_path / "build" return { diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index 339ebbf2..f726076a 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -78,13 +78,13 @@ def test_make_zim_file_working(build_data, png_image): assert build_data["fpath"].exists() reader = Archive(build_data["fpath"]) # welcome (actual) and two redirs - assert reader.entry_count == 7 # includes redirect + assert reader.entry_count == 8 # includes redirect assert reader.get_item("style.css").mimetype == "text/css" assert reader.get_item("app.js").mimetype == "application/javascript" - assert reader.get_suggestions_count("bienvenue") == 1 - assert reader.get_suggestions_count("coucou") == 0 - assert "Accueil" in list(reader.get_suggestions("bienvenue")) + assert reader.get_suggestions_count("bienvenue") == 0 + assert reader.get_suggestions_count("coucou") == 1 + assert "welcome" in list(reader.get_suggestions("coucou")) def test_make_zim_file_exceptions_while_building(tmp_path, png_image, build_data): diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 77e12f55..8de1b77f 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -83,12 +83,13 @@ def test_zim_creator(tmp_path, png_image, html_file, html_str): # make sure we have our redirects assert reader.get_entry_by_path("home2").is_redirect assert reader.get_entry_by_path("home2").get_redirect_entry().path == f"{main_path}" - # make sure titles were indexed (html with title for xapian) + # make sure titles were indexed (html with title for xapian ; redirects are not) # see https://github.com/openzim/python-libzim/issues/125 - assert "home2" in list(reader.get_suggestions("Home !!")) - assert "home3" in list(reader.get_suggestions("Home !!")) - assert "home4" in list(reader.get_suggestions("Home !!")) - assert "images/yahoo.png" not in list(reader.get_suggestions("Home !!")) + # see https://github.com/openzim/libzim/issues/642 + assert "home2" not in list(reader.get_suggestions("Home !!")) # no is_front > False + assert "home3" in list(reader.get_suggestions("Home !!")) # is_front=True + assert "home4" not in list(reader.get_suggestions("Home !!")) # is_front=False + assert "images/yahoo.png" in list(reader.get_suggestions("Home !!")) # is_frontTrue # make sure full text was indexed assert reader.get_search_results_count("PDF doc") >= 1 @@ -278,6 +279,8 @@ def test_urlitem_nonhtmlgzip(tmp_path, gzip_nonhtml_url): fpath = tmp_path / "test.zim" with Creator(fpath) as creator: creator.add_item(URLItem(url=gzip_nonhtml_url)) + + with Creator(fpath) as creator: creator.add_item(URLItem(url=gzip_nonhtml_url, use_disk=True)) zim = Archive(fpath) From 3bda03bb50ff2707ff0e89469c6b01284deec5bc Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 23 May 2022 14:32:04 +0000 Subject: [PATCH 005/335] tests on py3.6-3.10 --- .github/workflows/ci.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f20a8ae..1bf179c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle diff --git a/tox.ini b/tox.ini index 140882bc..15b70f95 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, black, coverage +envlist = py36, py37, py38, py39, py310, black, coverage [testenv] passenv = From 13b3cb263c70f3be422a47f4598083f6ce9535bb Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 23 May 2022 14:38:24 +0000 Subject: [PATCH 006/335] specify py version as string in workflow --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bf179c6..f09b87c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle @@ -43,7 +43,7 @@ jobs: - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: "3.8" - name: Install dependencies run: | python -m pip install --upgrade pip From e29e6bef99615f867a40e0ae2e30a05faa54209d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 26 Jul 2022 17:02:17 +0000 Subject: [PATCH 007/335] updated lxml dependency range --- CHANGELOG.md | 8 +++++++- requirements.txt | 4 ++-- src/zimscraperlib/VERSION | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48d2dbf3..0a73eab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,17 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [1.6.1] - 2022-07-26 + +### Changed + +- Wider range of accepted lxml dependency version as 4.9.1 fixes a security issue + ## [1.6.0] - 2022-05-23 ## Added - - `Archive.get_metadata_item()` to retrieve full item instead of just value +- `Archive.get_metadata_item()` to retrieve full item instead of just value ### Changed diff --git a/requirements.txt b/requirements.txt index 6a88a0e2..5e3c704f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ - iso-639==0.4.5 +iso-639==0.4.5 requests>=2.25.1,<3.0 colorthief==0.2.1 python-resize-image>=1.1.19,<1.2 @@ -6,7 +6,7 @@ Babel>=2.9,<3.0 file-magic>=0.4.0,<0.5 libzim>=1.1.0,<1.2 beautifulsoup4>=4.9.3,<4.10 -lxml>=4.6.3,<4.7 +lxml>=4.6.3,<4.10 optimize-images>=1.3.6,<1.6 # youtube-dl should be updated as frequently as possible youtube_dl diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index dc1e644a..9c6d6293 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.6.0 +1.6.1 From 4494a40500e66c56006b545e5b57054d038e4fb4 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 29 Jul 2022 14:58:57 +0000 Subject: [PATCH 008/335] Fixed `filesystem.get_content_mimetype()` crashing on non-guessable byte stream --- CHANGELOG.md | 6 ++ src/zimscraperlib/VERSION | 2 +- src/zimscraperlib/filesystem.py | 5 +- tests/conftest.py | 136 ++++++++++++++++++++++++++++ tests/filesystem/test_filesystem.py | 4 +- 5 files changed, 150 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a73eab3..e177cf08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [1.6.2] - 2022-07-29 + +### Changed + +- Fixed `filesystem.get_content_mimetype()` crashing on non-guessable byte stream + ## [1.6.1] - 2022-07-26 ### Changed diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 9c6d6293..fdd3be6d 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.6.1 +1.6.2 diff --git a/src/zimscraperlib/filesystem.py b/src/zimscraperlib/filesystem.py index a4c0ab0b..fb5b804d 100644 --- a/src/zimscraperlib/filesystem.py +++ b/src/zimscraperlib/filesystem.py @@ -31,7 +31,10 @@ def get_file_mimetype(fpath: pathlib.Path) -> str: def get_content_mimetype(content: bytes) -> str: """MIME Type of content retrieved from magic headers""" - detected_mime = magic.detect_from_content(content).mime_type + try: + detected_mime = magic.detect_from_content(content).mime_type + except UnicodeDecodeError: + return "application/octet-stream" return MIME_OVERRIDES.get(detected_mime, detected_mime) diff --git a/tests/conftest.py b/tests/conftest.py index 74a48ad9..49e706f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -165,3 +165,139 @@ def real_zim_file(tmpdir_factory): dst, ) return dst + + +@pytest.fixture(scope="session") +def undecodable_byte_stream(): + + return ( + b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x01\x00\xfb\xff~\x00\x08\x00\xfd\xff\x01\x00\x00\x00" + b"\xff\xff\x03\x00\xf6\xffz\x00\x12\x00\xfa\xff\x02\x00\x00\x00" + b"\xff\xff\x04\x00\xf3\xffv\x00\x1b\x00\xf7\xff\x03\x00\xff\xff" + b"\xff\xff\x04\x00\xf0\xffp\x00%\x00\xf5\xff\x04\x00\xff\xff" + b"\xff\xff\x05\x00\xee\xffi\x000\x00\xf2\xff\x04\x00\xff\xff" + b"\xff\xff\x05\x00\xed\xffa\x00:\x00\xf0\xff\x05\x00\xff\xff" + b"\xff\xff\x06\x00\xed\xffX\x00D\x00\xee\xff\x05\x00\xff\xff" + b"\xff\xff\x06\x00\xed\xffN\x00N\x00\xed\xff\x06\x00\xff\xff" + b"\xff\xff\x05\x00\xee\xffD\x00X\x00\xed\xff\x06\x00\xff\xff" + b"\xff\xff\x05\x00\xf0\xff:\x00a\x00\xed\xff\x05\x00\xff\xff" + b"\xff\xff\x04\x00\xf2\xff0\x00i\x00\xee\xff\x05\x00\xff\xff" + b"\xff\xff\x04\x00\xf5\xff%\x00p\x00\xf0\xff\x04\x00\xff\xff" + b"\xff\xff\x03\x00\xf7\xff\x1b\x00v\x00\xf3\xff\x04\x00\xff\xff" + b"\x00\x00\x02\x00\xfa\xff\x12\x00z\x00\xf6\xff\x03\x00\xff\xff" + b"\x00\x00\x01\x00\xfd\xff\x08\x00~\x00\xfb\xff\x01\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xfd\xff\xff\xff \x00@\x00&\x00\x01\x00\xfd\xff\x00\x00" + b"\xfe\xff\xfe\xff\x1d\x00?\x00)\x00\x02\x00\xfd\xff\x00\x00" + b"\xfe\xff\xfe\xff\x1a\x00?\x00+\x00\x04\x00\xfc\xff\x00\x00" + b"\xfe\xff\xfd\xff\x18\x00>\x00.\x00\x05\x00\xfc\xff\x00\x00" + b"\xfe\xff\xfd\xff\x15\x00<\x001\x00\x07\x00\xfc\xff\x00\x00" + b"\xff\xff\xfc\xff\x12\x00;\x003\x00\t\x00\xfc\xff\x00\x00" + b"\xff\xff\xfc\xff\x10\x009\x005\x00\x0c\x00\xfc\xff\xff\xff" + b"\xff\xff\xfc\xff\x0e\x007\x007\x00\x0e\x00\xfc\xff\xff\xff" + b"\xff\xff\xfc\xff\x0c\x005\x009\x00\x10\x00\xfc\xff\xff\xff" + b"\x00\x00\xfc\xff\t\x003\x00;\x00\x12\x00\xfc\xff\xff\xff" + b"\x00\x00\xfc\xff\x07\x001\x00<\x00\x15\x00\xfd\xff\xfe\xff" + b"\x00\x00\xfc\xff\x05\x00.\x00>\x00\x18\x00\xfd\xff\xfe\xff" + b"\x00\x00\xfc\xff\x04\x00+\x00?\x00\x1a\x00\xfe\xff\xfe\xff" + b"\x00\x00\xfd\xff\x02\x00)\x00?\x00\x1d\x00\xfe\xff\xfe\xff" + b"\x00\x00\xfd\xff\x01\x00&\x00@\x00 \x00\xff\xff\xfd\xff" + b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xff\xff\x03\x00\xf9\xff\x7f\x00\x08\x00\xfd\xff\x01\x00\x00\x00" + b"\xfe\xff\x05\x00\xf3\xff}\x00\x11\x00\xfa\xff\x03\x00\xff\xff" + b"\xfd\xff\x07\x00\xef\xffy\x00\x1b\x00\xf6\xff\x05\x00\xfe\xff" + b"\xfc\xff\t\x00\xec\xffs\x00%\x00\xf3\xff\x06\x00\xfe\xff" + b"\xfc\xff\n\x00\xe9\xffl\x000\x00\xf0\xff\x08\x00\xfd\xff" + b"\xfc\xff\n\x00\xe8\xffd\x00;\x00\xed\xff\t\x00\xfd\xff" + b"\xfc\xff\x0b\x00\xe8\xffZ\x00F\x00\xeb\xff\n\x00\xfc\xff" + b"\xfc\xff\x0b\x00\xe9\xffP\x00P\x00\xe9\xff\x0b\x00\xfc\xff" + b"\xfc\xff\n\x00\xeb\xffF\x00Z\x00\xe8\xff\x0b\x00\xfc\xff" + b"\xfd\xff\t\x00\xed\xff;\x00d\x00\xe8\xff\n\x00\xfc\xff" + b"\xfd\xff\x08\x00\xf0\xff0\x00l\x00\xe9\xff\n\x00\xfc\xff" + b"\xfe\xff\x06\x00\xf3\xff%\x00s\x00\xec\xff\t\x00\xfc\xff" + b"\xfe\xff\x05\x00\xf6\xff\x1b\x00y\x00\xef\xff\x07\x00\xfd\xff" + b"\xff\xff\x03\x00\xfa\xff\x11\x00}\x00\xf3\xff\x05\x00\xfe\xff" + b"\x00\x00\x01\x00\xfd\xff\x08\x00\x7f\x00\xf9\xff\x03\x00\xff\xff" + b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00x\x00\x08\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00p\x00\x10\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00h\x00\x18\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00`\x00 \x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00X\x00(\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00P\x000\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00H\x008\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00@\x00@\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x008\x00H\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x000\x00P\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00(\x00X\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00 \x00`\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x18\x00h\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x10\x00p\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x08\x00x\x00\x00\x00\x00\x00\x00\x00" + b"\x02\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x05\x00\x00\x00" + b"\x03\x00\x00\x00\x06\x00\x00\x00\x08\x01\x00\x00\x07\x00\x00\x00" + b"\x03\x01\x00\x00\x08\x00\x00\x00\x07\x01\x00\x00\t\x00\x00\x00" + b"\t\x01\x00\x00\n\x00\x00\x00\n\x01\x00\x00\x0b\x00\x00\x00" + b"\x0b\x01\x00\x00\x0c\x00\x00\x00\x00\x01\x00\x00\r\x00\x00\x00" + b"\x01\x01\x00\x00\x0e\x00\x00\x00\x80\x00\x00\x00\x0f\x00\x00\x00" + b"\x05\x01\x00\x00\x10\x00\x00\x00\x06\x01\x00\x00\x11\x00\x00\x00" + b"\x04\x01\x00\x00\x12\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00" + b"\x07\x14!.;HUbo|\x89\x96\xa3\xb0\xbd\xca" + b"\xd7\xe4\xf1\xfe\x01\x02\x03\x04\x05\x06\x08\t\n\x0b\x0c\r" + b"\x0e\x0f\x10\x11\x12\x13\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e" + b"\x1f \"#$%&'()*+,-/0" + b"123456789:<=>?@A" + b"BCDEFGIJKLMNOPQR" + b"STVWXYZ[\\]^_`acd" + b"efghijklmnpqrstu" + b"vwxyz{}~\x7f\x80\x81\x82\x83\x84\x85\x86" + b"\x87\x88\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x97\x98" + b"\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa4\xa5\xa6\xa7\xa8\xa9" + b"\xaa\xab\xac\xad\xae\xaf\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba" + b"\xbb\xbc\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xcb\xcc" + b"\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc\xdd" + b"\xde\xdf\xe0\xe1\xe2\xe3\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee" + b"\xef\xf0\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfd\x00" + b"\x00\x07\x06\x06\x05\x05\x05\x05\x04\x04\x04\x04\x04\x04\x04\x04" + b"\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03" + b"\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02" + b"\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02" + b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x9f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xa5\x91\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xad\x94\x8c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xb0\x9b\x8c\x87\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xb4\x9d\x8d\x86\x82\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xfe\xfe\xfe\xfc\xf9\xf3\xe6\xc4\xb1\x99\x8c\x85\x82\x81\x00\x00" + b"\x00\x01\x01\x02\x02\x02\x03\x03\x03\x03\x04\x04\x04\x04\x04\x04" + b"\x04\x04\x04\x04\x04\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + b"" + ) diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index ff64ed86..4d7ebd1d 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -14,13 +14,15 @@ def test_file_mimetype(png_image, jpg_image): assert get_file_mimetype(jpg_image) == "image/jpeg" -def test_content_mimetype(png_image, jpg_image): +def test_content_mimetype(png_image, jpg_image, undecodable_byte_stream): with open(png_image, "rb") as fh: assert get_content_mimetype(fh.read(64)) == "image/png" with open(jpg_image, "rb") as fh: assert get_content_mimetype(fh.read(64)) == "image/jpeg" + assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" + def test_mime_overrides(svg_image): mime_map = [(svg_image, "image/svg+xml")] From e494db488d66451cb13f9fbaf9a736dadd953f04 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 1 Aug 2022 11:13:46 +0000 Subject: [PATCH 009/335] mock magic so our fallback can be tested on systems with fixed magic version --- tests/filesystem/test_filesystem.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index 4d7ebd1d..bdd1ee89 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +import magic + from zimscraperlib.filesystem import ( delete_callback, get_content_mimetype, @@ -21,6 +23,17 @@ def test_content_mimetype(png_image, jpg_image, undecodable_byte_stream): with open(jpg_image, "rb") as fh: assert get_content_mimetype(fh.read(64)) == "image/jpeg" + +def test_content_mimetype_fallback(monkeypatch, undecodable_byte_stream): + + # use raw function first to test actual code + assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" + + # mock then so we keep coverage on systems where magic works + def raising_magic(*args): + raise UnicodeDecodeError("nocodec", b"", 0, 1, "noreason") + + monkeypatch.setattr(magic, "detect_from_content", raising_magic) assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" From 12b30d87f975174bb59681019c34766c802dd359 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 Aug 2022 16:01:49 +0000 Subject: [PATCH 010/335] Added (missing) headers to stream_file --- CHANGELOG.md | 6 ++++++ src/zimscraperlib/VERSION | 2 +- src/zimscraperlib/download.py | 2 ++ tests/download/test_download.py | 12 ++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e177cf08..8c099768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [1.6.3] - 2022-08-02 + +### Added + +- `download.stream_file()` supports passing `headers` (scrapers were already using it) + ## [1.6.2] - 2022-07-29 ### Changed diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index fdd3be6d..266146b8 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.6.2 +1.6.3 diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 6802f69d..e82aff19 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -165,6 +165,7 @@ def stream_file( proxies: Optional[dict] = None, only_first_block: Optional[bool] = False, max_retries: Optional[int] = 5, + headers: Optional[Dict[str, str]] = None, ) -> Union[int, requests.structures.CaseInsensitiveDict]: """Stream data from a URL to either a BytesIO object or a file Arguments - @@ -188,6 +189,7 @@ def stream_file( url, stream=True, proxies=proxies, + headers=headers, ) resp.raise_for_status() diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 15377ba6..858627a8 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -65,6 +65,18 @@ def test_first_block_download(valid_http_url): assert len(byte_stream.read()) <= expected +@pytest.mark.slow +def test_user_agent(): + ua = "zimscraperlib-test" + byte_stream = io.BytesIO() + stream_file( + url="http://useragentstring.com/", + byte_stream=byte_stream, + headers={"User-Agent": "zimscraperlib-test"}, + ) + assert ua in byte_stream.read().decode("utf-8") + + @pytest.mark.slow def test_save_http(tmp_path, valid_http_url): dest_file = tmp_path / "favicon.ico" From fe85a5847756e18bad2ace8c04ad135872dc4ab3 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 Aug 2022 18:10:38 +0000 Subject: [PATCH 011/335] Added duplicate failsafe feature to Creator - `zim.Creator` now supports `ignore_duplicates: bool` parameter to prevent duplicates from raising exceptions - `zim.Creator.add_item`, `zim.Creator.add_redirect` and `zim.Creator.add_item_for` now supports a `duplicate_ok: bool` parameter to prevent an exception should this item/redirect be a duplicate --- CHANGELOG.md | 10 ++++++++ src/zimscraperlib/zim/creator.py | 39 +++++++++++++++++++++++++++++--- tests/zim/test_zim_creator.py | 19 ++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c099768..b728da79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [1.7.0] - 2022-08-02 + +### Added + +- `zim.Creator` now supports `ignore_duplicates: bool` parameter to + prevent duplicates from raising exceptions +- `zim.Creator.add_item`, `zim.Creator.add_redirect` and `zim.Creator.add_item_for` + now supports a `duplicate_ok: bool` parameter to prevent an exception + should this item/redirect be a duplicate + ## [1.6.3] - 2022-08-02 ### Added diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 9e451877..9321e5e2 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -20,6 +20,7 @@ import datetime import pathlib +import re import weakref from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -30,6 +31,13 @@ from ..types import get_mime_for_name from .items import StaticItem +DUPLICATE_EXC_STR = re.compile( + r"^Impossible to add(.+)" + r"dirent\'s title to add is(.+)" + r"existing dirent's title is(.+)", + re.MULTILINE | re.DOTALL, +) + def mimetype_for( path: str, @@ -76,6 +84,7 @@ def __init__( language: Optional[str] = "eng", compression: Optional[str] = None, workaround_nocancel: Optional[bool] = True, + ignore_duplicates: Optional[bool] = False, **metadata: Dict[str, Union[str, datetime.date, datetime.datetime]] ): super().__init__(filename=filename) @@ -103,6 +112,7 @@ def __init__( self.metadata = metadata self.workaround_nocancel = workaround_nocancel + self.ignore_duplicates = ignore_duplicates def start(self): super().__enter__() @@ -129,6 +139,7 @@ def add_item_for( is_front: Optional[bool] = None, should_compress: Optional[bool] = None, delete_fpath: Optional[bool] = False, + duplicate_ok: Optional[bool] = None, callback: Optional[Union[callable, Tuple[callable, Any]]] = None, ): """Add a File or content at a specified path and get its path @@ -178,12 +189,15 @@ def add_item_for( cb += list(callback) callback = tuple(cb) - self.add_item(StaticItem(**kwargs), callback) + self.add_item( + StaticItem(**kwargs), callback=callback, duplicate_ok=duplicate_ok + ) return path def add_item( self, item: libzim.writer.Item, + duplicate_ok: Optional[bool] = None, callback: Optional[Union[Callable, Tuple[Callable, Any]]] = None, ): """Add a libzim.writer.Item @@ -196,8 +210,14 @@ def add_item( weakref.finalize(item, callback) else: weakref.finalize(item, *callback) + + duplicate_ok = duplicate_ok or self.ignore_duplicates try: - super().add_item(item) + try: + super().add_item(item) + except RuntimeError as exc: + if not DUPLICATE_EXC_STR.match(str(exc)) or not duplicate_ok: + raise exc except Exception: if self.workaround_nocancel: self.can_finish = False # pragma: no cover @@ -209,6 +229,7 @@ def add_redirect( target_path: str, title: Optional[str] = "", is_front: Optional[bool] = None, + duplicate_ok: Optional[bool] = None, ): """Add a redirect from path to target_path @@ -217,7 +238,19 @@ def add_redirect( hints = {} if is_front is not None: hints[libzim.writer.Hint.FRONT_ARTICLE] = bool(is_front) - super().add_redirection(path, title, target_path, hints) + + duplicate_ok = duplicate_ok or self.ignore_duplicates + + try: + try: + super().add_redirection(path, title, target_path, hints) + except RuntimeError as exc: + if not DUPLICATE_EXC_STR.match(str(exc)) or not duplicate_ok: + raise exc + except Exception: + if self.workaround_nocancel: + self.can_finish = False # pragma: no cover + raise def add_default_illustration(self, content: bytes): self.add_illustration(48, content) diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 8de1b77f..496febd4 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -458,3 +458,22 @@ def cb(*args): assert not html_file.exists() assert Store.called assert Store.called == 2 + + +def test_duplicates(tmp_path): + with Creator(tmp_path / "test.zim") as creator: + creator.add_item_for(path="A", content="A") + creator.add_item_for(path="C", content="C") + creator.add_redirect(path="B", target_path="A") + with pytest.raises(RuntimeError, match="existing dirent's title"): + creator.add_item_for(path="A", content="test2") + with pytest.raises(RuntimeError, match="existing dirent's title"): + creator.add_redirect(path="B", target_path="C") + + +def test_ignore_duplicates(tmp_path): + with Creator(tmp_path / "test.zim", ignore_duplicates=True) as creator: + creator.add_item_for(path="A", content="A") + creator.add_item_for(path="A", content="A2") + creator.add_redirect(path="B", target_path="A") + creator.add_redirect(path="B", target_path="C") From 996cef156359d4c54d90b52516c7362f71297265 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 Aug 2022 18:18:11 +0000 Subject: [PATCH 012/335] released 1.7.0 --- src/zimscraperlib/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 266146b8..bd8bf882 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.6.3 +1.7.0 From 619066b843b8767287eea8cd1dc6ea06b74ff3f3 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 5 Aug 2022 09:41:43 +0000 Subject: [PATCH 013/335] Added session creation and reuse in stream_file --- CHANGELOG.md | 10 ++++++++++ src/zimscraperlib/VERSION | 2 +- src/zimscraperlib/download.py | 17 +++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b728da79..e40f1a1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +### Added + +- `download.get_session()` to build a new requests Session + +### Changed + +- `download.stream_file()` accepts a `session` param to use instead of creating one + ## [1.7.0] - 2022-08-02 ### Added diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index bd8bf882..27f9cd32 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.7.0 +1.8.0 diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index e82aff19..f9b397b8 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -157,6 +157,13 @@ def _get_retry_adapter(max_retries: Optional[int] = 5) -> requests.adapters.Base return requests.adapters.HTTPAdapter(max_retries=retries) +def get_session(max_retries: Optional[int] = 5) -> requests.Session: + """Session to hold cookies and connection pool together""" + session = requests.Session() + session.mount("http", _get_retry_adapter(max_retries)) # tied to http and https + return session + + def stream_file( url: str, fpath: Optional[pathlib.Path] = None, @@ -166,6 +173,7 @@ def stream_file( only_first_block: Optional[bool] = False, max_retries: Optional[int] = 5, headers: Optional[Dict[str, str]] = None, + session: Optional[requests.Session] = None, ) -> Union[int, requests.structures.CaseInsensitiveDict]: """Stream data from a URL to either a BytesIO object or a file Arguments - @@ -175,16 +183,17 @@ def stream_file( proxies - A dict of proxies to be used https://requests.readthedocs.io/en/master/user/advanced/#proxies only_first_block - Whether to download only one (first) block - max_retries - Maximum number of retries after which error is raised + max_retries - Maximum number of retries after which error is raised. Does not + apply if using your own session + session - Session object to make the request with. A new one created otherwise Returns the total number of bytes downloaded and the response headers""" # if no output option is supplied if fpath is None and byte_stream is None: raise ValueError("Either file path or a bytesIO object is needed") - session = requests.Session() - retry_adapter = _get_retry_adapter(max_retries) - session.mount("http", retry_adapter) # tied to http and https + if not session: + session = get_session(max_retries) resp = session.get( url, stream=True, From d95cd6fee50d5b80d8f0b37d47eee7422bf71b22 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 3 Oct 2022 11:17:44 +0000 Subject: [PATCH 014/335] 1.8.0 has been released --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e40f1a1a..ce1710a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- n/a + +## [1.8.0] - 2022-08-05 + ### Added - `download.get_session()` to build a new requests Session From 00d7fb0a1fb39381d6501a96ef0dc0bb804d4c95 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 3 Oct 2022 11:58:34 +0000 Subject: [PATCH 015/335] changing target for UA change test due to frequent issues --- tests/download/test_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 858627a8..a2c123d9 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -70,7 +70,7 @@ def test_user_agent(): ua = "zimscraperlib-test" byte_stream = io.BytesIO() stream_file( - url="http://useragentstring.com/", + url="http://useragent.fr/", byte_stream=byte_stream, headers={"User-Agent": "zimscraperlib-test"}, ) From efaccfed328bb11436a4c636eb501f82d8903c8a Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 6 Dec 2022 15:01:25 +0000 Subject: [PATCH 016/335] Using pylibzim v2.1.0 (using libzim 8.1.0) --- CHANGELOG.md | 17 +++++++++++++++- requirements.txt | 2 +- src/zimscraperlib/zim/_libkiwix.py | 24 ---------------------- src/zimscraperlib/zim/archive.py | 32 ++++++++++++------------------ tests/zim/test_libkiwix.py | 12 +---------- tests/zim/test_zim_creator.py | 4 ++-- 6 files changed, 33 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce1710a7..9f038c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- n/a +- Using pylibzim v2.1.0 (using libzim 8.1.0) + +### Added + +- [libzim] `Entry.get_redirect_entry()` +- [libzim] `Item.get_indexdata()` to implement custom IndexData per entry (writer) +- [libzim] `Archive.media_count` + +### Changed +- [libzim] `Archive.article_count` updated to match scraperlib's version +- `Archive.article_counter` now deprecated. Now returns `Archive.article_count` +- `Archive.media_counter` now deprecated. Now returns `Archive.media_count` + +### Removed + +- [libzim] `lzma` compression algorithm ## [1.8.0] - 2022-08-05 diff --git a/requirements.txt b/requirements.txt index 5e3c704f..05691024 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ colorthief==0.2.1 python-resize-image>=1.1.19,<1.2 Babel>=2.9,<3.0 file-magic>=0.4.0,<0.5 -libzim>=1.1.0,<1.2 +libzim>=2.1.0,<3.0 beautifulsoup4>=4.9.3,<4.10 lxml>=4.6.3,<4.10 optimize-images>=1.3.6,<1.6 diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index 2971f209..d71db702 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -84,30 +84,6 @@ def parseMimetypeCounter( return counters -def getArticleCount(counterMap: CounterMap): - """Get the count of articles which can be indexed/displayed""" - counter = 0 - for mimetype, count in counterMap.items(): - if mimetype.startswith("text/html"): - counter += count - - return counter - - -def getMediaCount(counterMap: CounterMap) -> int: - """Get the count of medias content in the ZIM file""" - counter = 0 - for mimetype, count in counterMap.items(): - if ( - mimetype.startswith("image/") - or mimetype.startswith("video/") - or mimetype.startswith("audio/") - ): - counter += count - - return counter - - def convertTags(tags_str: str) -> List[str]: """List of tags expanded with libkiwix's additional hints for pic/vid/det/index""" tags = tags_str.split(";") diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 5444c37a..87da077c 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -10,13 +10,14 @@ - direct access to search results and number of results - public Entry access by Id""" +import warnings from typing import Dict, Iterable, List, Optional import libzim.reader import libzim.search # Query, Searcher import libzim.suggestion # SuggestionSearcher -from ._libkiwix import convertTags, getArticleCount, getMediaCount, parseMimetypeCounter +from ._libkiwix import convertTags, parseMimetypeCounter from .items import Item @@ -109,24 +110,17 @@ def counters(self) -> Dict[str, int]: @property def article_counter(self) -> int: - """Nb of *articles* in the ZIM, using counters (from libkiwix)""" - - # [libkiwix HACK] - # getArticleCount() returns different things depending on - # the "version" of the zim. - # On old zim (<=6), it returns the number of entry in `A` namespace - # On recent zim (>=7), it returns: - # - the number of entry in `C` namespace (==getEntryCount) - # if no frontArticleIndex is present - # - the number of front article if a frontArticleIndex is present - # The use case >=7 without frontArticleIndex is pretty rare so we don't care - # We can detect if we are reading a zim <= 6 - # by checking if we have a newNamespaceScheme. - if self.has_new_namespace_scheme: - return self.article_count - return getArticleCount(self.counters) + warnings.warn( + "Archive.article_counter now deprecated. " + "Use Archive.article_count instead", + DeprecationWarning, + ) + return self.article_count @property def media_counter(self) -> int: - """Nb of *medias* in the ZIM, using counters (from libkiwix)""" - return getMediaCount(self.counters) + warnings.warn( + "Archive.media_counter now deprecated. " "Use Archive.media_count instead", + DeprecationWarning, + ) + return self.media_count diff --git a/tests/zim/test_libkiwix.py b/tests/zim/test_libkiwix.py index c81ed0c2..aa3c29a7 100644 --- a/tests/zim/test_libkiwix.py +++ b/tests/zim/test_libkiwix.py @@ -6,7 +6,7 @@ import pytest -from zimscraperlib.zim._libkiwix import getArticleCount, getline, getMediaCount +from zimscraperlib.zim._libkiwix import getline from zimscraperlib.zim._libkiwix import parseMimetypeCounter as parse empty = {} @@ -81,13 +81,3 @@ def test_getline(): def test_counter_parsing(counterStr, counterMap): # https://github.com/kiwix/libkiwix/blob/master/test/counterParsing.cpp assert parse(counterStr) == counterMap - - -def test_article_count(counters): - assert getArticleCount({}) == 0 - assert getArticleCount(counters) == 6339 - - -def test_media_count(counters): - assert getMediaCount({}) == 0 - assert getMediaCount(counters) == 12862 diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 496febd4..001e8efc 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -179,10 +179,10 @@ def remove_source(item): def test_compression(tmp_path): fpath = tmp_path / "test.zim" - with Creator(tmp_path / "test.zim", "welcome", "", compression="lzma") as creator: + with Creator(tmp_path / "test.zim", "welcome", "", compression="zstd") as creator: creator.add_item(StaticItem(path="welcome", content="hello")) - with Creator(fpath, "welcome", "", compression=Compression.lzma) as creator: + with Creator(fpath, "welcome", "", compression=Compression.zstd) as creator: creator.add_item(StaticItem(path="welcome", content="hello")) From d80b74a0fbef1ef821ce54dbc35d9efa6f5ff78b Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 6 Dec 2022 15:10:23 +0000 Subject: [PATCH 017/335] released v2.0.0 --- CHANGELOG.md | 2 +- setup.py | 3 +++ src/zimscraperlib/VERSION | 2 +- tox.ini | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f038c6b..9e29c1c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [2.0.0] - 2022-12-06 - Using pylibzim v2.1.0 (using libzim 8.1.0) diff --git a/setup.py b/setup.py index 236fb36d..0b02de6f 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,9 @@ def read(*names, **kwargs): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ], python_requires=">=3.6", diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 27f9cd32..227cea21 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -1.8.0 +2.0.0 diff --git a/tox.ini b/tox.ini index 15b70f95..b3977b7f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, py310, black, coverage +envlist = py36, py37, py38, py39, py310, py311, black, coverage [testenv] passenv = From e67f0145c5879708508a34a784f086d741fdbdd0 Mon Sep 17 00:00:00 2001 From: Kelson Date: Tue, 13 Dec 2022 21:50:24 +0100 Subject: [PATCH 018/335] Delete move.yml --- .github/move.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/move.yml diff --git a/.github/move.yml b/.github/move.yml deleted file mode 100644 index bdb89031..00000000 --- a/.github/move.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Configuration for Move Issues - https://github.com/dessant/move-issues - -# Delete the command comment when it contains no other content -deleteCommand: true - -# Close the source issue after moving -closeSourceIssue: true - -# Lock the source issue after moving -lockSourceIssue: false - -# Mention issue and comment authors -mentionAuthors: true - -# Preserve mentions in the issue content -keepContentMentions: true - -# Move labels that also exist on the target repository -moveLabels: true - -# Set custom aliases for targets -# aliases: -# r: repo -# or: owner/repo - -# Repository to extend settings from -# _extends: repo From 0e2920a42d6c9dde25472b08310e6601314fde32 Mon Sep 17 00:00:00 2001 From: Kelson Date: Tue, 20 Dec 2022 22:23:05 +0100 Subject: [PATCH 019/335] =?UTF-8?q?=E2=80=9Cmain=E2=80=9D=20is=20the=20new?= =?UTF-8?q?=20default=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aeafbc5d..3616db3c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ zimscraperlib ============= -[![Build Status](https://github.com/openzim/python-scraperlib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/openzim/python-scraperlib/actions?query=branch%3Amaster) +[![Build Status](https://github.com/openzim/python-scraperlib/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/openzim/python-scraperlib/actions?query=branch%3Amain) [![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-scraperlib/badge)](https://www.codefactor.io/repository/github/openzim/python-scraperlib) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) [![PyPI version shields.io](https://img.shields.io/pypi/v/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib/) From 2d0cd09cf3f50967e7dd6aef64fbcab39f65be0d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 13 Feb 2023 15:14:02 +0100 Subject: [PATCH 020/335] Fixed URLProvider returning incomplete data --- CHANGELOG.md | 6 ++++++ src/zimscraperlib/zim/providers.py | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e29c1c4..89ca391e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +### Changed + +- Fixed `zim.providers.URLProvider` returning incomplete streams under certain circumstances (from https://github.com/openzim/kolibri/issues/40) + ## [2.0.0] - 2022-12-06 - Using pylibzim v2.1.0 (using libzim 8.1.0) diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index be40346c..6fa74e0f 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -95,6 +95,7 @@ def get_size(self) -> int: return self.size def gen_blob(self) -> libzim.writer.Blob: # pragma: nocover - bsize = 1048576 # 1MiB chunk - for chunk in self.resp.iter_content(bsize): - yield libzim.writer.Blob(chunk) + for chunk in self.resp.iter_content(10 * 1024): + if chunk: + yield libzim.writer.Blob(chunk) + yield libzim.writer.Blob(b"") From c9310492ec7b7808516a4bb56760d7f24ced77e7 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 6 Mar 2023 16:28:09 +0000 Subject: [PATCH 021/335] Fixed Creator(language=) now supporting multiple values --- CHANGELOG.md | 5 +++++ src/zimscraperlib/zim/creator.py | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ca391e..6ae185e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## Added + +- `zim.creator.Creator(language=)` can be specified as `List[str]`. `["eng", "fra"]`, `["eng"]`, `"eng,fra"`, "eng" are all valid values. + ### Changed - Fixed `zim.providers.URLProvider` returning incomplete streams under certain circumstances (from https://github.com/openzim/kolibri/issues/40) +- Fixed `zim.creator.Creator` not supporting multiple values in for Language metadata, as required by the spec ## [2.0.0] - 2022-12-06 diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 9321e5e2..1dac648f 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -22,7 +22,7 @@ import pathlib import re import weakref -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import libzim.writer @@ -81,7 +81,7 @@ def __init__( self, filename: pathlib.Path, main_path: str = None, - language: Optional[str] = "eng", + language: Optional[Union[str, List[str]]] = "eng", compression: Optional[str] = None, workaround_nocancel: Optional[bool] = True, ignore_duplicates: Optional[bool] = False, @@ -94,8 +94,10 @@ def __init__( self.main_path = main_path if language: - self.config_indexing(True, language) - ld = {"Language": language} + if not isinstance(language, list): + language = language.split(",") + self.config_indexing(True, language[0]) + ld = {"Language": ",".join(language)} if metadata: metadata.update(ld) else: From 6f93bccd2b941e76d9606972bb1d5a487ca97831 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 6 Mar 2023 16:31:09 +0000 Subject: [PATCH 022/335] released v2.1.0 --- CHANGELOG.md | 2 +- src/zimscraperlib/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae185e2..092723e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [2.1.0] - 2023-03-06 ## Added diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 227cea21..7ec1d6db 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -2.0.0 +2.1.0 From 1b71a9662a0b80c708b7dedfbb87643007aa4e4b Mon Sep 17 00:00:00 2001 From: Fledge Shiu Date: Sat, 11 Mar 2023 21:22:43 -0500 Subject: [PATCH 023/335] Introduce the `config_metadata`. --- src/zimscraperlib/constants.py | 28 +++++ src/zimscraperlib/zim/creator.py | 133 ++++++++++++++++------ src/zimscraperlib/zim/filesystem.py | 39 ++++--- tests/zim/test_zim_creator.py | 167 ++++++++++++++++++++-------- 4 files changed, 271 insertions(+), 96 deletions(-) diff --git a/src/zimscraperlib/constants.py b/src/zimscraperlib/constants.py index 726c6c25..4ce360e6 100644 --- a/src/zimscraperlib/constants.py +++ b/src/zimscraperlib/constants.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +import base64 import pathlib ROOT_DIR = pathlib.Path(__file__).parent @@ -17,3 +18,30 @@ # list of mimetypes we consider articles using it should default to FRONT_ARTICLE FRONT_ARTICLE_MIMETYPES = ["text/html"] + +# list of mandatory meta tags of the zim file. +MANDATORY_ZIM_METADATA_KEYS = [ + "Name", + "Title", + "Creator", + "Publisher", + "Date", + "Description", + "Language", + "Illustration_48x48@1", +] + +DEFAULT_DEV_ZIM_METADATA = { + "Name": "Test Name", + "Title": "Test Title", + "Creator": "Test Creator", + "Publisher": "Test Publisher", + "Date": "2023-01-01", + "Description": "Test Description", + "Language": "fra", + "Illustration_48x48_at_1": base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBB" + "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAA9JREFUeNpi+P//P0CAAQAF/gL+Lc6J7gAAAABJ" + "RU5ErkJggg==" + ), +} diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 1dac648f..5fe8185e 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -22,11 +22,15 @@ import pathlib import re import weakref -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Iterable, Optional, Tuple, Union import libzim.writer -from ..constants import FRONT_ARTICLE_MIMETYPES +from ..constants import ( + DEFAULT_DEV_ZIM_METADATA, + FRONT_ARTICLE_MIMETYPES, + MANDATORY_ZIM_METADATA_KEYS, +) from ..filesystem import delete_callback, get_content_mimetype, get_file_mimetype from ..types import get_mime_for_name from .items import StaticItem @@ -80,28 +84,16 @@ class Creator(libzim.writer.Creator): def __init__( self, filename: pathlib.Path, - main_path: str = None, - language: Optional[Union[str, List[str]]] = "eng", + main_path: str, compression: Optional[str] = None, workaround_nocancel: Optional[bool] = True, ignore_duplicates: Optional[bool] = False, - **metadata: Dict[str, Union[str, datetime.date, datetime.datetime]] ): super().__init__(filename=filename) + self._metadata = dict() self.can_finish = True - if main_path: - self.main_path = main_path - - if language: - if not isinstance(language, list): - language = language.split(",") - self.config_indexing(True, language[0]) - ld = {"Language": ",".join(language)} - if metadata: - metadata.update(ld) - else: - metadata = ld + self.set_mainpath(main_path) if compression: self.config_compression( @@ -110,26 +102,108 @@ def __init__( else compression ) - if metadata: - self.metadata = metadata - self.workaround_nocancel = workaround_nocancel self.ignore_duplicates = ignore_duplicates def start(self): + if not all( + [ + key in self._metadata.keys() and self._metadata.get(key, None) + for key in MANDATORY_ZIM_METADATA_KEYS + ] + ): + raise ValueError("Mandatory metadata are not all set.") + + for name, value in self._metadata.items(): + if value: + self._validate_metadata(name, value) + super().__enter__() - if getattr(self, "main_path", None): - self.set_mainpath(self.main_path) + self.add_illustration(48, self._metadata["Illustration_48x48@1"]) + del self._metadata["Illustration_48x48@1"] + for name, value in self._metadata.items(): + if value: + self.add_metadata(name, value) - if getattr(self, "metadata", None): - self.update_metadata(**self.metadata) return self - def update_metadata(self, **kwargs): - if kwargs: - for name, value in kwargs.items(): - self.add_metadata(name, value) + def _validate_metadata(self, name, value): + if name == "Counter": + raise ValueError("You do not need to set Counter.") + + if name == "Description" and len(value) > 80: + raise ValueError("Description is too long.") + + if name == "LongDescription" and len(value) > 4000: + raise ValueError("LongDescription is too long.") + + def config_metadata( + self, + *, + Name: str, + Language: str, + Title: str, + Description: str, + LongDescription: Optional[str] = None, + Creator: str, + Publisher: str, + Date: Union[datetime.datetime, datetime.date, str], + Illustration_48x48_at_1: bytes, + Tags: Optional[Union[Iterable[str], str]] = None, + Scraper: Optional[str] = None, + Flavour: Optional[str] = None, + Source: Optional[str] = None, + License: Optional[str] = None, + Relation: Optional[str] = None, + **extras: str, + ): + """ + A chaining functions which configures the metadata of the Creator class. + You must set all mandatory metadata in this phase. + + Parameters: + check out: https://wiki.openzim.org/wiki/Metadata + all the extra metadata must be plain text. + + Returns: + Self + """ + self._metadata.update( + { + "Name": Name, + "Title": Title, + "Creator": Creator, + "Publisher": Publisher, + "Date": Date, + "Description": Description, + "Language": Language, + "License": License, + "LongDescription": LongDescription, + "Tags": Tags, + "Relation": Relation, + "Flavour": Flavour, + "Source": Source, + "Scraper": Scraper, + "Illustration_48x48@1": Illustration_48x48_at_1, + } + ) + self._metadata.update(extras) + language = self._metadata.get("Language", "").split(",") + self.config_indexing(True, language[0]) + + return self + + def config_dev_metadata(self, **extras: str): + """ + A Test function. It will set the default test metadata for a Creator instance. + + Returns: + Self + """ + devel_default_metadata = DEFAULT_DEV_ZIM_METADATA.copy() + devel_default_metadata.update(extras) + return self.config_metadata(**devel_default_metadata) def add_item_for( self, @@ -254,9 +328,6 @@ def add_redirect( self.can_finish = False # pragma: no cover raise - def add_default_illustration(self, content: bytes): - self.add_illustration(48, content) - def finish(self, exc_type=None, exc_val=None, exc_tb=None): """Triggers finalization of ZIM creation and create final ZIM file.""" if not getattr(self, "can_finish", False): diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index a77552a9..2b3b948f 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -125,6 +125,7 @@ def make_zim_file( source: str = None, flavour: str = None, scraper: str = None, + long_description: str = None, without_fulltext_index: bool = False, redirects: Sequence[Tuple[str, str, str]] = None, redirects_file: pathlib.Path = None, @@ -149,39 +150,37 @@ def make_zim_file( if not favicon_path.exists() or not favicon_path.is_file(): raise IOError(f"Incorrect favicon: {favicon} ({favicon_path})") - zim_file = Creator( - filename=fpath, - main_path=main_page, - index_language="" if without_fulltext_index else language, + with open(favicon_path, "rb") as fh: + favicon_data = fh.read() + + zim_file = Creator(filename=fpath, main_path=main_page).config_metadata( **{ k: v for k, v in { # (somewhat) mandatory - "name": name, - "title": title, - "description": description, - "date": date or datetime.date.today(), - "language": language, - "creator": creator, - "publisher": publisher, + "Name": name, + "Title": title, + "Description": description, + "Date": date or datetime.date.today(), + "Language": language, + "Creator": creator, + "Publisher": publisher, # optional - "tags": ";".join(tags) if tags else None, - "source": source, - "flavour": flavour, - "scraper": scraper, + "Tags": ";".join(tags) if tags else None, + "Source": source, + "Flavour": flavour, + "Scraper": scraper, + "LongDescription": long_description, + "Illustration_48x48_at_1": favicon_data, }.items() if v is not None - }, + } ) zim_file.start() try: logger.debug(f"Preparing zimfile at {zim_file.filename}") - # add favicon as illustration - with open(favicon_path, "rb") as fh: - zim_file.add_default_illustration(fh.read()) - # recursively add content from build_dir logger.debug(f"Recursively adding files from {build_dir}") add_to_zim(build_dir, zim_file, build_dir) diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 001e8efc..bc31cc32 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -15,7 +15,7 @@ import pytest from libzim.writer import Compression -from zimscraperlib.constants import UTF8 +from zimscraperlib.constants import DEFAULT_DEV_ZIM_METADATA, UTF8 from zimscraperlib.download import save_large_file, stream_file from zimscraperlib.filesystem import delete_callback from zimscraperlib.zim import Archive, Creator, StaticItem, URLItem @@ -41,13 +41,13 @@ def get_contentprovider(self): def test_zim_creator(tmp_path, png_image, html_file, html_str): fpath = tmp_path / "test.zim" - main_path, language, title = "welcome", "fra", "My Title" + main_path = "welcome" tags = ";".join(["toto", "tata"]) - with open(png_image, "rb") as fh: png_data = fh.read() - - with Creator(fpath, main_path, language, title=title, tags=tags) as creator: + with Creator(fpath, main_path).config_dev_metadata( + Tags=tags, Illustration_48x48_at_1=png_data + ) as creator: # verbatim HTML from string creator.add_item_for("welcome", "wel", content=html_str, is_front=True) # verbatim HTML from file @@ -68,15 +68,12 @@ def test_zim_creator(tmp_path, png_image, html_file, html_str): with pytest.raises(ValueError, match="One of fpath or content is required"): creator.add_item_for("images/yahoo.png") - with open(png_image, "rb") as fh: - creator.add_default_illustration(png_data) - assert fpath.exists() reader = Archive(fpath) - assert reader.get_metadata("Title").decode(UTF8) == title - assert reader.get_metadata("Language").decode(UTF8) == language - assert reader.get_metadata("Tags").decode(UTF8) == tags + assert reader.get_text_metadata("Title") == DEFAULT_DEV_ZIM_METADATA["Title"] + assert reader.get_text_metadata("Language") == DEFAULT_DEV_ZIM_METADATA["Language"] + assert reader.get_text_metadata("Tags") == tags assert reader.main_entry.get_item().path == f"{main_path}" # make sure we have our image assert reader.get_item("images/yahoo.png") @@ -106,21 +103,22 @@ def test_create_without_workaround(tmp_path): fpath = tmp_path / "test.zim" with Creator( - fpath, "welcome", "fra", title="My Title", workaround_nocancel=False - ) as creator: + fpath, "welcome", workaround_nocancel=False + ).config_dev_metadata() as creator: with pytest.raises(RuntimeError, match="AttributeError"): creator.add_item("hello") def test_noindexlanguage(tmp_path): fpath = tmp_path / "test.zim" - with Creator(fpath, "welcome", "") as creator: + creator = Creator(fpath, "welcome").config_dev_metadata(Language="bam") + creator.config_indexing(False, "") + with creator as creator: creator.add_item(StaticItem(path="welcome", content="hello")) - creator.update_metadata(language="bam") creator.add_item_for("index", "Index", content="-", mimetype="text/html") reader = Archive(fpath) - assert reader.get_metadata("Language").decode(UTF8) == "bam" + assert reader.get_text_metadata("Language") == "bam" # html content triggers both title and content xapian indexes # but since indexing is disabled, we should only have title one assert reader.has_title_index @@ -130,11 +128,11 @@ def test_noindexlanguage(tmp_path): def test_add_item_for(tmp_path): fpath = tmp_path / "test.zim" # test without mimetype - with Creator(fpath, "welcome", "") as creator: + with Creator(fpath, "welcome").config_dev_metadata() as creator: creator.add_item_for(path="welcome", title="hello", content="hello") # test missing fpath and content - with Creator(fpath, "welcome", "") as creator: + with Creator(fpath, "welcome").config_dev_metadata() as creator: with pytest.raises(ValueError): creator.add_item_for(path="welcome", title="hello") @@ -146,7 +144,7 @@ def test_add_item_for_delete(tmp_path, html_file): # copy file to local path shutil.copyfile(html_file, local_path) - with Creator(fpath, "welcome", "") as creator: + with Creator(fpath, "welcome").config_dev_metadata() as creator: creator.add_item_for(fpath=local_path, path="index", delete_fpath=True) assert not local_path.exists() @@ -166,7 +164,7 @@ def remove_source(item): print("##########", "remove_source") os.remove(item.filepath) - with Creator(fpath, "welcome", "") as creator: + with Creator(fpath, "welcome").config_dev_metadata() as creator: creator.add_item( StaticItem(filepath=local_path, path="index", callback=remove_source), callback=(delete_callback, local_path), @@ -179,16 +177,20 @@ def remove_source(item): def test_compression(tmp_path): fpath = tmp_path / "test.zim" - with Creator(tmp_path / "test.zim", "welcome", "", compression="zstd") as creator: + with Creator( + tmp_path / "test.zim", "welcome", compression="zstd" + ).config_dev_metadata() as creator: creator.add_item(StaticItem(path="welcome", content="hello")) - with Creator(fpath, "welcome", "", compression=Compression.zstd) as creator: + with Creator( + fpath, "welcome", compression=Compression.zstd + ).config_dev_metadata() as creator: creator.add_item(StaticItem(path="welcome", content="hello")) def test_double_finish(tmp_path): fpath = tmp_path / "test.zim" - with Creator(fpath, "welcome", "fra") as creator: + with Creator(fpath, "welcome").config_dev_metadata() as creator: creator.add_item(StaticItem(path="welcome", content="hello")) # ensure we can finish an already finished creator @@ -196,15 +198,14 @@ def test_double_finish(tmp_path): def test_cannot_finish(tmp_path): - creator = Creator(tmp_path / "test.zim") + creator = Creator(tmp_path / "test.zim", "") creator.can_finish = False creator.finish() def test_sourcefile_removal(tmp_path, html_file): - fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: # using a temp dir so file still have a meaningful name tmpdir = tempfile.TemporaryDirectory(dir=tmp_path) # can't use contextmgr # copy html to folder @@ -217,10 +218,9 @@ def test_sourcefile_removal(tmp_path, html_file): def test_sourcefile_removal_std(tmp_path, html_file): - fpath = tmp_path / "test.zim" paths = [] - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: for idx in range(0, 4): # copy html to folder paths.append(pathlib.Path(tmp_path / f"source{idx}.html")) @@ -243,15 +243,14 @@ def test_sourcefile_noremoval(tmp_path, html_file): shutil.copyfile(html_file, src_path) fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(StaticItem(path=src_path.name, filepath=src_path)) assert src_path.exists() def test_urlitem_badurl(tmp_path): - - with Creator(tmp_path / "test.zim") as creator: + with Creator(tmp_path / "test.zim", "").config_dev_metadata() as creator: with pytest.raises(IOError, match="Unable to access URL"): creator.add_item(URLItem(url="httpo://hello:helloe:hello/")) @@ -263,7 +262,7 @@ def test_urlitem_html(tmp_path, gzip_html_url): file_bytes = fh.read() fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(URLItem(url=gzip_html_url)) zim = Archive(fpath) @@ -277,10 +276,10 @@ def test_urlitem_nonhtmlgzip(tmp_path, gzip_nonhtml_url): file_bytes = fh.read() fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(URLItem(url=gzip_nonhtml_url)) - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(URLItem(url=gzip_nonhtml_url, use_disk=True)) zim = Archive(fpath) @@ -294,7 +293,7 @@ def test_urlitem_binary(tmp_path, png_image_url): file_bytes = fh.read() fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(URLItem(url=png_image_url)) zim = Archive(fpath) @@ -306,7 +305,7 @@ def test_urlitem_binary(tmp_path, png_image_url): def test_urlitem_staticcontent(tmp_path, gzip_nonhtml_url): fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(URLItem(url=gzip_nonhtml_url, content="hello")) zim = Archive(fpath) @@ -318,7 +317,7 @@ def test_filelikeprovider_nosize(tmp_path, png_image_url): stream_file(png_image_url, byte_stream=fileobj) fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(FileLikeProviderItem(fileobj=fileobj, path="one.png")) zim = Archive(fpath) @@ -332,7 +331,7 @@ def test_urlprovider(tmp_path, png_image_url): file_bytes = fh.read() fpath = tmp_path / "test.zim" - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item(SpecialURLProviderItem(url=png_image_url, path="one.png")) zim = Archive(fpath) @@ -340,7 +339,6 @@ def test_urlprovider(tmp_path, png_image_url): def test_urlprovider_nolength(tmp_path, png_image_url, png_image): - # save url's content locally using external tool png_image = tmp_path / "original.png" save_large_file(png_image_url, png_image) @@ -379,7 +377,9 @@ def do_GET(self): fpath = tmp_path / "test.zim" try: - with tempfile.TemporaryDirectory() as tmp_dir, Creator(fpath) as creator: + with tempfile.TemporaryDirectory() as tmp_dir, Creator( + fpath, "" + ).config_dev_metadata() as creator: tmp_dir = pathlib.Path(tmp_dir) creator.add_item( URLItem( @@ -415,7 +415,7 @@ class Store: def cb(): Store.called = True - with Creator(fpath) as creator: + with Creator(fpath, "").config_dev_metadata() as creator: creator.add_item( StaticItem(path=html_file.name, filepath=html_file), callback=cb ) @@ -424,7 +424,7 @@ def cb(): def test_compess_hints(tmp_path, html_file): - with Creator(tmp_path / "test.zim") as creator: + with Creator(tmp_path / "test.zim", "").config_dev_metadata() as creator: creator.add_item_for( path=html_file.name, fpath=html_file, @@ -444,7 +444,7 @@ def cb(*args): html_file2 = html_file.with_suffix(f".2{html_file.suffix}") shutil.copyfile(html_file, html_file2) - with Creator(tmp_path / "test.zim") as creator: + with Creator(tmp_path / "test.zim", "").config_dev_metadata() as creator: creator.add_item_for( path=html_file.name, fpath=html_file, delete_fpath=True, callback=cb ) @@ -461,7 +461,7 @@ def cb(*args): def test_duplicates(tmp_path): - with Creator(tmp_path / "test.zim") as creator: + with Creator(tmp_path / "test.zim", "").config_dev_metadata() as creator: creator.add_item_for(path="A", content="A") creator.add_item_for(path="C", content="C") creator.add_redirect(path="B", target_path="A") @@ -472,8 +472,85 @@ def test_duplicates(tmp_path): def test_ignore_duplicates(tmp_path): - with Creator(tmp_path / "test.zim", ignore_duplicates=True) as creator: + with Creator( + tmp_path / "test.zim", "", ignore_duplicates=True + ).config_dev_metadata() as creator: creator.add_item_for(path="A", content="A") creator.add_item_for(path="A", content="A2") creator.add_redirect(path="B", target_path="A") creator.add_redirect(path="B", target_path="C") + + +def test_without_metadata(tmp_path): + with pytest.raises(ValueError, match="Mandatory metadata are not all set."): + Creator(tmp_path, "").start() + + +def test_check_metadata(tmp_path): + with pytest.raises(ValueError, match="You do not need to set Counter."): + Creator(tmp_path, "").config_dev_metadata(Counter=1).start() + + with pytest.raises(ValueError, match="Description is too long."): + Creator(tmp_path, "").config_dev_metadata(Description="T" * 90).start() + + with pytest.raises(ValueError, match="LongDescription is too long."): + Creator(tmp_path, "").config_dev_metadata(LongDescription="T" * 5000).start() + + +def test_config_metadata(tmp_path, png_image): + fpath = tmp_path / "test_config.zim" + with open(png_image, "rb") as fh: + png_data = fh.read() + creator = Creator(fpath, "").config_metadata( + Name="wikipedia_fr_football", + Title="English Wikipedia", + Creator="English speaking Wikipedia contributors", + Publisher="Wikipedia user Foobar", + Date="2009-11-21", + Description="All articles (without images) from the english Wikipedia", + LongDescription="This ZIM file contains all articles (without images)" + " from the english Wikipedia by 2009-11-10. The topics are...", + Language="eng", + License="CC-BY", + Tags="wikipedia;_category:wikipedia;_pictures:no;_videos:no;" + "_details:yes;_ftindex:yes", + Flavour="nopic", + Source="https://en.wikipedia.org/", + Scraper="mwoffliner 1.2.3", + Illustration_48x48_at_1=png_data, + TestMetadata="Test Metadata", + ) + with creator: + pass + + assert fpath.exists() + + reader = Archive(fpath) + assert reader.get_text_metadata("Name") == "wikipedia_fr_football" + assert reader.get_text_metadata("Title") == "English Wikipedia" + assert ( + reader.get_text_metadata("Creator") == "English speaking Wikipedia contributors" + ) + assert reader.get_text_metadata("Publisher") == "Wikipedia user Foobar" + assert reader.get_text_metadata("Date") == "2009-11-21" + assert ( + reader.get_text_metadata("Description") + == "All articles (without images) from the english Wikipedia" + ) + assert ( + reader.get_text_metadata("Longdescription") + == "This ZIM file contains all articles (without images)" + " from the english Wikipedia by 2009-11-10. The topics are..." + ) + assert reader.get_text_metadata("Language") == "eng" + assert reader.get_text_metadata("License") == "CC-BY" + assert ( + reader.get_text_metadata("Tags") + == "wikipedia;_category:wikipedia;_pictures:no;_videos:no;" + "_details:yes;_ftindex:yes" + ) + assert reader.get_text_metadata("Flavour") == "nopic" + assert reader.get_text_metadata("Source") == "https://en.wikipedia.org/" + assert reader.get_text_metadata("Scraper") == "mwoffliner 1.2.3" + assert reader.get_metadata("Illustration_48x48@1") == png_data + assert reader.get_text_metadata("Testmetadata") == "Test Metadata" From 61a4e40af1363afd235a38933495ebc1989fa053 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Mar 2023 09:18:59 +0000 Subject: [PATCH 024/335] Added is_valid_iso_639_3 check to i18n module --- src/zimscraperlib/i18n.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/zimscraperlib/i18n.py b/src/zimscraperlib/i18n.py index 775d83ba..0e721680 100644 --- a/src/zimscraperlib/i18n.py +++ b/src/zimscraperlib/i18n.py @@ -179,3 +179,8 @@ def get_language_details(query: str, failsafe: Optional[bool] = False) -> Dict: } ) return iso_data + + +def is_valid_iso_639_3(code: str) -> bool: + """whether code is a valid ISO-639-3 code""" + return (get_language_details(code, failsafe=True) or {}).get("iso-639-3") == code From 639cae681828ce746a70abd1783aa1f8f583a396 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Mar 2023 09:19:55 +0000 Subject: [PATCH 025/335] Renamed favicon to illustration in zim.filesystem module --- src/zimscraperlib/zim/filesystem.py | 16 ++++++++-------- tests/zim/conftest.py | 2 +- tests/zim/test_fs.py | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 2b3b948f..f047a8c6 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -114,7 +114,7 @@ def make_zim_file( fpath: pathlib.Path, name: str, main_page: str, - favicon: str, + illustration: str, title: str, description: str, date: datetime.date = None, @@ -135,7 +135,7 @@ def make_zim_file( """Creates a zimwriterfs-like ZIM file at {fpath} from {build_dir} main_page: path of item to serve as main page - favicon: relative path to favicon file in build_dir + illustration: relative path to illustration file in build_dir tags: list of str tags to add to meta redirects: list of (src, dst, title) tuple to create redirects from rewrite_links controls whether to rewrite HTML/CSS content @@ -146,12 +146,12 @@ def make_zim_file( if not build_dir.exists() or not build_dir.is_dir(): raise IOError(f"Incorrect build_dir: {build_dir}") - favicon_path = build_dir / favicon - if not favicon_path.exists() or not favicon_path.is_file(): - raise IOError(f"Incorrect favicon: {favicon} ({favicon_path})") + illustration_path = build_dir / illustration + if not illustration_path.exists() or not illustration_path.is_file(): + raise IOError(f"Incorrect illustration: {illustration} ({illustration_path})") - with open(favicon_path, "rb") as fh: - favicon_data = fh.read() + with open(illustration_path, "rb") as fh: + illustration_data = fh.read() zim_file = Creator(filename=fpath, main_path=main_page).config_metadata( **{ @@ -171,7 +171,7 @@ def make_zim_file( "Flavour": flavour, "Scraper": scraper, "LongDescription": long_description, - "Illustration_48x48_at_1": favicon_data, + "Illustration_48x48_at_1": illustration_data, }.items() if v is not None } diff --git a/tests/zim/conftest.py b/tests/zim/conftest.py index 5e10e58e..0f6c33a3 100644 --- a/tests/zim/conftest.py +++ b/tests/zim/conftest.py @@ -47,7 +47,7 @@ def build_data(tmp_path, png_image): "fpath": fpath, "name": "test-zim", "main_page": "welcome", - "favicon": png_image.name, + "illustration": png_image.name, "title": "Test ZIM", "description": "A test ZIM", "date": None, diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index f726076a..698bd331 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -37,7 +37,7 @@ def test_redirects_file(tmp_path, png_image, build_data): fpath=build_data["fpath"], name="test-zim", main_page="welcome", - favicon=png_image.name, + illustration=png_image.name, title="Test ZIM", description="A test ZIM", redirects_file=build_data["redirects_file"], @@ -51,8 +51,8 @@ def test_make_zim_file_fail_nobuildir(build_data): assert not build_data["fpath"].exists() -def test_make_zim_file_fail_nofavicon(build_data): - # ensure we fail on missing favicon +def test_make_zim_file_fail_noillustration(build_data): + # ensure we fail on missing illustration build_data["build_dir"].mkdir() with pytest.raises(IOError): make_zim_file(**build_data) @@ -110,7 +110,7 @@ def test_make_zim_file_no_file_on_error(tmp_path, png_image, build_data): fpath=pathlib.Path("{build_data['fpath']}"), name="test-zim", main_page="welcome", - favicon="{png_image.name}", + illustration="{png_image.name}", title="Test ZIM", description="A test ZIM", redirects_file="{build_data["redirects_file"]}") From cd9c63f77fbef17cfad6db2922689e31e5cb2244 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Mar 2023 09:23:16 +0000 Subject: [PATCH 026/335] Removed deprecated media_counter and article_counter properties from zim.archive.Archive --- src/zimscraperlib/zim/archive.py | 18 ------------------ tests/zim/test_archive.py | 16 ---------------- 2 files changed, 34 deletions(-) diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 87da077c..01c317a1 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -10,7 +10,6 @@ - direct access to search results and number of results - public Entry access by Id""" -import warnings from typing import Dict, Iterable, List, Optional import libzim.reader @@ -107,20 +106,3 @@ def counters(self) -> Dict[str, int]: return parseMimetypeCounter(self.get_text_metadata("Counter")) except RuntimeError: # pragma: no cover (no ZIM avail to test itl) return {} # pragma: no cover - - @property - def article_counter(self) -> int: - warnings.warn( - "Archive.article_counter now deprecated. " - "Use Archive.article_count instead", - DeprecationWarning, - ) - return self.article_count - - @property - def media_counter(self) -> int: - warnings.warn( - "Archive.media_counter now deprecated. " "Use Archive.media_count instead", - DeprecationWarning, - ) - return self.media_count diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index 2a9d05d6..060be38a 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -67,22 +67,6 @@ def test_counters(small_zim_file): assert zim.counters == {"image/png": 1, "text/html": 1} -def test_article_counter(small_zim_file, ns_zim_file): - with Archive(small_zim_file) as zim: - assert zim.article_counter == 0 - - with Archive(ns_zim_file) as zim: - assert zim.article_counter == 66 - - -def test_media_counter(small_zim_file, ns_zim_file): - with Archive(small_zim_file) as zim: - assert zim.media_counter == 1 - - with Archive(ns_zim_file) as zim: - assert zim.media_counter == 34 - - def test_get_tags(small_zim_file, real_zim_file): with Archive(small_zim_file) as zim: assert zim.get_tags() == ["_ftindex:no"] From 9d531a2c042a6bece0ebdc8876e1d38b70a241c6 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Mar 2023 10:22:08 +0000 Subject: [PATCH 027/335] Improvements to Creator API - Requiring libzim 3.0 to support setting LongDescription metadata - Exposing Creator.validate_metadata - Expanded validate_metadata to support all current checks for mandatory and recommendations - Added tests for all validate_metadata cases - Exposing max length for (Long)Description so scrapers can use it in their entrypoints - Marking `Creator.config_indexing`'s `language` argument optional (when disabling) - Only indexing on start() if config_indexing() was not already called manually - Fixed default dev illustration to be 48x48 - Added new image format and size validation check in image.probing module - Fixed test image (commons.png) to be 48x48 - Adapted tests for commons48 --- CHANGELOG.md | 30 ++++++ requirements.txt | 2 +- src/zimscraperlib/constants.py | 15 ++- src/zimscraperlib/image/__init__.py | 3 +- src/zimscraperlib/image/probing.py | 19 ++++ src/zimscraperlib/zim/creator.py | 147 +++++++++++++++++++++------- tests/conftest.py | 5 + tests/files/commons48.png | Bin 0 -> 3274 bytes tests/image/test_image.py | 36 +++++-- tests/zim/conftest.py | 6 +- tests/zim/test_archive.py | 2 +- tests/zim/test_fs.py | 4 +- tests/zim/test_zim_creator.py | 100 ++++++++++++++++++- 13 files changed, 312 insertions(+), 57 deletions(-) create mode 100644 tests/files/commons48.png diff --git a/CHANGELOG.md b/CHANGELOG.md index 092723e4..c9afaebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file` + +### Added + +- `zim.creator.Creator.config_metadata` method (returning Self) exposing all mandatory Metdata, all standard ones and allowing extra text metdadata. +- `zim.creator.Creator.config_dev_metadata` method setting stub metdata for all mandatory ones (allowing overrides) +- `zim.creator.Creator.validate_metadata` (called on `start`) to verify metadata respects the spec (and its recommendations) +- `zim.filesystem.make_zim_file` accepts a new optional `long_description` param. +- `i18n.is_valid_iso_639_3` to check ISO-639-3 codes +- `image.probing.is_valid_image` to check Image format and size + +### Changed + +- `zim.creator.Creator` `main_path` argument now mandatory +- `zim.creator.Creator.start` now fails on missing required or invalid metadata +- `zim.creator.Creator.add_metadata` nows enforces validation checks +- `zim.filesystem.make_zim_file` renamed its `favicon_path` param to `illustration_path` +- `zim.creator.Creator.config_indexing` `language` argument now optionnal when `indexing=False` +- `zim.creator.Creator.config_indexing` now validates `language` is ISO- 639-3 when `indexing=True` + +### Removed + +- `zim.creator.Creator.update_metadata`. See `.config_metadata()` instead +- `zim.creator.Creator` `language` argument. See `.config_metadata()` instead +- `zim.creator.Creator` keyword arguments. See `.config_metadata()` instead +- `zim.creator.Creator.add_default_illustration`. See `.config_metadata()` instead +- `zim.archibe.Archive.media_counter` (deprecated in `2.0.0`) + ## [2.1.0] - 2023-03-06 ## Added diff --git a/requirements.txt b/requirements.txt index 05691024..7ea268b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ colorthief==0.2.1 python-resize-image>=1.1.19,<1.2 Babel>=2.9,<3.0 file-magic>=0.4.0,<0.5 -libzim>=2.1.0,<3.0 +libzim>=3.0.0,<3.1 beautifulsoup4>=4.9.3,<4.10 lxml>=4.6.3,<4.10 optimize-images>=1.3.6,<1.6 diff --git a/src/zimscraperlib/constants.py b/src/zimscraperlib/constants.py index 4ce360e6..65914417 100644 --- a/src/zimscraperlib/constants.py +++ b/src/zimscraperlib/constants.py @@ -4,6 +4,7 @@ import base64 import pathlib +import re ROOT_DIR = pathlib.Path(__file__).parent NAME = pathlib.Path(__file__).parent.name @@ -39,9 +40,17 @@ "Date": "2023-01-01", "Description": "Test Description", "Language": "fra", + # blank 48x48 transparent PNG "Illustration_48x48_at_1": base64.b64decode( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBB" - "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAA9JREFUeNpi+P//P0CAAQAF/gL+Lc6J7gAAAABJ" - "RU5ErkJggg==" + "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwAQMAAABtzGvEAAAAGXRFWHRTb2Z0d2FyZQBB" + "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAANQTFRFR3BMgvrS0gAAAAF0Uk5TAEDm2GYAAAAN" + "SURBVBjTY2AYBdQEAAFQAAGn4toWAAAAAElFTkSuQmCC" ), } + +MAXIMUM_DESCRIPTION_METADATA_LENGTH = 80 +MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH = 4000 + +ILLUSTRATIONS_METADATA_RE = re.compile( + r"^Illustration_(?P\d+)x(?P\d+)@(?P\d+)$" +) diff --git a/src/zimscraperlib/image/__init__.py b/src/zimscraperlib/image/__init__.py index d9e24fd3..3834bc18 100644 --- a/src/zimscraperlib/image/__init__.py +++ b/src/zimscraperlib/image/__init__.py @@ -5,6 +5,7 @@ # flake8: noqa from .convertion import convert_image from .optimization import optimize_image +from .probing import is_valid_image from .transformation import resize_image -__all__ = ["convert_image", "optimize_image", "resize_image"] +__all__ = ["convert_image", "is_valid_image", "optimize_image", "resize_image"] diff --git a/src/zimscraperlib/image/probing.py b/src/zimscraperlib/image/probing.py index b26d2bce..173aca63 100644 --- a/src/zimscraperlib/image/probing.py +++ b/src/zimscraperlib/image/probing.py @@ -60,3 +60,22 @@ def format_for(src: Union[pathlib.Path, io.BytesIO], from_suffix: bool = True) - init_pil() return ext_fmt_map[src.suffix] # might raise KeyError on unknown extension + + +def is_valid_image( + image: Union[pathlib.Path, io.IOBase, bytes], + imformat: str, + size: Optional[Tuple[int, int]] = None, +) -> bool: + """whether image is a valid imformat (PNG) image, optionnaly of requested size""" + if isinstance(image, bytes): + image = io.BytesIO(image) + try: + img = PIL.Image.open(image) + if img.format != imformat: + return False + if size and img.size != size: + return False + except Exception: + return False + return True diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 5fe8185e..679d6cbf 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -22,6 +22,7 @@ import pathlib import re import weakref +from collections.abc import Iterable as IterableT from typing import Any, Callable, Iterable, Optional, Tuple, Union import libzim.writer @@ -29,9 +30,14 @@ from ..constants import ( DEFAULT_DEV_ZIM_METADATA, FRONT_ARTICLE_MIMETYPES, + ILLUSTRATIONS_METADATA_RE, MANDATORY_ZIM_METADATA_KEYS, + MAXIMUM_DESCRIPTION_METADATA_LENGTH, + MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH, ) from ..filesystem import delete_callback, get_content_mimetype, get_file_mimetype +from ..i18n import is_valid_iso_639_3 +from ..image.probing import is_valid_image from ..types import get_mime_for_name from .items import StaticItem @@ -91,6 +97,7 @@ def __init__( ): super().__init__(filename=filename) self._metadata = dict() + self.__indexing_configured = False self.can_finish = True self.set_mainpath(main_path) @@ -105,18 +112,28 @@ def __init__( self.workaround_nocancel = workaround_nocancel self.ignore_duplicates = ignore_duplicates + def config_indexing(self, indexing: bool, language: Optional[str] = None): + """Toggle full-text and title indexing of entries + + Uses Language metadata's value (or "") if not set""" + language = language or self._metadata.get("Language", "") + if indexing and not is_valid_iso_639_3(language): + raise ValueError("Not a valid ISO-639-3 language code") + super().config_indexing(indexing, language) + self.__indexing_configured = True + return self + def start(self): - if not all( - [ - key in self._metadata.keys() and self._metadata.get(key, None) - for key in MANDATORY_ZIM_METADATA_KEYS - ] - ): + if not all([self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS]): raise ValueError("Mandatory metadata are not all set.") for name, value in self._metadata.items(): if value: - self._validate_metadata(name, value) + self.validate_metadata(name, value) + + language = self._metadata.get("Language", "").split(",") + if language[0] and not self.__indexing_configured: + self.config_indexing(True, language[0]) super().__enter__() @@ -128,15 +145,97 @@ def start(self): return self - def _validate_metadata(self, name, value): + def validate_metadata( + self, + name: str, + value: Union[bytes, str, datetime.datetime, datetime.date, Iterable[str]], + ): + """Ensures metadata value for name is conform with the openZIM spec on Metadata + + Also enforces recommendations + See https://wiki.openzim.org/wiki/Metadata""" + + # spec doesnt require any value but empty strings are not useful + if name in MANDATORY_ZIM_METADATA_KEYS and not value: + raise ValueError(f"Missing value for {name}") + + # most require/standard and al + if name in ( + "Name", + "Title", + "Creator", + "Publisher", + "Description", + "LongDescription", + "License", + "Relation", + "Relation", + "Flavour", + "Source", + "Scraper", + ) and not isinstance(value, str): + raise ValueError(f"Invalid type for {name}") + + if name == "Title" and len(value) > 30: + raise ValueError(f"{name} is too long.") + + if name == "Date": + if not isinstance(value, (datetime.datetime, datetime.date, str)): + raise ValueError(f"Invalid type for {name}.") + elif isinstance(value, str): + match = re.match( + r"(?P\d{4})-(?P\d{2})-(?P\d{2})", value + ) + try: + datetime.date(**{k: int(v) for k, v in match.groupdict().items()}) + except Exception as exc: + raise ValueError(f"Invalid {name} format: {exc}") + + if name == "Language" and not is_valid_iso_639_3(value): + raise ValueError(f"{value} is not ISO-639-3.") + if name == "Counter": - raise ValueError("You do not need to set Counter.") + raise ValueError(f"{name} cannot be set. libzim sets it.") + + if name == "Description" and len(value) > MAXIMUM_DESCRIPTION_METADATA_LENGTH: + raise ValueError(f"{name} is too long.") - if name == "Description" and len(value) > 80: - raise ValueError("Description is too long.") + if ( + name == "LongDescription" + and len(value) > MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH + ): + raise ValueError(f"{name} is too long.") - if name == "LongDescription" and len(value) > 4000: - raise ValueError("LongDescription is too long.") + if name == "Tags" and ( + not isinstance(value, IterableT) + or not all([isinstance(tag, str) for tag in value]) + ): + raise ValueError(f"Invalid type(s) for {name}") + + if name.startswith("Illustration_"): + match = ILLUSTRATIONS_METADATA_RE.match(name) + if match and not is_valid_image( + image=value, + imformat="PNG", + size=( + int(match.groupdict()["width"]), + int(match.groupdict()["height"]), + ), + ): + raise ValueError( + f"{name} is not a " + f"{match.groupdict()['width']}x{match.groupdict()['height']} " + "PNG Image" + ) + + def add_metadata( + self, + name: str, + content: Union[str, bytes, datetime.date, datetime.datetime], + mimetype: str = "text/plain;charset=UTF-8", + ): + self.validate_metadata(name, content) + super().add_metadata(name, content, mimetype) def config_metadata( self, @@ -158,17 +257,7 @@ def config_metadata( Relation: Optional[str] = None, **extras: str, ): - """ - A chaining functions which configures the metadata of the Creator class. - You must set all mandatory metadata in this phase. - - Parameters: - check out: https://wiki.openzim.org/wiki/Metadata - all the extra metadata must be plain text. - - Returns: - Self - """ + """Sets all mandatory Metadata as well as standard and any other text ones""" self._metadata.update( { "Name": Name, @@ -189,18 +278,10 @@ def config_metadata( } ) self._metadata.update(extras) - language = self._metadata.get("Language", "").split(",") - self.config_indexing(True, language[0]) - return self def config_dev_metadata(self, **extras: str): - """ - A Test function. It will set the default test metadata for a Creator instance. - - Returns: - Self - """ + """Calls config_metadata with default (yet overridable) values for dev""" devel_default_metadata = DEFAULT_DEV_ZIM_METADATA.copy() devel_default_metadata.update(extras) return self.config_metadata(**devel_default_metadata) diff --git a/tests/conftest.py b/tests/conftest.py index 49e706f2..bb1a5e94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,6 +85,11 @@ def file_src(fname): @pytest.fixture(scope="module") def png_image(): + return file_src("commons48.png") + + +@pytest.fixture(scope="module") +def png_image2(): return file_src("commons.png") diff --git a/tests/files/commons48.png b/tests/files/commons48.png new file mode 100644 index 0000000000000000000000000000000000000000..f8ba39a7b8dba7ba3cd6e28f821af1c3e852f474 GIT binary patch literal 3274 zcmV;*3^ntKP)!~LP$cA$u`NXuiw3IvJ6?wOpupz^XA?6-hcn)`?vcPRaMc~p**%6Q6IV>;5A-~ z7Q2gA$Dybg>Y=Wg)91XQ-#2{5=vNaAt9Tv(y$7o9dc?K?K^?w}qkjQ9EJIM2Vu#*; z4N~fWLl@`Z#ARiObzfis^nsxme2P5zNj~27Ix?%Kz~G~0E~EguAqWuE-5}5HF`yNG zi1afvD9p!0^M$Dq3J@)36*Y}xE@2dMW|KItg2A6hnm##yK`bB^jvV`>7`U5o36B5) z{czr!Pd`|pIk<8G1c>Q~5jcpf}8-%R=C9I30Y^shcWd`DrTGzBpguV~vkF+Z*tA8?<*%JNW8$7-B-M*mj**!( zj<_E0JJ4nw&anV#H4A7h4Q7I-2-jwhc2+5f$qUq`0##W2?KqL1UelJ$)Wp2erzI<^y96fBEQZN|yjH*m)yZAXEJCu`vw|?%3cvm^ znb+RSAk*bWO@JtKgf4>kuAK!&9nECkuQ4Xh3PgMlNJkcowOc-v@aTL;!?}e8-WSXV zcOj--vz}i+C`Ua%0(3GWn6$F4$vUl>+Gpr?3IQ|0oUQdBBgwP9!l|Ohli`0bSCmo_ z^C)cUQ1d_(=~f8ABo`YJLJ2`6sMiEbWi(2zr!bzUDZox@JUnqpe`?Q1eB&yX8a0IB zD#FVt--mhu1kkD7nOZO`a2yPU!TLl-j|`4XpwS`br!evy2&=1v6@|b8YC;$*EfLpz zf|mU=OYiSc04?H-V7MF8g>3;J;ncwpo<%6^9EO0fR2%A(#?w!pm+$w6RLFXgjmK$# zeVj}FZUhia#kBvE4q8If=fLFahD1*%7l~0JSz8i?pd+g?+&)gITtuNZ4Oj;$!SQ^5 z2Nk6sx&WP;6o*Ld(>K0E!k0_aw_5k2EjCvson936guNH(<@=#C(BTuAkc^D)dhapl z{L`szJI7*K6N&d_0_!1Eb6g&-_~LUUG!8PZ^jAv!YI)D@>}D?XK`6#+&=I9UR#gGH zJ&N*d4IlbIP-tEJ{q272#sk)Fx=S5^kYo*1*={=(3~` zMntm~`(k*_PsyCYIrYtZBeZ}hrmSMPlS%OJGOkM;LrY*m#fs{Tui=h$P>8wI>TPg~ zN429UR>@j!^$m-@WRRmFs5V|VKtI|p%?yAL>f!5f84*(6n84jk^+h%vDsRm zME0pdDhYB#3?WhXG{ghd>5q^}nSuW^tqhKttQO?W=iYaO#^25YPvaMBrt03x)hN|6A}Id|yMc!G7incV%R+bY2< zSnZPw%XUL_26W9aw9_wncL6C~#j*{TRBk1{&!)hv&eTG$QLHnUinxrl-liXH>BWP| zJ8hsHjs67~aY|_Q8tJ$_O>6Z1A(W6k6k<4xo-|l2(=cMxW_@EtVEmCCNi6%0lf4p- z7x?5-jx9`gtz)W=PR=bYvBNobC*th4!DDcuHCD$Kbf-*;*0wjgs@%>H9qTSa@>y%- zQRso!(&(IrNi3ne-}MUz8_UFWP=dox`4a-tF-INVf_wICn4Rxxc(1Mx&4_?38U(l5 z{JW0}uK)cfQY-!llHL;qdoIced4HSd?36v)b=}zW$EBGt>e9Cp?J||vLQt^fXgpn z2{%j`pdB%UggDA)#WU?Tp=h^wVJ5Y|Q#v*9eprbu^MWTiashlsXP5v&A!u0dt)BqthMH*;z|l`@XBM>7Lwgzd`; zVPf0~kqe-x=CEtOL1l8e7v_!*YUNU`5L5EeX85E$b7E$@<`LhX`a^m>mBIGeX#NO_ zf2#B+Rk1+tKg2fk1^!mgDfkT02oN5-41R_6&ucWbLOAV~nlT05BNt zuA30nQm@#zjmfg*Dm0}mmHwp<_((I8+Mma@m2;%_MdJhC^xChncw=R_(*-wi(Ys-^ z-5S$edg89*n;28^!JziNG(R|SYvmz~KlEi2%$bXrtO}TI*OF#6nf3Nj$~QzKKxr@I z>;2bJLtMi26cfzb@akA^DflQSM})~2*~XM$fWAtoF^!3E1n^9;JsD^04DS|~$mDxT zcdpjt=xzz~Jq<#2k?%?f0CN!^2-; zcFD&3rhT)!xK0cCy);h&aUjk4@Y=$n)5r3zCIrd6`v_4uOzZ5xwVE9>We?FAf#kkX z+y2NckkG^;&G$*yF0~{^!tu>NK|+I;7etlO(LNE4wSTM~bM4_(w?AP%>4#cusrjxQYciZ(y^!dYLAYB!RpVAu=2T$(CBWtTata#C;z!ru)c)a6y+235w1H(r2WnEC*mk9W%Dk`-DxXwB{VyM<=$dM&+=Rn6P&toF~UI)1ymiqRH2( z|3PwH6dl<^t2>|g1^=%I(@I=3WIf;H7)y=az-LnFge&1y9z_P@k)8E>vvAky6EL(m zae7)3+?ECm+d+-8(DqvAkg+w?_;F_6TEuj$ro#S;HmfDf;;OQSwC|TcUw$Y;2#JUw zhR^s47b*{{P1+be+H6ai8)gyUVk7r@BsFYA&S_xpMJxs;ux_fj2l^%_0`vgWo@a23 z?`k`>(4DiA6+naiQz(Znp_ujjd*CKx{ z;=Rqp@!%FDRfDzO^ zLi`_*JAIapC?K@>>7E}afHP?e;V~}>(^B_>N66?xp4TI#PUWIGIFp{txj*swLY`+K zvA%+d@VMOk@)--M*(I$1f6(VPi{J1L;dy7*w+FGryG0Q?aaqneeOSy=X5rC<&781} zTTRG(&VR@S%h`s$L`PjmjI-RpS>E&08UKftxI}dLAK<$`4GYSQpi|0X?J;(XmotE9%8u(19)MD|Fzmo>^-^edK2H!$iFh=;HBMJdR zX7%gqh($ literal 0 HcmV?d00001 diff --git a/tests/image/test_image.py b/tests/image/test_image.py index d7cb4c47..5032e921 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -37,7 +37,12 @@ WebpLow, WebpMedium, ) -from zimscraperlib.image.probing import format_for, get_colors, is_hex_color +from zimscraperlib.image.probing import ( + format_for, + get_colors, + is_hex_color, + is_valid_image, +) from zimscraperlib.image.transformation import resize_image from zimscraperlib.image.utils import save_image @@ -93,7 +98,7 @@ def test_colors_noimage(): def test_colors_png_nopalette(png_image): - assert get_colors(png_image, False) == ("#04649C", "#E7F6FF") + assert get_colors(png_image, False) == ("#04659B", "#E7F6FF") def test_colors_jpg_nopalette(jpg_image): @@ -101,7 +106,7 @@ def test_colors_jpg_nopalette(jpg_image): def test_colors_png_palette(png_image): - assert get_colors(png_image, True) == ("#04649C", "#FFE7E7") + assert get_colors(png_image, True) == ("#9E0404", "#E7F6FF") def test_colors_jpg_palette(jpg_image): @@ -285,8 +290,8 @@ def test_change_image_format_defaults(png_image, jpg_image, tmp_path): "fmt,exp_size", [("png", 128), ("jpg", 128)], ) -def test_create_favicon(png_image, jpg_image, tmp_path, fmt, exp_size): - src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) +def test_create_favicon(png_image2, jpg_image, tmp_path, fmt, exp_size): + src, dst = get_src_dst(tmp_path, fmt, png_image=png_image2, jpg_image=jpg_image) dst = dst.parent.joinpath("favicon.ico") create_favicon(src, dst) @@ -328,12 +333,12 @@ def test_wrong_extension(square_png_image, square_jpg_image, tmp_path, fmt): ["png", "jpg", "gif", "webp"], ) def test_optimize_image_default( - png_image, jpg_image, gif_image, webp_image, tmp_path, fmt + png_image2, jpg_image, gif_image, webp_image, tmp_path, fmt ): src, dst = get_src_dst( tmp_path, fmt, - png_image=png_image, + png_image=png_image2, jpg_image=jpg_image, gif_image=gif_image, webp_image=webp_image, @@ -476,7 +481,7 @@ def test_preset_has_mime_and_ext(): def test_remove_png_transparency(png_image, tmp_path): dst = tmp_path / "out.png" optimize_png(src=png_image, dst=dst, remove_transparency=True) - assert os.path.getsize(dst) == 10686 + assert os.path.getsize(dst) == 2352 def test_jpeg_exif_preserve(jpg_exif_image, tmp_path): @@ -547,3 +552,18 @@ def test_wrong_extension_optim(tmp_path, png_image): shutil.copy(png_image, dst) with pytest.raises(Exception): optimize_jpeg(dst, dst) + + +def test_is_valid_image(png_image, png_image2, jpg_image, font): + assert is_valid_image(png_image, "PNG") + assert not is_valid_image(png_image, "JPEG") + assert is_valid_image(jpg_image, "JPEG") + assert is_valid_image(png_image, "PNG", (48, 48)) + assert not is_valid_image(png_image2, "PNG", (48, 48)) + assert not is_valid_image(b"", "PNG") + assert not is_valid_image(34, "PNG") + assert not is_valid_image(font, "PNG") + with open(png_image, "rb") as fh: + assert is_valid_image(fh.read(), "PNG", (48, 48)) + fh.seek(0) + assert is_valid_image(io.BytesIO(fh.read()), "PNG", (48, 48)) diff --git a/tests/zim/conftest.py b/tests/zim/conftest.py index 0f6c33a3..779cdab0 100644 --- a/tests/zim/conftest.py +++ b/tests/zim/conftest.py @@ -39,8 +39,8 @@ def build_data(tmp_path, png_image): redirects_file = tmp_path / "redirects.tsv" with open(redirects_file, "w") as fh: fh.write(" \tAccueil\tBienvenue !!\twelcome\n") - fh.write(" \tAccueil2\t\tcommons.png\n") - fh.write(" \timage\t\tcommons.png\n") + fh.write(" \tAccueil2\t\tcommons48.png\n") + fh.write(" \timage\t\tcommons48.png\n") build_dir = tmp_path / "build" return { "build_dir": build_dir, @@ -55,7 +55,7 @@ def build_data(tmp_path, png_image): "creator": "test", "publisher": "test", "tags": ["test"], - "redirects": [("picture", "commons.png", "")], + "redirects": [("picture", "commons48.png", "")], "redirects_file": redirects_file, } diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index 060be38a..580e9004 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -39,7 +39,6 @@ def test_get_item(small_zim_file): def test_suggestions(small_zim_file): with Archive(small_zim_file) as zim: - assert zim.get_suggestions_count("test") == 1 assert "main.html" in list(zim.get_suggestions("test")) @@ -56,6 +55,7 @@ def test_search_no_fti(small_zim_file): zim.get_search_results("test") +@pytest.mark.slow def test_search(real_zim_file): with Archive(real_zim_file) as zim: assert zim.get_search_results_count("test") > 0 diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index 698bd331..d8ed4d36 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -18,7 +18,7 @@ def test_fileitem(tmp_path, png_image): # ensure all properties of a FileItem representing a binary are correct article = FileItem(tmp_path, fpath) - assert article.get_path() == "commons.png" + assert article.get_path() == "commons48.png" assert article.get_title() == "" assert article.get_mimetype() == "image/png" @@ -29,7 +29,7 @@ def test_redirects_file(tmp_path, png_image, build_data): build_data["redirects_file"] = tmp_path / "toto.tsv" with open(build_data["redirects_file"], "w") as fh: # write a redirect with a namespace (old ns scheme) - fh.write("A\tAccueil\t\tcommons.png\n") + fh.write("A\tAccueil\t\tcommons48.png\n") # call make_zim_file with redirects_file make_zim_file( diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index bc31cc32..72774098 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +import base64 +import datetime import io import os import pathlib @@ -15,7 +17,11 @@ import pytest from libzim.writer import Compression -from zimscraperlib.constants import DEFAULT_DEV_ZIM_METADATA, UTF8 +from zimscraperlib.constants import ( + DEFAULT_DEV_ZIM_METADATA, + MANDATORY_ZIM_METADATA_KEYS, + UTF8, +) from zimscraperlib.download import save_large_file, stream_file from zimscraperlib.filesystem import delete_callback from zimscraperlib.zim import Archive, Creator, StaticItem, URLItem @@ -112,7 +118,7 @@ def test_create_without_workaround(tmp_path): def test_noindexlanguage(tmp_path): fpath = tmp_path / "test.zim" creator = Creator(fpath, "welcome").config_dev_metadata(Language="bam") - creator.config_indexing(False, "") + creator.config_indexing(False) with creator as creator: creator.add_item(StaticItem(path="welcome", content="hello")) creator.add_item_for("index", "Index", content="-", mimetype="text/html") @@ -487,7 +493,7 @@ def test_without_metadata(tmp_path): def test_check_metadata(tmp_path): - with pytest.raises(ValueError, match="You do not need to set Counter."): + with pytest.raises(ValueError, match="Counter cannot be set"): Creator(tmp_path, "").config_dev_metadata(Counter=1).start() with pytest.raises(ValueError, match="Description is too long."): @@ -538,7 +544,7 @@ def test_config_metadata(tmp_path, png_image): == "All articles (without images) from the english Wikipedia" ) assert ( - reader.get_text_metadata("Longdescription") + reader.get_text_metadata("LongDescription") == "This ZIM file contains all articles (without images)" " from the english Wikipedia by 2009-11-10. The topics are..." ) @@ -553,4 +559,88 @@ def test_config_metadata(tmp_path, png_image): assert reader.get_text_metadata("Source") == "https://en.wikipedia.org/" assert reader.get_text_metadata("Scraper") == "mwoffliner 1.2.3" assert reader.get_metadata("Illustration_48x48@1") == png_data - assert reader.get_text_metadata("Testmetadata") == "Test Metadata" + assert reader.get_text_metadata("TestMetadata") == "Test Metadata" + + +@pytest.mark.parametrize( + "name,value,valid", + [ + ("Name", 4, False), + ("Title", 4, False), + ("Creator", 4, False), + ("Publisher", 4, False), + ("Description", 4, False), + ("LongDescription", 4, False), + ("License", 4, False), + ("Relation", 4, False), + ("Relation", 4, False), + ("Flavour", 4, False), + ("Source", 4, False), + ("Scraper", 4, False), + ("Title", "X" * 30, True), + ("Title", "X" * 31, False), + ("Date", 4, False), + ("Date", datetime.datetime.now(), True), + ("Date", datetime.datetime(1969, 12, 31, 23, 59), True), + ("Date", datetime.date(1969, 12, 31), True), + ("Date", datetime.date.today(), True), + ("Date", "1969-12-31", True), + ("Date", "1969-13-31", False), + ("Date", "2023/02/29", False), + ("Language", "xxx", False), + ("Language", "rmr", False), + ("Language", "eng", True), + ("Language", "fra", True), + ("Language", "bam", True), + ("Language", "fr", False), + ("Language", "en", False), + ("Counter", "1", False), + ("Description", "X" * 80, True), + ("Description", "X" * 81, False), + ("LongDescription", "X" * 4000, True), + ("LongDescription", "X" * 4001, False), + ("Tags", 4, False), + ("Tags", ["wikipedia", 4, "football"], False), + ("Tags", ("wikipedia", "football"), True), + ("Tags", ["wikipedia", "football"], True), + ("Tags", "wikipedia;football", True), + # 1x1 PNG image + ( + "Illustration_48x48@1", + base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBB" + "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAA9JREFUeNpi+P//P0CAAQAF/gL+Lc6J7gAAAABJ" + "RU5ErkJggg==" + ), + False, + ), + ( + "Illustration_48x48@1", + DEFAULT_DEV_ZIM_METADATA["Illustration_48x48_at_1"], + True, + ), + ( + "Illustration_96x96@1", + DEFAULT_DEV_ZIM_METADATA["Illustration_48x48_at_1"], + False, + ), + ] + + [(name, "", False) for name in MANDATORY_ZIM_METADATA_KEYS], +) +def test_validate_metadata(tmp_path, name, value, valid): + if valid: + Creator(tmp_path / "_.zim", "").validate_metadata(name, value) + else: + with pytest.raises(ValueError): + Creator(tmp_path / "_.zim", "").validate_metadata(name, value) + + +def test_config_indexing(tmp_path): + with pytest.raises(ValueError): + Creator(tmp_path / "_.zim", "").config_indexing(True, "fr") + with pytest.raises(ValueError): + Creator(tmp_path / "_.zim", "").config_indexing(True, "") + assert Creator(tmp_path / "_.zim", "").config_indexing(True, "fra") + assert Creator(tmp_path / "_.zim", "").config_indexing(True, "bam") + assert Creator(tmp_path / "_.zim", "").config_indexing(False, "bam") + assert Creator(tmp_path / "_.zim", "").config_indexing(False) From e155063783cbca7e43d0f5c17f35903a283bcfa0 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Mar 2023 16:08:43 +0000 Subject: [PATCH 028/335] Refactored around a new metadata module --- CHANGELOG.md | 1 + src/zimscraperlib/zim/creator.py | 100 ++++++-------------------- src/zimscraperlib/zim/metadata.py | 112 ++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 77 deletions(-) create mode 100644 src/zimscraperlib/zim/metadata.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c9afaebf..ed0f6889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `zim.creator.Creator.config_metadata` method (returning Self) exposing all mandatory Metdata, all standard ones and allowing extra text metdadata. - `zim.creator.Creator.config_dev_metadata` method setting stub metdata for all mandatory ones (allowing overrides) +- `zim.metadata` module with a list of per-metadata validation functions - `zim.creator.Creator.validate_metadata` (called on `start`) to verify metadata respects the spec (and its recommendations) - `zim.filesystem.make_zim_file` accepts a new optional `long_description` param. - `i18n.is_valid_iso_639_3` to check ISO-639-3 codes diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 679d6cbf..7b0ec5fc 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -22,7 +22,6 @@ import pathlib import re import weakref -from collections.abc import Iterable as IterableT from typing import Any, Callable, Iterable, Optional, Tuple, Union import libzim.writer @@ -30,16 +29,24 @@ from ..constants import ( DEFAULT_DEV_ZIM_METADATA, FRONT_ARTICLE_MIMETYPES, - ILLUSTRATIONS_METADATA_RE, MANDATORY_ZIM_METADATA_KEYS, - MAXIMUM_DESCRIPTION_METADATA_LENGTH, - MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH, ) from ..filesystem import delete_callback, get_content_mimetype, get_file_mimetype from ..i18n import is_valid_iso_639_3 -from ..image.probing import is_valid_image from ..types import get_mime_for_name from .items import StaticItem +from .metadata import ( + validate_counter, + validate_date, + validate_description, + validate_illustrations, + validate_language, + validate_longdescription, + validate_required_values, + validate_standard_str_types, + validate_tags, + validate_title, +) DUPLICATE_EXC_STR = re.compile( r"^Impossible to add(.+)" @@ -155,78 +162,17 @@ def validate_metadata( Also enforces recommendations See https://wiki.openzim.org/wiki/Metadata""" - # spec doesnt require any value but empty strings are not useful - if name in MANDATORY_ZIM_METADATA_KEYS and not value: - raise ValueError(f"Missing value for {name}") - - # most require/standard and al - if name in ( - "Name", - "Title", - "Creator", - "Publisher", - "Description", - "LongDescription", - "License", - "Relation", - "Relation", - "Flavour", - "Source", - "Scraper", - ) and not isinstance(value, str): - raise ValueError(f"Invalid type for {name}") - - if name == "Title" and len(value) > 30: - raise ValueError(f"{name} is too long.") - - if name == "Date": - if not isinstance(value, (datetime.datetime, datetime.date, str)): - raise ValueError(f"Invalid type for {name}.") - elif isinstance(value, str): - match = re.match( - r"(?P\d{4})-(?P\d{2})-(?P\d{2})", value - ) - try: - datetime.date(**{k: int(v) for k, v in match.groupdict().items()}) - except Exception as exc: - raise ValueError(f"Invalid {name} format: {exc}") - - if name == "Language" and not is_valid_iso_639_3(value): - raise ValueError(f"{value} is not ISO-639-3.") - - if name == "Counter": - raise ValueError(f"{name} cannot be set. libzim sets it.") - - if name == "Description" and len(value) > MAXIMUM_DESCRIPTION_METADATA_LENGTH: - raise ValueError(f"{name} is too long.") - - if ( - name == "LongDescription" - and len(value) > MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH - ): - raise ValueError(f"{name} is too long.") - - if name == "Tags" and ( - not isinstance(value, IterableT) - or not all([isinstance(tag, str) for tag in value]) - ): - raise ValueError(f"Invalid type(s) for {name}") - - if name.startswith("Illustration_"): - match = ILLUSTRATIONS_METADATA_RE.match(name) - if match and not is_valid_image( - image=value, - imformat="PNG", - size=( - int(match.groupdict()["width"]), - int(match.groupdict()["height"]), - ), - ): - raise ValueError( - f"{name} is not a " - f"{match.groupdict()['width']}x{match.groupdict()['height']} " - "PNG Image" - ) + validate_required_values(name, value) + validate_standard_str_types(name, value) + + validate_title(name, value) + validate_date(name, value) + validate_language(name, value) + validate_counter(name, value) + validate_description(name, value) + validate_longdescription(name, value) + validate_tags(name, value) + validate_illustrations(name, value) def add_metadata( self, diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py new file mode 100644 index 00000000..56abedb8 --- /dev/null +++ b/src/zimscraperlib/zim/metadata.py @@ -0,0 +1,112 @@ +import datetime +import re +from collections.abc import Iterable as IterableT +from typing import Any, Iterable, Union + +from ..constants import ( + ILLUSTRATIONS_METADATA_RE, + MANDATORY_ZIM_METADATA_KEYS, + MAXIMUM_DESCRIPTION_METADATA_LENGTH, + MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH, +) +from ..i18n import is_valid_iso_639_3 +from ..image.probing import is_valid_image + + +def validate_required_values(name: str, value: Any): + """ensures required ones have a value (spec doesnt requires it but makes sense)""" + if name in MANDATORY_ZIM_METADATA_KEYS and not value: + raise ValueError(f"Missing value for {name}") + + +def validate_standard_str_types(name: str, value: str): + """ensures standard string metadata are indeed str""" + if name in ( + "Name", + "Title", + "Creator", + "Publisher", + "Description", + "LongDescription", + "License", + "Relation", + "Relation", + "Flavour", + "Source", + "Scraper", + ) and not isinstance(value, str): + raise ValueError(f"Invalid type for {name}") + + +def validate_title(name: str, value: str): + """ensures Title metadata is within recommended length""" + if name == "Title" and len(value) > 30: + raise ValueError(f"{name} is too long.") + + +def validate_date(name: str, value: Union[datetime.datetime, datetime.date, str]): + """ensures Date metadata can be casted to an ISO 8601 string""" + if name == "Date": + if not isinstance(value, (datetime.datetime, datetime.date, str)): + raise ValueError(f"Invalid type for {name}.") + elif isinstance(value, str): + match = re.match(r"(?P\d{4})-(?P\d{2})-(?P\d{2})", value) + try: + datetime.date(**{k: int(v) for k, v in match.groupdict().items()}) + except Exception as exc: + raise ValueError(f"Invalid {name} format: {exc}") + + +def validate_language(name: str, value: Union[Iterable[str], str]): + """ensures Language metadata is a single or list of ISO-639-3 codes""" + if name == "Language" and not is_valid_iso_639_3(value): + raise ValueError(f"{value} is not ISO-639-3.") + + +def validate_counter(name: str, value: str): + """ensures Counter metadata is not manually set""" + if name == "Counter": + raise ValueError(f"{name} cannot be set. libzim sets it.") + + +def validate_description(name: str, value: str): + """ensures Description metadata is with required length""" + if name == "Description" and len(value) > MAXIMUM_DESCRIPTION_METADATA_LENGTH: + raise ValueError(f"{name} is too long.") + + +def validate_longdescription(name: str, value: str): + """ensures LongDescription metadata is with required length""" + if ( + name == "LongDescription" + and len(value) > MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH + ): + raise ValueError(f"{name} is too long.") + + +def validate_tags(name: str, value: Union[Iterable[str], str]): + """ensures Tags metadata is either one or a list of strings""" + if name == "Tags" and ( + not isinstance(value, IterableT) + or not all([isinstance(tag, str) for tag in value]) + ): + raise ValueError(f"Invalid type(s) for {name}") + + +def validate_illustrations(name: str, value: bytes): + """ensures illustrations are PNG images or the advertised size""" + if name.startswith("Illustration_"): + match = ILLUSTRATIONS_METADATA_RE.match(name) + if match and not is_valid_image( + image=value, + imformat="PNG", + size=( + int(match.groupdict()["width"]), + int(match.groupdict()["height"]), + ), + ): + raise ValueError( + f"{name} is not a " + f"{match.groupdict()['width']}x{match.groupdict()['height']} " + "PNG Image" + ) From b132f5c6362a0363a25f9f82ef6f1656cf671be6 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Mar 2023 16:16:32 +0000 Subject: [PATCH 029/335] removed deprecated contextmanager use of pathlib (py3.13) --- tests/video/test_video.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/video/test_video.py b/tests/video/test_video.py index aec9afd5..ca8ed2fe 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -128,7 +128,8 @@ def test_config_build_from(): ], ) def test_get_media_info(media_format, media, expected, test_files): - with tempfile.TemporaryDirectory() as t, pathlib.Path(t).joinpath(media) as src: + with tempfile.TemporaryDirectory() as t: + src = pathlib.Path(t).joinpath(media) shutil.copy2(test_files[media_format], src) assert get_media_info(src) == expected @@ -342,7 +343,8 @@ def test_preset_voice_mp3_low(): ], ) def test_reencode_media(src, dest, ffmpeg_args, expected, test_files): - with tempfile.TemporaryDirectory() as t, pathlib.Path(t) as temp_dir: + with tempfile.TemporaryDirectory() as t: + temp_dir = pathlib.Path(t) copy_media_and_reencode(temp_dir, src, dest, ffmpeg_args, test_files) converted_details = get_media_info(temp_dir.joinpath(dest)) assert expected["duration"] == converted_details["duration"] @@ -368,9 +370,10 @@ def test_reencode_media(src, dest, ffmpeg_args, expected, test_files): ], ) def test_reencode_delete_src(src, dest, ffmpeg_args, delete_src, test_files): - with tempfile.TemporaryDirectory() as t, pathlib.Path( - t - ) as temp_dir, temp_dir.joinpath(src) as src_path: + with tempfile.TemporaryDirectory() as t: + temp_dir = pathlib.Path(t) + src_path = temp_dir.joinpath(src) + copy_media_and_reencode( temp_dir, src, dest, ffmpeg_args, test_files, delete_src=delete_src ) @@ -401,7 +404,8 @@ def test_reencode_delete_src(src, dest, ffmpeg_args, delete_src, test_files): def test_reencode_return_ffmpeg_output( src, dest, ffmpeg_args, return_output, test_files ): - with tempfile.TemporaryDirectory() as t, pathlib.Path(t) as temp_dir: + with tempfile.TemporaryDirectory() as t: + temp_dir = pathlib.Path(t) ret = copy_media_and_reencode( temp_dir, src, @@ -437,7 +441,8 @@ def test_reencode_return_ffmpeg_output( ], ) def test_reencode_failsafe(src, dest, ffmpeg_args, failsafe, test_files): - with tempfile.TemporaryDirectory() as t, pathlib.Path(t) as temp_dir: + with tempfile.TemporaryDirectory() as t: + temp_dir = pathlib.Path(t) if not failsafe: with pytest.raises(subprocess.CalledProcessError) as exc_info: copy_media_and_reencode( From 0d4b1ac9427c98bed3a2c85ca3d0c0fb7a3d53f5 Mon Sep 17 00:00:00 2001 From: Fledge Shiu Date: Mon, 20 Mar 2023 23:07:22 -0400 Subject: [PATCH 030/335] Add Python 3.11 to CI. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f09b87c8..7d5fa26d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle From 7c243a03386ef0581315d897f2d0f27debc485fd Mon Sep 17 00:00:00 2001 From: Fledge Shiu Date: Wed, 22 Mar 2023 12:11:23 -0400 Subject: [PATCH 031/335] Update .gitignore. --- .gitignore | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index b6e47617..9e773ced 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,55 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux +# Edit at https://www.toptal.com/developers/gitignore?templates=python,macos,windows,linux + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -20,7 +72,6 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -50,6 +101,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -72,6 +124,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -82,7 +135,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -91,7 +146,22 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# PEP 582; used by e.g. github.com/David-OConnor/pyflow +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff @@ -127,3 +197,54 @@ dmypy.json # Pyre type checker .pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux From 88fbe759ac391addcdc397f768b41b12255daa9d Mon Sep 17 00:00:00 2001 From: Fledge Shiu Date: Wed, 22 Mar 2023 12:11:48 -0400 Subject: [PATCH 032/335] Better QA configuration. --- .github/workflows/ci.yml | 2 +- .pre-commit-config.yaml | 10 +++++----- README.md | 10 ++++++++++ tox.ini | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d5fa26d..e2f0480d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,6 @@ jobs: python -m pip install --upgrade pip pip install -U tox - name: Run tox - run: tox -e py38,black,coverage + run: tox -e py38,black,isort,flake8,coverage env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 973eef29..bef63f03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,21 +2,21 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/pycqa/isort - rev: "5.9.3" + rev: "5.12.0" hooks: - id: isort args: ["--profile", "black", "--filter-files"] - repo: https://github.com/psf/black - rev: "22.3.0" + rev: "23.1.0" hooks: - id: black -- repo: https://gitlab.com/pycqa/flake8 - rev: "3.9.2" +- repo: https://github.com/pycqa/flake8 + rev: "6.0.0" hooks: - id: flake8 args: ["--max-line-length", "88", "--extend-ignore=E203"] diff --git a/README.md b/README.md index 3616db3c..ab3478a9 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,16 @@ sudo apt install libmagic1 wget ffmpeg \ libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle ``` +# Contribution + +```shell +pip -r requirements.txt +pip install tox pre-commit +pre-commit install +# For tests +tox +``` + # Users Non-exhaustive list of scrapers using it (check status when updating API): diff --git a/tox.ini b/tox.ini index b3977b7f..1b65ca01 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, py310, py311, black, coverage +envlist = py36, py37, py38, py39, py310, py311, black, isort, flake8, coverage [testenv] passenv = @@ -15,9 +15,19 @@ commands = [testenv:black] deps = - black>=22.3.0,<23 + black>=23.1.0,<24 commands = black --check . +[testenv:isort] +deps = + isort>=5.12.0,<5.13 +commands = isort --profile black --check src tests + +[testenv:flake8] +deps = + flake8>=6.0.0,<7.0 +commands = flake8 src --count --max-line-length=88 --statistics --extend-ignore=E203 + [testenv:coverage] passenv = CODECOV_TOKEN From 28ee9e3f18c25614160d1749cebc6e1c59a9f386 Mon Sep 17 00:00:00 2001 From: Fledge Shiu Date: Wed, 22 Mar 2023 12:14:23 -0400 Subject: [PATCH 033/335] Format code. --- src/zimscraperlib/image/optimization.py | 3 --- tests/conftest.py | 1 - tests/filesystem/test_filesystem.py | 1 - tests/logging/test_logging.py | 1 - tests/ogvjs/test_ogvjs.py | 1 - 5 files changed, 7 deletions(-) diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 478136c5..0a21e922 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -61,7 +61,6 @@ def optimize_png( background_color: Optional[Tuple[int, int, int]] = (255, 255, 255), **options, ) -> Union[pathlib.Path, io.BytesIO]: - """method to optimize PNG files using a pure python external optimizer Arguments: @@ -107,7 +106,6 @@ def optimize_jpeg( keep_exif: Optional[bool] = True, **options, ) -> Union[pathlib.Path, io.BytesIO]: - """method to optimize JPEG files using a pure python external optimizer quality: JPEG quality (integer between 1 and 100) values: 50 | 55 | 35 | 100 | XX @@ -155,7 +153,6 @@ def optimize_jpeg( dst.seek(0) if keep_exif and had_exif: - piexif.transplant( exif_src=str(src.resolve()) if isinstance(src, pathlib.Path) diff --git a/tests/conftest.py b/tests/conftest.py index bb1a5e94..208d370a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -174,7 +174,6 @@ def real_zim_file(tmpdir_factory): @pytest.fixture(scope="session") def undecodable_byte_stream(): - return ( b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" b"\x00\x00\x01\x00\xfb\xff~\x00\x08\x00\xfd\xff\x01\x00\x00\x00" diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index bdd1ee89..bcbabcd5 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -25,7 +25,6 @@ def test_content_mimetype(png_image, jpg_image, undecodable_byte_stream): def test_content_mimetype_fallback(monkeypatch, undecodable_byte_stream): - # use raw function first to test actual code assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" diff --git a/tests/logging/test_logging.py b/tests/logging/test_logging.py index 3c91c0cf..e207d4c1 100644 --- a/tests/logging/test_logging.py +++ b/tests/logging/test_logging.py @@ -80,7 +80,6 @@ def test_critical_level(random_id, console): def test_format(): - # assert is_in_log(message, console) # "[%(asctime)s] %(levelname)s:%(message)s" pass diff --git a/tests/ogvjs/test_ogvjs.py b/tests/ogvjs/test_ogvjs.py index 806f0c26..fe9087f2 100644 --- a/tests/ogvjs/test_ogvjs.py +++ b/tests/ogvjs/test_ogvjs.py @@ -13,7 +13,6 @@ def prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): - videojs_zip = tmp_path / "video-js-7.6.4.zip" if not videojs_zip.exists(): save_large_file(videojs_url, videojs_zip) From 75eeae581a038d8d9c22522ee2c6651b0ed587d9 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 31 Mar 2023 10:58:56 +0000 Subject: [PATCH 034/335] added release workflow --- .github/workflows/release.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..48e21e29 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,34 @@ +name: release +on: + release: + types: [published] + tags: + - v* + +env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + +jobs: + release: + environment: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: x64 + + - name: Build sdist and wheel + run: | + pip install --upgrade pip build wheel + python3 -m build + + - name: Push release to PyPI + if: github.event_name == 'release' + run: | + pip install --upgrade twine + twine check dist/* + twine upload dist/* From 42eeea668bea9e6e5eb11c390339701a998b5766 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 31 Mar 2023 11:00:42 +0000 Subject: [PATCH 035/335] releasing 3.0.0 --- CHANGELOG.md | 2 +- setup.py | 2 +- src/zimscraperlib/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed0f6889..b9841611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [3.0.0] - 2023-03-31 ⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file` diff --git a/setup.py b/setup.py index 0b02de6f..ecf74797 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ def read(*names, **kwargs): "console_scripts": ["fix_ogvjs_dist=zimscraperlib.fix_ogvjs_dist:run"] }, classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3.6", diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 7ec1d6db..4a36342f 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -2.1.0 +3.0.0 From a7185fd334ab750e3931d571ce293fc586561950 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 May 2023 10:06:06 +0000 Subject: [PATCH 036/335] Using pylibzim 3.1.0 Updating suggestion tests as punctuation in suggestion text is not considered noise anymore and thus affects suggestion results. See https://github.com/openzim/libzim/pull/765 https://github.com/openzim/libzim/blob/main/src/writer/xapianIndexer.cpp#L131 As well as other commits in March 2023 --- CHANGELOG.md | 6 ++++++ requirements.txt | 2 +- tests/zim/test_zim_creator.py | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9841611..efe745e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +### Changed + +- Using pylibzim `3.1.0` + ## [3.0.0] - 2023-03-31 ⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file` diff --git a/requirements.txt b/requirements.txt index 7ea268b8..9978dccf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ colorthief==0.2.1 python-resize-image>=1.1.19,<1.2 Babel>=2.9,<3.0 file-magic>=0.4.0,<0.5 -libzim>=3.0.0,<3.1 +libzim>=3.1.0,<3.2 beautifulsoup4>=4.9.3,<4.10 lxml>=4.6.3,<4.10 optimize-images>=1.3.6,<1.6 diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 72774098..6cc5a22b 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -89,10 +89,10 @@ def test_zim_creator(tmp_path, png_image, html_file, html_str): # make sure titles were indexed (html with title for xapian ; redirects are not) # see https://github.com/openzim/python-libzim/issues/125 # see https://github.com/openzim/libzim/issues/642 - assert "home2" not in list(reader.get_suggestions("Home !!")) # no is_front > False - assert "home3" in list(reader.get_suggestions("Home !!")) # is_front=True - assert "home4" not in list(reader.get_suggestions("Home !!")) # is_front=False - assert "images/yahoo.png" in list(reader.get_suggestions("Home !!")) # is_frontTrue + assert "home2" not in list(reader.get_suggestions("Home")) # no is_front > False + # assert "home3" in list(reader.get_suggestions("Home")) # is_front=True + assert "home4" not in list(reader.get_suggestions("Home")) # is_front=False + assert "images/yahoo.png" in list(reader.get_suggestions("Home")) # is_frontTrue # make sure full text was indexed assert reader.get_search_results_count("PDF doc") >= 1 From 3349b83ee5ad309a281ee5b1e7e901739f6ddaf1 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 May 2023 10:18:28 +0000 Subject: [PATCH 037/335] Dropped python 3.6 support --- .github/workflows/ci.yml | 6 +++--- .github/workflows/release.yml | 2 +- CHANGELOG.md | 4 ++++ setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2f0480d..c19452e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,11 +4,11 @@ on: [push, pull_request] jobs: unit-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: max-parallel: 3 matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle @@ -33,7 +33,7 @@ jobs: run: tox -e $toxpyv qa: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 48e21e29..d72c5d8d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ env: jobs: release: environment: release - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - name: Set up Python 3.11 diff --git a/CHANGELOG.md b/CHANGELOG.md index efe745e8..c19a5b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Using pylibzim `3.1.0` +### Removed + +- Dropped support for Python 3.6 + ## [3.0.0] - 2023-03-31 ⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file` diff --git a/setup.py b/setup.py index ecf74797..9a0d0a65 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ def read(*names, **kwargs): "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -50,5 +49,5 @@ def read(*names, **kwargs): "Programming Language :: Python :: 3.11", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ], - python_requires=">=3.6", + python_requires=">=3.7", ) diff --git a/tox.ini b/tox.ini index 1b65ca01..9a7f38ac 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, py310, py311, black, isort, flake8, coverage +envlist = py37, py38, py39, py310, py311, black, isort, flake8, coverage [testenv] passenv = From 95b52c63039423576867cfffa8f429bb7b253cff Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 May 2023 10:34:48 +0000 Subject: [PATCH 038/335] Use text/javascript MIME type `application/javascript` is obsolete and has long been replaced with `text/javascript`. Now using it where we input it (in tests). Also accounting for both in tests involving filesystem as tests can be run on a version of libmagic that returns the obsolete version. --- tests/zim/conftest.py | 1 - tests/zim/test_fs.py | 5 ++++- tests/zim/test_libkiwix.py | 11 ++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/zim/conftest.py b/tests/zim/conftest.py index 779cdab0..d73b37f4 100644 --- a/tests/zim/conftest.py +++ b/tests/zim/conftest.py @@ -63,7 +63,6 @@ def build_data(tmp_path, png_image): @pytest.fixture(scope="function") def counters(): return { - "application/javascript": 8, "text/html": 3, "application/warc-headers": 28364, "text/html;raw=true": 6336, diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index d8ed4d36..41be535e 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -81,7 +81,10 @@ def test_make_zim_file_working(build_data, png_image): assert reader.entry_count == 8 # includes redirect assert reader.get_item("style.css").mimetype == "text/css" - assert reader.get_item("app.js").mimetype == "application/javascript" + assert reader.get_item("app.js").mimetype in ( + "text/javascript", + "application/javascript", + ) assert reader.get_suggestions_count("bienvenue") == 0 assert reader.get_suggestions_count("coucou") == 1 assert "welcome" in list(reader.get_suggestions("coucou")) diff --git a/tests/zim/test_libkiwix.py b/tests/zim/test_libkiwix.py index aa3c29a7..1e5a2e24 100644 --- a/tests/zim/test_libkiwix.py +++ b/tests/zim/test_libkiwix.py @@ -13,16 +13,14 @@ def test_geline_nodelim(): - string = "application/javascript=8;text/html=3;application/warc-headers=28364;" + string = "text/javascript=8;text/html=3;application/warc-headers=28364;" ins = io.StringIO(string) assert getline(ins) == (True, string) def test_getline(): - ins = io.StringIO( - "application/javascript=8;text/html=3;application/warc-headers=28364;" - ) - assert getline(ins, ";") == (False, "application/javascript=8") + ins = io.StringIO("text/javascript=8;text/html=3;application/warc-headers=28364;") + assert getline(ins, ";") == (False, "text/javascript=8") assert getline(ins, ";") == (False, "text/html=3") assert getline(ins, ";") == (False, "application/warc-headers=28364") assert getline(ins, ";") == (True, "") @@ -40,14 +38,13 @@ def test_getline(): {"foo": 1, "text/html;raw=true": 50, "bar": 2}, ), ( - "application/javascript=8;text/html=3;application/warc-headers=28364;" + "text/html=3;application/warc-headers=28364;" "text/html;raw=true=6336;text/css=47;text/javascript=98;image/png=968;" "image/webp=24;application/json=3694;image/gif=10274;image/jpeg=1582;" "font/woff2=25;text/plain=284;application/atom+xml=247;" "application/x-www-form-urlencoded=9;video/mp4=9;" "application/x-javascript=7;application/xml=1;image/svg+xml=5", { - "application/javascript": 8, "text/html": 3, "application/warc-headers": 28364, "text/html;raw=true": 6336, From 1957d2ff7f93603492a1cd6da6041a7ad7f6b90e Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 May 2023 12:21:06 +0000 Subject: [PATCH 039/335] More use of text/javascript --- tests/types/test_types.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/types/test_types.py b/tests/types/test_types.py index 2884bbc3..3809be11 100644 --- a/tests/types/test_types.py +++ b/tests/types/test_types.py @@ -35,7 +35,12 @@ def test_constants(): ("assets/ogv.wasm", "application/wasm", None, None), # make sure our custom mapping is used ("assets/subtite.vtt", "text/vtt", None, None), - ("assets/jquery.min.js", "application/javascript", None, None), + ( + "assets/jquery.min.js", + ("application/javascript", "text/javascript"), + None, + None, + ), ("assets/test.css", "text/css", None, None), ], ) @@ -45,4 +50,7 @@ def test_mime_for_name(filename, fallback, expected_mime, no_ext_to): kwargs.update({"fallback": fallback}) if no_ext_to is not None: kwargs.update({"no_ext_to": no_ext_to}) - assert get_mime_for_name(filename, **kwargs) == expected_mime + if isinstance(expected_mime, tuple): + assert get_mime_for_name(filename, **kwargs) in expected_mime + else: + assert get_mime_for_name(filename, **kwargs) == expected_mime From aee1f9e282938a9a9d7ce8864c2cda4caa8474a2 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 2 May 2023 12:25:27 +0000 Subject: [PATCH 040/335] Updated undecodable_byte_stream fixture Previous fixture was now discovered as a valid mime by libmagic in Ubuntu:22.04 --- tests/conftest.py | 135 ++-------------------------- tests/filesystem/test_filesystem.py | 2 +- 2 files changed, 7 insertions(+), 130 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 208d370a..cece10ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -174,134 +174,11 @@ def real_zim_file(tmpdir_factory): @pytest.fixture(scope="session") def undecodable_byte_stream(): + """bytes that is not recognized by some libmagic and raises UnicodeDecodeError""" return ( - b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x01\x00\xfb\xff~\x00\x08\x00\xfd\xff\x01\x00\x00\x00" - b"\xff\xff\x03\x00\xf6\xffz\x00\x12\x00\xfa\xff\x02\x00\x00\x00" - b"\xff\xff\x04\x00\xf3\xffv\x00\x1b\x00\xf7\xff\x03\x00\xff\xff" - b"\xff\xff\x04\x00\xf0\xffp\x00%\x00\xf5\xff\x04\x00\xff\xff" - b"\xff\xff\x05\x00\xee\xffi\x000\x00\xf2\xff\x04\x00\xff\xff" - b"\xff\xff\x05\x00\xed\xffa\x00:\x00\xf0\xff\x05\x00\xff\xff" - b"\xff\xff\x06\x00\xed\xffX\x00D\x00\xee\xff\x05\x00\xff\xff" - b"\xff\xff\x06\x00\xed\xffN\x00N\x00\xed\xff\x06\x00\xff\xff" - b"\xff\xff\x05\x00\xee\xffD\x00X\x00\xed\xff\x06\x00\xff\xff" - b"\xff\xff\x05\x00\xf0\xff:\x00a\x00\xed\xff\x05\x00\xff\xff" - b"\xff\xff\x04\x00\xf2\xff0\x00i\x00\xee\xff\x05\x00\xff\xff" - b"\xff\xff\x04\x00\xf5\xff%\x00p\x00\xf0\xff\x04\x00\xff\xff" - b"\xff\xff\x03\x00\xf7\xff\x1b\x00v\x00\xf3\xff\x04\x00\xff\xff" - b"\x00\x00\x02\x00\xfa\xff\x12\x00z\x00\xf6\xff\x03\x00\xff\xff" - b"\x00\x00\x01\x00\xfd\xff\x08\x00~\x00\xfb\xff\x01\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xfd\xff\xff\xff \x00@\x00&\x00\x01\x00\xfd\xff\x00\x00" - b"\xfe\xff\xfe\xff\x1d\x00?\x00)\x00\x02\x00\xfd\xff\x00\x00" - b"\xfe\xff\xfe\xff\x1a\x00?\x00+\x00\x04\x00\xfc\xff\x00\x00" - b"\xfe\xff\xfd\xff\x18\x00>\x00.\x00\x05\x00\xfc\xff\x00\x00" - b"\xfe\xff\xfd\xff\x15\x00<\x001\x00\x07\x00\xfc\xff\x00\x00" - b"\xff\xff\xfc\xff\x12\x00;\x003\x00\t\x00\xfc\xff\x00\x00" - b"\xff\xff\xfc\xff\x10\x009\x005\x00\x0c\x00\xfc\xff\xff\xff" - b"\xff\xff\xfc\xff\x0e\x007\x007\x00\x0e\x00\xfc\xff\xff\xff" - b"\xff\xff\xfc\xff\x0c\x005\x009\x00\x10\x00\xfc\xff\xff\xff" - b"\x00\x00\xfc\xff\t\x003\x00;\x00\x12\x00\xfc\xff\xff\xff" - b"\x00\x00\xfc\xff\x07\x001\x00<\x00\x15\x00\xfd\xff\xfe\xff" - b"\x00\x00\xfc\xff\x05\x00.\x00>\x00\x18\x00\xfd\xff\xfe\xff" - b"\x00\x00\xfc\xff\x04\x00+\x00?\x00\x1a\x00\xfe\xff\xfe\xff" - b"\x00\x00\xfd\xff\x02\x00)\x00?\x00\x1d\x00\xfe\xff\xfe\xff" - b"\x00\x00\xfd\xff\x01\x00&\x00@\x00 \x00\xff\xff\xfd\xff" - b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xff\xff\x03\x00\xf9\xff\x7f\x00\x08\x00\xfd\xff\x01\x00\x00\x00" - b"\xfe\xff\x05\x00\xf3\xff}\x00\x11\x00\xfa\xff\x03\x00\xff\xff" - b"\xfd\xff\x07\x00\xef\xffy\x00\x1b\x00\xf6\xff\x05\x00\xfe\xff" - b"\xfc\xff\t\x00\xec\xffs\x00%\x00\xf3\xff\x06\x00\xfe\xff" - b"\xfc\xff\n\x00\xe9\xffl\x000\x00\xf0\xff\x08\x00\xfd\xff" - b"\xfc\xff\n\x00\xe8\xffd\x00;\x00\xed\xff\t\x00\xfd\xff" - b"\xfc\xff\x0b\x00\xe8\xffZ\x00F\x00\xeb\xff\n\x00\xfc\xff" - b"\xfc\xff\x0b\x00\xe9\xffP\x00P\x00\xe9\xff\x0b\x00\xfc\xff" - b"\xfc\xff\n\x00\xeb\xffF\x00Z\x00\xe8\xff\x0b\x00\xfc\xff" - b"\xfd\xff\t\x00\xed\xff;\x00d\x00\xe8\xff\n\x00\xfc\xff" - b"\xfd\xff\x08\x00\xf0\xff0\x00l\x00\xe9\xff\n\x00\xfc\xff" - b"\xfe\xff\x06\x00\xf3\xff%\x00s\x00\xec\xff\t\x00\xfc\xff" - b"\xfe\xff\x05\x00\xf6\xff\x1b\x00y\x00\xef\xff\x07\x00\xfd\xff" - b"\xff\xff\x03\x00\xfa\xff\x11\x00}\x00\xf3\xff\x05\x00\xfe\xff" - b"\x00\x00\x01\x00\xfd\xff\x08\x00\x7f\x00\xf9\xff\x03\x00\xff\xff" - b"\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00x\x00\x08\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00p\x00\x10\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00h\x00\x18\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00`\x00 \x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00X\x00(\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00P\x000\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00H\x008\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00@\x00@\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x008\x00H\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x000\x00P\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00(\x00X\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00 \x00`\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x18\x00h\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x10\x00p\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x08\x00x\x00\x00\x00\x00\x00\x00\x00" - b"\x02\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x05\x00\x00\x00" - b"\x03\x00\x00\x00\x06\x00\x00\x00\x08\x01\x00\x00\x07\x00\x00\x00" - b"\x03\x01\x00\x00\x08\x00\x00\x00\x07\x01\x00\x00\t\x00\x00\x00" - b"\t\x01\x00\x00\n\x00\x00\x00\n\x01\x00\x00\x0b\x00\x00\x00" - b"\x0b\x01\x00\x00\x0c\x00\x00\x00\x00\x01\x00\x00\r\x00\x00\x00" - b"\x01\x01\x00\x00\x0e\x00\x00\x00\x80\x00\x00\x00\x0f\x00\x00\x00" - b"\x05\x01\x00\x00\x10\x00\x00\x00\x06\x01\x00\x00\x11\x00\x00\x00" - b"\x04\x01\x00\x00\x12\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00" - b"\x07\x14!.;HUbo|\x89\x96\xa3\xb0\xbd\xca" - b"\xd7\xe4\xf1\xfe\x01\x02\x03\x04\x05\x06\x08\t\n\x0b\x0c\r" - b"\x0e\x0f\x10\x11\x12\x13\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e" - b"\x1f \"#$%&'()*+,-/0" - b"123456789:<=>?@A" - b"BCDEFGIJKLMNOPQR" - b"STVWXYZ[\\]^_`acd" - b"efghijklmnpqrstu" - b"vwxyz{}~\x7f\x80\x81\x82\x83\x84\x85\x86" - b"\x87\x88\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x97\x98" - b"\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa4\xa5\xa6\xa7\xa8\xa9" - b"\xaa\xab\xac\xad\xae\xaf\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba" - b"\xbb\xbc\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xcb\xcc" - b"\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc\xdd" - b"\xde\xdf\xe0\xe1\xe2\xe3\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee" - b"\xef\xf0\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfd\x00" - b"\x00\x07\x06\x06\x05\x05\x05\x05\x04\x04\x04\x04\x04\x04\x04\x04" - b"\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03" - b"\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02" - b"\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02" - b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" - b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" - b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" - b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x9f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xa5\x91\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xad\x94\x8c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xb0\x9b\x8c\x87\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xb4\x9d\x8d\x86\x82\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\xfe\xfe\xfe\xfc\xf9\xf3\xe6\xc4\xb1\x99\x8c\x85\x82\x81\x00\x00" - b"\x00\x01\x01\x02\x02\x02\x03\x03\x03\x03\x04\x04\x04\x04\x04\x04" - b"\x04\x04\x04\x04\x04\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" - b"" + b"\x03\x04\x14\x00\x06\x00\x08\x00\x00\x00!\x00\xd9\x85nc\x81\x01\x00" + b"\x00c\x04\x00\x00\x13\x00\x08\x02[Content_Types].xml \xa2\x04\x02(" + b"\xa0\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" ) diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index bcbabcd5..76524f42 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -16,7 +16,7 @@ def test_file_mimetype(png_image, jpg_image): assert get_file_mimetype(jpg_image) == "image/jpeg" -def test_content_mimetype(png_image, jpg_image, undecodable_byte_stream): +def test_content_mimetype(png_image, jpg_image): with open(png_image, "rb") as fh: assert get_content_mimetype(fh.read(64)) == "image/png" From d7fe9366fb6f4efd0b1be24b177fb24edf5ffd6e Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 5 May 2023 09:56:12 +0000 Subject: [PATCH 041/335] Allow Language metadata to have multiple values --- CHANGELOG.md | 1 + src/zimscraperlib/zim/metadata.py | 8 ++++++-- tests/zim/test_zim_creator.py | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c19a5b98..3848a23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Using pylibzim `3.1.0` +- ZIM metadata check now allows multiple values (comma-separated) for `Language` ### Removed diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index 56abedb8..c121b651 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -59,8 +59,12 @@ def validate_date(name: str, value: Union[datetime.datetime, datetime.date, str] def validate_language(name: str, value: Union[Iterable[str], str]): """ensures Language metadata is a single or list of ISO-639-3 codes""" - if name == "Language" and not is_valid_iso_639_3(value): - raise ValueError(f"{value} is not ISO-639-3.") + if name == "Language": + if isinstance(value, str): + value = value.split(",") + for code in value: + if not is_valid_iso_639_3(code): + raise ValueError(f"{code} is not ISO-639-3.") def validate_counter(name: str, value: str): diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 6cc5a22b..151a906f 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -594,6 +594,11 @@ def test_config_metadata(tmp_path, png_image): ("Language", "bam", True), ("Language", "fr", False), ("Language", "en", False), + ("Language", "fra,eng", True), + ("Language", "fra,eng,bam", True), + ("Language", "fra,en,bam", False), + ("Language", "eng,", False), + ("Language", "eng, fra", False), ("Counter", "1", False), ("Description", "X" * 80, True), ("Description", "X" * 81, False), From e37764e06039d635cb8cc7179608ba88672cc853 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 5 May 2023 09:56:49 +0000 Subject: [PATCH 042/335] Using yt_dlp instead of youtube_dl --- CHANGELOG.md | 1 + requirements.txt | 2 +- src/zimscraperlib/download.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3848a23a..b73e896f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Using pylibzim `3.1.0` - ZIM metadata check now allows multiple values (comma-separated) for `Language` +- Using `yt_dlp` instead of `youtube_dl` ### Removed diff --git a/requirements.txt b/requirements.txt index 9978dccf..98a56d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ beautifulsoup4>=4.9.3,<4.10 lxml>=4.6.3,<4.10 optimize-images>=1.3.6,<1.6 # youtube-dl should be updated as frequently as possible -youtube_dl +yt-dlp diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index f9b397b8..ae6153f5 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -9,7 +9,7 @@ from typing import Dict, Optional, Union import requests -import youtube_dl +import yt_dlp as youtube_dl from . import logger From 36ae90375e691af1312651b536affa08bb622dbb Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 5 May 2023 10:16:36 +0000 Subject: [PATCH 043/335] releasing v3.1.0 --- CHANGELOG.md | 2 +- src/zimscraperlib/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b73e896f..4d932791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [3.1.0] - 2023-05-05 ### Changed diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 4a36342f..fd2a0186 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -3.0.0 +3.1.0 From 0421b3bc7cb6038057324ae8f8fe04a06a70dfaf Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 13 Jul 2023 21:31:32 +0200 Subject: [PATCH 044/335] Fix return type since it is a Tuple and not a Union --- src/zimscraperlib/download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index ae6153f5..85c4b1bd 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -6,7 +6,7 @@ import pathlib import subprocess from concurrent.futures import Future, ThreadPoolExecutor -from typing import Dict, Optional, Union +from typing import Dict, Optional, Tuple, Union import requests import yt_dlp as youtube_dl @@ -174,7 +174,7 @@ def stream_file( max_retries: Optional[int] = 5, headers: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None, -) -> Union[int, requests.structures.CaseInsensitiveDict]: +) -> Tuple[int, requests.structures.CaseInsensitiveDict]: """Stream data from a URL to either a BytesIO object or a file Arguments - fpath - Path of the file where data is sent From 200d6c2253f210594bfb4a25208e8f68f16b5384 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 14 Jul 2023 10:36:47 +0200 Subject: [PATCH 045/335] Use tuple instead of Tuple (PEP 585) --- src/zimscraperlib/download.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 85c4b1bd..913f0990 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -2,11 +2,13 @@ # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +from __future__ import annotations + import io import pathlib import subprocess from concurrent.futures import Future, ThreadPoolExecutor -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional, Union import requests import yt_dlp as youtube_dl @@ -174,7 +176,7 @@ def stream_file( max_retries: Optional[int] = 5, headers: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None, -) -> Tuple[int, requests.structures.CaseInsensitiveDict]: +) -> tuple[int, requests.structures.CaseInsensitiveDict]: """Stream data from a URL to either a BytesIO object or a file Arguments - fpath - Path of the file where data is sent From 23da5cec945e5bffc9732df6335df3c23ccbf6a5 Mon Sep 17 00:00:00 2001 From: Amirreza Aflakparast <84932095+AmirAflak@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:46:33 +0330 Subject: [PATCH 046/335] fix: #107 to handle string type --- src/zimscraperlib/zim/creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 7b0ec5fc..a87a24e6 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -237,7 +237,7 @@ def add_item_for( path: str, title: Optional[str] = None, fpath: Optional[pathlib.Path] = None, - content: Optional[bytes] = None, + content: Optional[Union[bytes, str]] = None, mimetype: Optional[str] = None, is_front: Optional[bool] = None, should_compress: Optional[bool] = None, From 7ce21f3a9db4ab57367e7bbfac539a42ed8f8ecb Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 18 Jul 2023 16:26:18 +0000 Subject: [PATCH 047/335] Updated CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d932791..0e257a0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +### Changed + +- Fixed declared (hint) return type of `download.stream_file` #104 +- Fixed declared (hint) type of `content` param for `Creator.add_item_for` #107 + ## [3.1.0] - 2023-05-05 ### Changed From 4857e319a72cb25e076fcc90fa8c109bd994b580 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 18 Jul 2023 16:26:46 +0000 Subject: [PATCH 048/335] released v3.1.1 --- src/zimscraperlib/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index fd2a0186..94ff29cc 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -3.1.0 +3.1.1 From 4f8c3cc0ab5ffd3f2ea9c52937b86a5ec4c42bf6 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 18 Jul 2023 16:27:32 +0000 Subject: [PATCH 049/335] Added version to version --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e257a0c..4f34aeae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [3.1.1] ### Changed From 3dd78877cea4c5a44a377cc1060d923688f8e184 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 21 Aug 2023 16:15:00 +0200 Subject: [PATCH 050/335] Add utility function to compute/check ZIM descriptions based on default and user provided values --- src/zimscraperlib/inputs.py | 62 +++++++++++++ tests/inputs/test_inputs.py | 175 +++++++++++++++++++++++++++++++++++- 2 files changed, 236 insertions(+), 1 deletion(-) diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index 936f9404..1c2ef819 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -5,9 +5,12 @@ import pathlib import shutil import tempfile +from dataclasses import dataclass from typing import Optional, Union from . import logger +from .constants import MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH +from .constants import MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH from .download import stream_file @@ -49,3 +52,62 @@ def handle_user_provided_file( shutil.copy(source, dest) return dest + + +@dataclass +class Descriptions: + description: str + long_description: Optional[str] + + +def compute_descriptions( + default_description: str, + user_description: Optional[str], + user_long_description: Optional[str], +) -> Descriptions: + """Computes short and long descriptions compliant with ZIM standard. + + Based on provided parameters, the function computes a short and a long description + which are compliant with the ZIM standard (in terms of length). + + User description(s) are used if set. They are checked to not exceed ZIM standard + maximum length ; an error is thrown otherwise ; if ok, they are returned. + + If user_description is not set, the description is computed based on the default + description, truncated if needed. + + If user_long_description is not set and default description is too long for the + description field, the long_description is computed based on the default description + (truncated if needed), otherwise no long description is returned. + + args: + default_description: the description which will be used if user descriptions + are not set (typically fetched online) + user_description: the description set by the user (typically set by a + CLI argument) + user_long_description: the long description set by the user (typically set by a + CLI argument) + """ + + if user_description and len(user_description) > MAX_DESC_LENGTH: + raise ValueError( + f"Description too long ({len(user_description)}>{MAX_DESC_LENGTH})" + ) + if user_long_description and len(user_long_description) > MAX_LONG_DESC_LENGTH: + raise ValueError( + f"LongDescription too long ({len(user_long_description)}" + f">{MAX_LONG_DESC_LENGTH})" + ) + + if not user_long_description and len(default_description) > MAX_DESC_LENGTH: + user_long_description = default_description[0:MAX_LONG_DESC_LENGTH] + if len(default_description) > MAX_LONG_DESC_LENGTH: + user_long_description = user_long_description[:-1] + "…" + if not user_description: + user_description = default_description[0:MAX_DESC_LENGTH] + if len(default_description) > MAX_DESC_LENGTH: + user_description = user_description[:-1] + "…" + + return Descriptions( + description=user_description, long_description=user_long_description + ) diff --git a/tests/inputs/test_inputs.py b/tests/inputs/test_inputs.py index 639a34c9..4dd4dee6 100644 --- a/tests/inputs/test_inputs.py +++ b/tests/inputs/test_inputs.py @@ -3,10 +3,17 @@ # vim: ai ts=4 sts=4 et sw=4 nu import pathlib +from typing import Optional import pytest -from zimscraperlib.inputs import handle_user_provided_file +from zimscraperlib.constants import ( + MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH, +) +from zimscraperlib.constants import ( + MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH, +) +from zimscraperlib.inputs import compute_descriptions, handle_user_provided_file def test_with_none(): @@ -72,3 +79,169 @@ def test_remote_indir(tmp_path, valid_http_url): assert fpath is not None assert fpath.exists() assert fpath.parent == tmp_path + + +TEXT_NOT_USED = "text not used" + +LONG_TEXT = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor " + "incididunt ut labore et dolore magna aliqua. At erat pellentesque adipiscing " + "commodo elit at imperdiet. Rutrum tellus pellentesque eu tincidunt tortor aliquam" + " nulla facilisi. Eget lorem dolor sed viverra ipsum nunc. Ipsum nunc aliquet " + "bibendum enim facilisis gravida neque convallis. Aliquam malesuada bibendum arcu " + "vitae elementum curabitur. Platea dictumst quisque sagittis purus sit amet " + "volutpat. Blandit libero volutpat sed cras ornare. In eu mi bibendum neque " + "egestas. Egestas dui id ornare arcu odio. Pulvinar neque laoreet suspendisse " + "interdum. Fames ac turpis egestas integer eget aliquet nibh praesent tristique. Et" + " egestas quis ipsum suspendisse ultrices gravida dictum fusce. Malesuada fames ac " + "turpis egestas. Tincidunt nunc pulvinar sapien et ligula ullamcorper malesuada " + "proin libero. In arcu cursus euismod quis viverra. Faucibus in ornare quam viverra" + ". Curabitur vitae nunc sed velit dignissim sodales ut eu sem. Velit scelerisque in" + " dictum non consectetur a erat nam. Proin fermentum leo vel orci porta non. Fames" + " ac turpis egestas sed tempus. Vitae justo eget magna fermentum iaculis eu non. " + "Imperdiet massa tincidunt nunc pulvinar sapien et ligula. Laoreet sit amet cursus " + "sit amet dictum sit amet. Quis hendrerit dolor magna eget. Orci ac auctor augue " + "mauris augue. Consequat interdum varius sit amet mattis. At ultrices mi tempus " + "imperdiet nulla malesuada pellentesque elit. Volutpat est velit egestas dui. " + "Potenti nullam ac tortor vitae. At tempor commodo ullamcorper a lacus vestibulum " + "sed arcu non. Duis ut diam quam nulla. Vestibulum mattis ullamcorper velit sed " + "ullamcorper. Sit amet commodo nulla facilisi nullam vehicula. Faucibus purus in " + "massa tempor nec feugiat. Sem fringilla ut morbi tincidunt augue interdum velit. " + "Etiam dignissim diam quis enim lobortis scelerisque fermentum dui. Nunc vel risus " + "commodo viverra maecenas accumsan. Aenean sed adipiscing diam donec adipiscing " + "tristique. Maecenas accumsan lacus vel facilisis volutpat est velit egestas. Nulla" + " aliquet porttitor lacus luctus accumsan tortor posuere ac. Habitant morbi " + "tristique senectus et netus et. Eget mi proin sed libero enim sed faucibus turpis " + "in. Vulputate enim nulla aliquet porttitor lacus. Dui ut ornare lectus sit amet " + "est. Quam lacus suspendisse faucibus interdum posuere. Sagittis orci a scelerisque" + " purus semper eget duis at tellus. Tellus molestie nunc non blandit massa. Feugiat" + " vivamus at augue eget arcu dictum varius duis at. Varius morbi enim nunc faucibus" + " a pellentesque sit. Id aliquet lectus proin nibh nisl condimentum id venenatis a." + " Tortor dignissim convallis aenean et tortor at risus viverra adipiscing. Aliquam " + "malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Habitasse platea" + " dictumst quisque sagittis purus sit amet volutpat. Vitae auctor eu augue ut " + "lectus. At varius vel pharetra vel turpis nunc eget. Dictum at tempor commodo " + "ullamcorper a lacus vestibulum sed arcu. Pellentesque massa placerat duis " + "ultricies. Enim nunc faucibus a pellentesque sit amet porttitor eget dolor. " + "Volutpat blandit aliquam etiam erat velit scelerisque in. Amet mattis vulputate " + "enim nulla aliquet porttitor. Egestas maecenas pharetra convallis posuere morbi " + "leo urna molestie. Duis ut diam quam nulla porttitor massa id. In fermentum " + "posuere urna nec tincidunt praesent. Turpis egestas sed tempus urna et pharetra " + "pharetra massa. Tellus molestie nunc non blandit massa. Diam phasellus vestibulum " + "lorem sed risus ultricies. Egestas erat imperdiet sed euismod nisi porta lorem. " + "Quam viverra orci sagittis eu volutpat odio facilisis mauris sit. Ornare aenean " + "euismod elementum nisi quis. Laoreet non curabitur gravida arcu ac tortor " + "dignissim convallis aenean. Sagittis aliquam malesuada bibendum arcu vitae " + "elementum. Sed blandit libero volutpat sed cras ornare. Sagittis eu volutpat odio " + "facilisis mauris. Facilisis volutpat est velit egestas dui id ornare arcu odio. " + "Eu feugiat pretium nibh." +) + + +@pytest.mark.parametrize( + "user_description, user_long_description, default_description, raises, " + "expected_description, expected_long_description", + [ + # user description set and is short, user long descripion not set, default + # description doe not matter + ( + LONG_TEXT[0:MAX_DESC_LENGTH], + None, + TEXT_NOT_USED, + False, + LONG_TEXT[0:MAX_DESC_LENGTH], + None, + ), + # user description set and is too long, default description does not matter + (LONG_TEXT[0 : MAX_DESC_LENGTH + 1], None, TEXT_NOT_USED, True, None, None), + # user description not set and default description is short enough + ( + None, + None, + LONG_TEXT[0:MAX_DESC_LENGTH], + False, + LONG_TEXT[0:MAX_DESC_LENGTH], + None, + ), + # user description not set and default description is too long for description + # but ok for long description + ( + None, + None, + LONG_TEXT[0 : MAX_DESC_LENGTH + 1], + False, + LONG_TEXT[0 : MAX_DESC_LENGTH - 1] + "…", + LONG_TEXT[0 : MAX_DESC_LENGTH + 1], + ), + ( + None, + None, + LONG_TEXT[0:MAX_LONG_DESC_LENGTH], + False, + LONG_TEXT[0 : MAX_DESC_LENGTH - 1] + "…", + LONG_TEXT[0:MAX_LONG_DESC_LENGTH], + ), + # user description not set and default description is too long for description + # and long description + ( + None, + None, + LONG_TEXT[0 : MAX_LONG_DESC_LENGTH + 1], + False, + LONG_TEXT[0 : MAX_DESC_LENGTH - 1] + "…", + LONG_TEXT[0 : MAX_LONG_DESC_LENGTH - 1] + "…", + ), + # user description set and is short, user long descripion set and is short, + # default description does not matter + ( + LONG_TEXT[0:MAX_DESC_LENGTH], + LONG_TEXT[0:MAX_LONG_DESC_LENGTH], + TEXT_NOT_USED, + False, + LONG_TEXT[0:MAX_DESC_LENGTH], + LONG_TEXT[0:MAX_LONG_DESC_LENGTH], + ), + # user description set and is short, user long descripion set and is too long, + # default description does not matter + ( + LONG_TEXT[0:MAX_DESC_LENGTH], + LONG_TEXT[0 : MAX_LONG_DESC_LENGTH + 1], + TEXT_NOT_USED, + True, + None, + None, + ), + # user description not set, user long descripion set and is short, + # default description set to something different than long desc + ( + None, + LONG_TEXT[0:MAX_LONG_DESC_LENGTH], + LONG_TEXT[10:MAX_LONG_DESC_LENGTH], + False, + LONG_TEXT[10 : MAX_DESC_LENGTH + 9] + "…", + LONG_TEXT[0:MAX_LONG_DESC_LENGTH], + ), + ], +) +def test_description( + user_description: str, + user_long_description: Optional[str], + default_description: str, + *, + raises: bool, + expected_description: str, + expected_long_description: str, +): + if raises: + with pytest.raises(ValueError): + compute_descriptions( + default_description, user_description, user_long_description + ) + return + else: + descriptions = compute_descriptions( + default_description, user_description, user_long_description + ) + + assert descriptions.description == expected_description + assert descriptions.long_description == expected_long_description From 8da15e85f939779c6d7e297110418db92424f924 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 21 Aug 2023 17:13:30 +0200 Subject: [PATCH 051/335] Add {posargs} placeholder to pytest so that we can test only few thinks on dev machine --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9a7f38ac..e48a3d0f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = pytest-cov -r{toxinidir}/requirements.txt commands = - pytest --runslow --runinstalled --cov=zimscraperlib --cov-report=term --cov-report term-missing + pytest --runslow --runinstalled --cov=zimscraperlib --cov-report=term --cov-report term-missing {posargs} [testenv:black] deps = From 1eaa484a0f6040e570b259b1c5026608fc87124c Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 21 Aug 2023 17:14:03 +0200 Subject: [PATCH 052/335] Specify Python version for isort because it can't run on 3.7 (which is used by default) --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e48a3d0f..d27ca9f7 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ deps = commands = black --check . [testenv:isort] +base_python=py38 deps = isort>=5.12.0,<5.13 commands = isort --profile black --check src tests From 8550c4b5a813de51b61d92ddcdc2ffc44a4ab4bd Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 22 Aug 2023 08:38:49 +0200 Subject: [PATCH 053/335] Rework to return tuple instead of dataclass --- src/zimscraperlib/inputs.py | 17 +++++------------ tests/inputs/test_inputs.py | 6 +++--- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index 1c2ef819..51feb2d1 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -5,8 +5,7 @@ import pathlib import shutil import tempfile -from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional, Tuple, Union from . import logger from .constants import MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH @@ -54,17 +53,11 @@ def handle_user_provided_file( return dest -@dataclass -class Descriptions: - description: str - long_description: Optional[str] - - def compute_descriptions( default_description: str, user_description: Optional[str], user_long_description: Optional[str], -) -> Descriptions: +) -> Tuple[str, Optional[str]]: """Computes short and long descriptions compliant with ZIM standard. Based on provided parameters, the function computes a short and a long description @@ -87,6 +80,8 @@ def compute_descriptions( CLI argument) user_long_description: the long description set by the user (typically set by a CLI argument) + + Returns a tuple of (description, long_description) """ if user_description and len(user_description) > MAX_DESC_LENGTH: @@ -108,6 +103,4 @@ def compute_descriptions( if len(default_description) > MAX_DESC_LENGTH: user_description = user_description[:-1] + "…" - return Descriptions( - description=user_description, long_description=user_long_description - ) + return (user_description, user_long_description) diff --git a/tests/inputs/test_inputs.py b/tests/inputs/test_inputs.py index 4dd4dee6..197ce760 100644 --- a/tests/inputs/test_inputs.py +++ b/tests/inputs/test_inputs.py @@ -239,9 +239,9 @@ def test_description( ) return else: - descriptions = compute_descriptions( + (description, long_description) = compute_descriptions( default_description, user_description, user_long_description ) - assert descriptions.description == expected_description - assert descriptions.long_description == expected_long_description + assert description == expected_description + assert long_description == expected_long_description From d72a132880978e15c1bba8dc18fd73c5976af688 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 22 Aug 2023 08:39:53 +0200 Subject: [PATCH 054/335] Fix type hint --- src/zimscraperlib/inputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index 51feb2d1..ad6878c8 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -16,7 +16,7 @@ def handle_user_provided_file( source: Optional[Union[pathlib.Path, str]] = None, dest: Optional[pathlib.Path] = None, - in_dir: pathlib.Path = None, + in_dir: Optional[pathlib.Path] = None, nocopy: bool = False, ) -> Union[pathlib.Path, None]: """path to downloaded or copied a user provided file (URL or path) From f10f2ca51d559626479495a49dd537bda7841ce8 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 22 Aug 2023 08:16:54 +0000 Subject: [PATCH 055/335] using a variable for recomended max title length --- src/zimscraperlib/constants.py | 1 + src/zimscraperlib/zim/metadata.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zimscraperlib/constants.py b/src/zimscraperlib/constants.py index 65914417..a42c16af 100644 --- a/src/zimscraperlib/constants.py +++ b/src/zimscraperlib/constants.py @@ -48,6 +48,7 @@ ), } +RECOMMENDED_MAX_TITLE_LENGTH = 30 MAXIMUM_DESCRIPTION_METADATA_LENGTH = 80 MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH = 4000 diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index c121b651..bedcddd6 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -8,6 +8,7 @@ MANDATORY_ZIM_METADATA_KEYS, MAXIMUM_DESCRIPTION_METADATA_LENGTH, MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH, + RECOMMENDED_MAX_TITLE_LENGTH, ) from ..i18n import is_valid_iso_639_3 from ..image.probing import is_valid_image @@ -40,7 +41,7 @@ def validate_standard_str_types(name: str, value: str): def validate_title(name: str, value: str): """ensures Title metadata is within recommended length""" - if name == "Title" and len(value) > 30: + if name == "Title" and len(value) > RECOMMENDED_MAX_TITLE_LENGTH: raise ValueError(f"{name} is too long.") From 591c6dc223b9adb952a569f15945eef1c148d5cf Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 22 Aug 2023 15:08:53 +0200 Subject: [PATCH 056/335] Update CHANGELOG for descriptions function addition --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f34aeae..5ab74cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [3.1.1] +## [Unreleased] + +### Added +- Add utility function to compute/check ZIM descriptions #110 + +## [3.1.1] - 2023-07-18 ### Changed From 3f1a26d4e53fee7e66ef6e4d78e3a078db944631 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 16 Dec 2023 18:09:49 +0000 Subject: [PATCH 057/335] removed support for py3.7 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 5 +++++ tox.ini | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c19452e2..c0d9ca6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab74cdd..deaba225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added + - Add utility function to compute/check ZIM descriptions #110 +### Removed + +- Support for Python 3.7 (EOL) + ## [3.1.1] - 2023-07-18 ### Changed diff --git a/tox.ini b/tox.ini index d27ca9f7..aa50c813 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, py311, black, isort, flake8, coverage +envlist = py38, py39, py310, py311, py312, black, isort, flake8, coverage [testenv] passenv = From aea479a31c92f3c91d9664c49b5e73a214e5565c Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 16 Dec 2023 18:11:46 +0000 Subject: [PATCH 058/335] using pylibzim 3.4.0 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 4 ++++ requirements.txt | 2 +- tox.ini | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0d9ca6d..953a6867 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle diff --git a/CHANGELOG.md b/CHANGELOG.md index deaba225..0135ce9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add utility function to compute/check ZIM descriptions #110 +### Changed + +- Using pylibzim `3.4.0` + ### Removed - Support for Python 3.7 (EOL) diff --git a/requirements.txt b/requirements.txt index 98a56d1e..c7fc799c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ colorthief==0.2.1 python-resize-image>=1.1.19,<1.2 Babel>=2.9,<3.0 file-magic>=0.4.0,<0.5 -libzim>=3.1.0,<3.2 +libzim>=3.4.0,<3.5 beautifulsoup4>=4.9.3,<4.10 lxml>=4.6.3,<4.10 optimize-images>=1.3.6,<1.6 diff --git a/tox.ini b/tox.ini index aa50c813..d9176d58 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38, py39, py310, py311, py312, black, isort, flake8, coverage +envlist = py38, py39, py310, py311, black, isort, flake8, coverage [testenv] passenv = From 4dc30126a54040b4383ffed3617a1394a51b5a78 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 16 Dec 2023 18:24:07 +0000 Subject: [PATCH 059/335] releasing v3.2.0 --- CHANGELOG.md | 2 +- src/zimscraperlib/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0135ce9e..a9bfade3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). -## [Unreleased] +## [3.2.0] - 2023-12-16 ### Added diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION index 94ff29cc..944880fa 100644 --- a/src/zimscraperlib/VERSION +++ b/src/zimscraperlib/VERSION @@ -1 +1 @@ -3.1.1 +3.2.0 From 2d105dcbae0e1c41321462c285329492b62227f3 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 12:05:04 +0100 Subject: [PATCH 060/335] Start migration to python bootstrap conventions --- .gitignore | 3 + .pre-commit-config.yaml | 43 +++--- CHANGELOG.md | 6 + MANIFEST.in | 4 - pyproject.toml | 234 +++++++++++++++++++++++++++++++++ requirements.txt | 12 -- setup.py | 53 -------- src/zimscraperlib/VERSION | 1 - src/zimscraperlib/__about__.py | 1 + src/zimscraperlib/constants.py | 5 +- tasks.py | 109 +++++++++++++++ test | 4 - tox.ini | 37 ------ 13 files changed, 379 insertions(+), 133 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100644 src/zimscraperlib/VERSION create mode 100644 src/zimscraperlib/__about__.py create mode 100644 tasks.py delete mode 100755 test delete mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 9e773ced..7b75b672 100644 --- a/.gitignore +++ b/.gitignore @@ -248,3 +248,6 @@ $RECYCLE.BIN/ *.lnk # End of https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux + +# ignore all vscode, this is not standard configuration in this place +.vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bef63f03..45c2335e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,27 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer -- repo: https://github.com/pycqa/isort - rev: "5.12.0" - hooks: - - id: isort - args: ["--profile", "black", "--filter-files"] -- repo: https://github.com/psf/black - rev: "23.1.0" - hooks: - - id: black -- repo: https://github.com/pycqa/flake8 - rev: "6.0.0" - hooks: - - id: flake8 - args: ["--max-line-length", "88", "--extend-ignore=E203"] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer +- repo: https://github.com/psf/black + rev: "24.1.1" + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.2.1 + hooks: + - id: ruff +- repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.350 + hooks: + - id: pyright + name: pyright (system) + description: 'pyright static type checker' + entry: pyright + language: system + 'types_or': [python, pyi] + require_serial: true + minimum_pre_commit_version: '2.9.2' diff --git a/CHANGELOG.md b/CHANGELOG.md index a9bfade3..c849d881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0). +## [Unreleased] + +### Changed + +- Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120 + ## [3.2.0] - 2023-12-16 ### Added diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 68276b73..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -graft src -include *.md -include requirements.txt -include LICENSE diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2c5c0a6c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,234 @@ +[build-system] +requires = ["hatchling", "hatch-openzim"] +build-backend = "hatchling.build" + +[project] +name = "zimscraperlib" +requires-python = ">=3.8,<3.12" +description = "Collection of python tools to re-use common code across scrapers" +readme = "README.md" +dependencies = [ + "iso-639==0.4.5", + "requests==2.31.0", + "colorthief==0.2.1", + "python-resize-image==1.1.20", + "Babel==2.14.0", + "file-magic==0.4.1", + "libzim==3.4.0", + "beautifulsoup4==4.9.3", # upgrade to 4.10 and later to be done + "lxml==4.9.4", # upgrade to 4.10 and later to be done + "optimize-images==1.5.1", + # youtube-dl should be updated as frequently as possible + "yt-dlp" +] +dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"] + +[tool.hatch.metadata.hooks.openzim-metadata] +kind = "scraper" +# not yet supported in hatch-openzim 0.1 +# additional-classifiers = [ +# "Development Status :: 5 - Production/Stable", +# "Intended Audience :: Developers", +# ] + +[project.optional-dependencies] +scripts = [ + "invoke==2.2.0", +] +lint = [ + "black==24.1.1", + "ruff==0.2.1", +] +check = [ + "pyright==1.1.350", +] +test = [ + "pytest==8.0.0", + "coverage==7.4.1", +] +dev = [ + "pre-commit==3.5.0", + "debugpy==1.8.0", + "zimscraperlib[scripts]", + "zimscraperlib[lint]", + "zimscraperlib[test]", + "zimscraperlib[check]", +] + +[tool.hatch.version] +path = "src/zimscraperlib/__about__.py" + +[tool.hatch.build] +exclude = [ + "/.github", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/zimscraperlib"] + +[tool.hatch.envs.default] +features = ["dev"] + +[tool.hatch.envs.test] +features = ["scripts", "test"] + +[[tool.hatch.envs.test.matrix]] +python = ["3.8", "3.9", "3.10", "3.11"] + +[tool.hatch.envs.test.scripts] +run = "inv test --args '{args}'" +run-cov = "inv test-cov --args '{args}'" +report-cov = "inv report-cov" +coverage = "inv coverage --args '{args}'" +html = "inv coverage --html --args '{args}'" + +[tool.hatch.envs.lint] +template = "lint" +skip-install = false +features = ["scripts", "lint"] + +[tool.hatch.envs.lint.scripts] +black = "inv lint-black --args '{args}'" +ruff = "inv lint-ruff --args '{args}'" +all = "inv lintall --args '{args}'" +fix-black = "inv fix-black --args '{args}'" +fix-ruff = "inv fix-ruff --args '{args}'" +fixall = "inv fixall --args '{args}'" + +[tool.hatch.envs.check] +features = ["scripts", "check"] + +[tool.hatch.envs.check.scripts] +pyright = "inv check-pyright --args '{args}'" +all = "inv checkall --args '{args}'" + +[tool.black] +line-length = 88 +target-version = ['py38'] + +[tool.ruff] +target-version = "py38" +line-length = 88 +src = ["src"] + +[tool.ruff.lint] +select = [ + "A", # flake8-builtins + # "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments + # "ASYNC", # flake8-async + "B", # flake8-bugbear + # "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "C90", # mccabe + # "COM", # flake8-commas + # "D", # pydocstyle + # "DJ", # flake8-django + "DTZ", # flake8-datetimez + "E", # pycodestyle (default) + "EM", # flake8-errmsg + # "ERA", # eradicate + # "EXE", # flake8-executable + "F", # Pyflakes (default) + # "FA", # flake8-future-annotations + "FBT", # flake8-boolean-trap + # "FLY", # flynt + # "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + # "INP", # flake8-no-pep420 + # "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + # "NPY", # NumPy-specific rules + # "PD", # pandas-vet + # "PGH", # pygrep-hooks + # "PIE", # flake8-pie + # "PL", # Pylint + "PLC", # Pylint: Convention + "PLE", # Pylint: Error + "PLR", # Pylint: Refactor + "PLW", # Pylint: Warning + # "PT", # flake8-pytest-style + # "PTH", # flake8-use-pathlib + # "PYI", # flake8-pyi + "Q", # flake8-quotes + # "RET", # flake8-return + # "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "S", # flake8-bandit + # "SIM", # flake8-simplify + # "SLF", # flake8-self + "T10", # flake8-debugger + "T20", # flake8-print + # "TCH", # flake8-type-checking + # "TD", # flake8-todos + "TID", # flake8-tidy-imports + # "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + # Allow non-abstract empty methods in abstract base classes + "B027", + # Remove flake8-errmsg since we consider they bloat the code and provide limited value + "EM", + # Allow boolean positional values in function calls, like `dict.get(... True)` + "FBT003", + # Ignore checks for possible passwords + "S105", "S106", "S107", + # Ignore warnings on subprocess.run / popen + "S603", + # Ignore complexity + "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", +] +unfixable = [ + # Don't touch unused imports + "F401", +] + +[tool.ruff.lint.isort] +known-first-party = ["zimscraperlib"] + +[tool.ruff.lint.flake8-bugbear] +# add exceptions to B008 for fastapi. +extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"] + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.lint.per-file-ignores] +# Tests can use magic values, assertions, and relative imports +"tests/**/*" = ["PLR2004", "S101", "TID252"] + +[tool.pytest.ini_options] +minversion = "7.3" +testpaths = ["tests"] +pythonpath = [".", "src"] + +[tool.coverage.paths] +zimscraperlib = ["src/zimscraperlib"] +tests = ["tests"] + +[tool.coverage.run] +source_pkgs = ["zimscraperlib"] +branch = true +parallel = true +omit = [ + "src/zimscraperlib/__about__.py", +] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.pyright] +include = ["src", "tests", "tasks.py"] +exclude = [".env/**", ".venv/**"] +extraPaths = ["src"] +pythonVersion = "3.8" +typeCheckingMode="basic" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c7fc799c..00000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -iso-639==0.4.5 -requests>=2.25.1,<3.0 -colorthief==0.2.1 -python-resize-image>=1.1.19,<1.2 -Babel>=2.9,<3.0 -file-magic>=0.4.0,<0.5 -libzim>=3.4.0,<3.5 -beautifulsoup4>=4.9.3,<4.10 -lxml>=4.6.3,<4.10 -optimize-images>=1.3.6,<1.6 -# youtube-dl should be updated as frequently as possible -yt-dlp diff --git a/setup.py b/setup.py deleted file mode 100644 index 9a0d0a65..00000000 --- a/setup.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# vim: ai ts=4 sts=4 et sw=4 nu - -import pathlib - -from setuptools import find_packages, setup - -root_dir = pathlib.Path(__file__).parent - - -def read(*names, **kwargs): - with open(root_dir.joinpath(*names), "r") as fh: - return fh.read() - - -setup( - name="zimscraperlib", - version=read("src", "zimscraperlib", "VERSION").strip(), - description="Collection of python tools to re-use common code across scrapers", - long_description=read("README.md"), - long_description_content_type="text/markdown", - author="kiwix", - author_email="reg@kiwix.org", - url="https://github.com/openzim/python_scraperlib", - keywords="kiwix zim offline", - license="GPLv3+", - packages=find_packages("src"), - package_dir={"": "src"}, - install_requires=[ - line.strip() - for line in read("requirements.txt").splitlines() - if not line.strip().startswith("#") - ], - setup_requires=["pytest-runner"], - zip_safe=False, - include_package_data=True, - entry_points={ - "console_scripts": ["fix_ogvjs_dist=zimscraperlib.fix_ogvjs_dist:run"] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Programming Language :: Python", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", - ], - python_requires=">=3.7", -) diff --git a/src/zimscraperlib/VERSION b/src/zimscraperlib/VERSION deleted file mode 100644 index 944880fa..00000000 --- a/src/zimscraperlib/VERSION +++ /dev/null @@ -1 +0,0 @@ -3.2.0 diff --git a/src/zimscraperlib/__about__.py b/src/zimscraperlib/__about__.py new file mode 100644 index 00000000..49fc2a48 --- /dev/null +++ b/src/zimscraperlib/__about__.py @@ -0,0 +1 @@ +__version__ = "3.2.1-dev0" \ No newline at end of file diff --git a/src/zimscraperlib/constants.py b/src/zimscraperlib/constants.py index a42c16af..686288a4 100644 --- a/src/zimscraperlib/constants.py +++ b/src/zimscraperlib/constants.py @@ -5,12 +5,11 @@ import base64 import pathlib import re +from zimscraperlib.__about__ import __version__ ROOT_DIR = pathlib.Path(__file__).parent NAME = pathlib.Path(__file__).parent.name -with open(ROOT_DIR.joinpath("VERSION"), "r") as fh: - VERSION = fh.read().strip() -SCRAPER = f"{NAME} {VERSION}" +SCRAPER = f"{NAME} {__version__}" UTF8 = "UTF-8" diff --git a/tasks.py b/tasks.py new file mode 100644 index 00000000..90854e86 --- /dev/null +++ b/tasks.py @@ -0,0 +1,109 @@ +# pyright: strict, reportUntypedFunctionDecorator=false +import os + +from invoke.context import Context +from invoke.tasks import task # pyright: ignore [reportUnknownVariableType] + +use_pty = not os.getenv("CI", "") + + +@task(optional=["args"], help={"args": "pytest additional arguments"}) +def test(ctx: Context, args: str = ""): + """run tests (without coverage)""" + ctx.run(f"pytest {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "pytest additional arguments"}) +def test_cov(ctx: Context, args: str = ""): + """run test vith coverage""" + ctx.run(f"coverage run -m pytest {args}", pty=use_pty) + + +@task(optional=["html"], help={"html": "flag to export html report"}) +def report_cov(ctx: Context, *, html: bool = False): + """report coverage""" + ctx.run("coverage combine", warn=True, pty=use_pty) + ctx.run("coverage report --show-missing", pty=use_pty) + if html: + ctx.run("coverage html", pty=use_pty) + + +@task( + optional=["args", "html"], + help={ + "args": "pytest additional arguments", + "html": "flag to export html report", + }, +) +def coverage(ctx: Context, args: str = "", *, html: bool = False): + """run tests and report coverage""" + test_cov(ctx, args=args) + report_cov(ctx, html=html) + + +@task(optional=["args"], help={"args": "black additional arguments"}) +def lint_black(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("black --version", pty=use_pty) + ctx.run(f"black --check --diff {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def lint_ruff(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) + + +@task( + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, +) +def lintall(ctx: Context, args: str = "."): + """Check linting""" + args = args or "." # needed for hatch script + lint_black(ctx, args) + lint_ruff(ctx, args) + + +@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) +def check_pyright(ctx: Context, args: str = ""): + """check static types with pyright""" + ctx.run("pyright --version") + ctx.run(f"pyright {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) +def checkall(ctx: Context, args: str = ""): + """check static types""" + check_pyright(ctx, args) + + +@task(optional=["args"], help={"args": "black additional arguments"}) +def fix_black(ctx: Context, args: str = "."): + """fix black formatting""" + args = args or "." # needed for hatch script + ctx.run(f"black {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_ruff(ctx: Context, args: str = "."): + """fix all ruff rules""" + args = args or "." # needed for hatch script + ctx.run(f"ruff --fix {args}", pty=use_pty) + + +@task( + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, +) +def fixall(ctx: Context, args: str = "."): + """Fix everything automatically""" + args = args or "." # needed for hatch script + fix_black(ctx, args) + fix_ruff(ctx, args) + lintall(ctx, args) diff --git a/test b/test deleted file mode 100755 index b45a31d2..00000000 --- a/test +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -pip install pytest coverage pytest-cov - -PYTHONPATH=$(pwd)/src/ pytest "$@" tests/ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index d9176d58..00000000 --- a/tox.ini +++ /dev/null @@ -1,37 +0,0 @@ -[tox] -envlist = py38, py39, py310, py311, black, isort, flake8, coverage - -[testenv] -passenv = - WGET_BINARY - ZIMWRITERFS_BINARY -deps = - pytest - coverage - pytest-cov - -r{toxinidir}/requirements.txt -commands = - pytest --runslow --runinstalled --cov=zimscraperlib --cov-report=term --cov-report term-missing {posargs} - -[testenv:black] -deps = - black>=23.1.0,<24 -commands = black --check . - -[testenv:isort] -base_python=py38 -deps = - isort>=5.12.0,<5.13 -commands = isort --profile black --check src tests - -[testenv:flake8] -deps = - flake8>=6.0.0,<7.0 -commands = flake8 src --count --max-line-length=88 --statistics --extend-ignore=E203 - -[testenv:coverage] -passenv = - CODECOV_TOKEN -deps = - codecov -commands = codecov From c0c0a971749e8840e532e1cfceb8884efad2fa3d Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 15:15:35 +0100 Subject: [PATCH 061/335] Remove useless fixture mark - never had any effect --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index cece10ec..a6d8be2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -158,7 +158,6 @@ def ns_zim_file(tmpdir_factory): return dst -@pytest.mark.slow @pytest.fixture(scope="session") def real_zim_file(tmpdir_factory): from zimscraperlib.download import stream_file From 13f69d7cf9a9012d9ad35c86a8b70366e40f5cf7 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 14:47:52 +0100 Subject: [PATCH 062/335] Minimal stubs for libzim --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7b75b672..288bff6b 100644 --- a/.gitignore +++ b/.gitignore @@ -251,3 +251,4 @@ $RECYCLE.BIN/ # ignore all vscode, this is not standard configuration in this place .vscode +src/libzim-stubs From 2a18db4b4475142f17a579f6621090578930411c Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 16:04:38 +0100 Subject: [PATCH 063/335] Automatic fixes by ruff and black --- src/zimscraperlib/__about__.py | 2 +- src/zimscraperlib/__init__.py | 1 - src/zimscraperlib/constants.py | 2 +- src/zimscraperlib/download.py | 1 - src/zimscraperlib/filesystem.py | 3 +-- src/zimscraperlib/fix_ogvjs_dist.py | 3 +-- src/zimscraperlib/html.py | 5 ++--- src/zimscraperlib/i18n.py | 1 - src/zimscraperlib/image/convertion.py | 1 - src/zimscraperlib/image/optimization.py | 15 +++++++-------- src/zimscraperlib/image/presets.py | 1 - src/zimscraperlib/image/probing.py | 3 +-- src/zimscraperlib/image/transformation.py | 1 - src/zimscraperlib/image/utils.py | 1 - src/zimscraperlib/inputs.py | 3 +-- src/zimscraperlib/logging.py | 3 +-- src/zimscraperlib/types.py | 1 - src/zimscraperlib/video/config.py | 1 - src/zimscraperlib/video/encoding.py | 6 +++--- src/zimscraperlib/video/presets.py | 1 - src/zimscraperlib/video/probing.py | 3 +-- src/zimscraperlib/zim/__init__.py | 1 - src/zimscraperlib/zim/archive.py | 1 - src/zimscraperlib/zim/creator.py | 2 -- src/zimscraperlib/zim/filesystem.py | 7 +++---- src/zimscraperlib/zim/items.py | 3 +-- src/zimscraperlib/zim/providers.py | 1 - tests/conftest.py | 3 +-- tests/download/test_download.py | 1 - tests/filesystem/test_filesystem.py | 1 - tests/html/conftest.py | 1 - tests/html/test_html.py | 1 - tests/i18n/test_i18n.py | 1 - tests/image/test_image.py | 1 - tests/inputs/test_inputs.py | 1 - tests/logging/conftest.py | 1 - tests/logging/test_logging.py | 3 +-- tests/ogvjs/conftest.py | 1 - tests/ogvjs/test_ogvjs.py | 9 +++++---- tests/types/test_types.py | 1 - tests/video/conftest.py | 1 - tests/video/test_video.py | 1 - tests/zim/conftest.py | 1 - tests/zim/test_archive.py | 1 - tests/zim/test_fs.py | 3 +-- tests/zim/test_libkiwix.py | 1 - tests/zim/test_zim_creator.py | 1 - 47 files changed, 32 insertions(+), 75 deletions(-) diff --git a/src/zimscraperlib/__about__.py b/src/zimscraperlib/__about__.py index 49fc2a48..c178c960 100644 --- a/src/zimscraperlib/__about__.py +++ b/src/zimscraperlib/__about__.py @@ -1 +1 @@ -__version__ = "3.2.1-dev0" \ No newline at end of file +__version__ = "3.2.1-dev0" diff --git a/src/zimscraperlib/__init__.py b/src/zimscraperlib/__init__.py index d568bb0b..4bb2781e 100644 --- a/src/zimscraperlib/__init__.py +++ b/src/zimscraperlib/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import logging as stdlogging diff --git a/src/zimscraperlib/constants.py b/src/zimscraperlib/constants.py index 686288a4..ca2ec163 100644 --- a/src/zimscraperlib/constants.py +++ b/src/zimscraperlib/constants.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import base64 import pathlib import re + from zimscraperlib.__about__ import __version__ ROOT_DIR = pathlib.Path(__file__).parent diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 913f0990..42a05309 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu from __future__ import annotations diff --git a/src/zimscraperlib/filesystem.py b/src/zimscraperlib/filesystem.py index fb5b804d..509675eb 100644 --- a/src/zimscraperlib/filesystem.py +++ b/src/zimscraperlib/filesystem.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ Files manipulation tools @@ -41,7 +40,7 @@ def get_content_mimetype(content: bytes) -> str: def delete_callback( fpath: Union[str, pathlib.Path], callback: Optional[Callable] = None, - *callback_args: Any + *callback_args: Any, ): """helper deleting passed filepath, optionnaly calling an additional callback""" diff --git a/src/zimscraperlib/fix_ogvjs_dist.py b/src/zimscraperlib/fix_ogvjs_dist.py index 2dca4625..7bfea969 100755 --- a/src/zimscraperlib/fix_ogvjs_dist.py +++ b/src/zimscraperlib/fix_ogvjs_dist.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu @@ -19,7 +18,7 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]): root = pathlib.Path(source_vendors_path) logger.info("fixing videosjs-ogvjs.js") plugin_path = root.joinpath("videojs-ogvjs.js") - with open(plugin_path, "r") as fp: + with open(plugin_path) as fp: content = fp.read() content = content.replace( diff --git a/src/zimscraperlib/html.py b/src/zimscraperlib/html.py index 9823e5d0..42d02100 100644 --- a/src/zimscraperlib/html.py +++ b/src/zimscraperlib/html.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ Tools to work with HTML contents """ @@ -27,7 +26,7 @@ def find_title_in(content: Union[str, BinaryIO, TextIO], mime_type: str) -> str: def find_title_in_file(fpath: pathlib.Path, mime_type: str) -> str: """Extracted title from an HTML file""" try: - with open(fpath, "r") as fh: + with open(fpath) as fh: return find_title_in(fh, mime_type) except Exception: return "" @@ -60,7 +59,7 @@ def find_language_in(content: Union[str, BinaryIO, TextIO], mime_type: str) -> s def find_language_in_file(fpath: pathlib.Path, mime_type: str) -> str: """Extracted language from an HTML file""" try: - with open(fpath, "r") as fh: + with open(fpath) as fh: return find_language_in(fh, mime_type) except Exception: return "" diff --git a/src/zimscraperlib/i18n.py b/src/zimscraperlib/i18n.py index 0e721680..87b51fdf 100644 --- a/src/zimscraperlib/i18n.py +++ b/src/zimscraperlib/i18n.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import gettext diff --git a/src/zimscraperlib/image/convertion.py b/src/zimscraperlib/image/convertion.py index 029742c0..6c7d5327 100644 --- a/src/zimscraperlib/image/convertion.py +++ b/src/zimscraperlib/image/convertion.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pathlib diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 0a21e922..094ff663 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu @@ -154,12 +153,12 @@ def optimize_jpeg( if keep_exif and had_exif: piexif.transplant( - exif_src=str(src.resolve()) - if isinstance(src, pathlib.Path) - else src.getvalue(), - image=str(dst.resolve()) - if isinstance(dst, pathlib.Path) - else dst.getvalue(), + exif_src=( + str(src.resolve()) if isinstance(src, pathlib.Path) else src.getvalue() + ), + image=( + str(dst.resolve()) if isinstance(dst, pathlib.Path) else dst.getvalue() + ), new_file=dst, ) @@ -251,7 +250,7 @@ def optimize_gif( args += ["--interlace"] args += [str(src)] with open(dst, "w") as out_file: - gifsicle = subprocess.run(args, stdout=out_file) + gifsicle = subprocess.run(args, stdout=out_file, check=False) # remove dst if gifsicle failed and src is different from dst if gifsicle.returncode != 0 and src.resolve() != dst.resolve() and dst.exists(): diff --git a/src/zimscraperlib/image/presets.py b/src/zimscraperlib/image/presets.py index b79294b2..230fb299 100644 --- a/src/zimscraperlib/image/presets.py +++ b/src/zimscraperlib/image/presets.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ presets for ImageOptimizer in zimscraperlib.image.optimization module """ diff --git a/src/zimscraperlib/image/probing.py b/src/zimscraperlib/image/probing.py index 173aca63..42d2277c 100644 --- a/src/zimscraperlib/image/probing.py +++ b/src/zimscraperlib/image/probing.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import colorsys @@ -24,7 +23,7 @@ def rgb_to_hex(r: int, g: int, b: int) -> str: def solarize(r: int, g: int, b: int) -> Tuple[int, int, int]: # calculate solarized color for main h, l, s = colorsys.rgb_to_hls(float(r) / 256, float(g) / 256, float(b) / 256) - r2, g2, b2 = [int(x * 256) for x in colorsys.hls_to_rgb(h, 0.95, s)] + r2, g2, b2 = (int(x * 256) for x in colorsys.hls_to_rgb(h, 0.95, s)) return r2, g2, b2 ct = colorthief.ColorThief(src) diff --git a/src/zimscraperlib/image/transformation.py b/src/zimscraperlib/image/transformation.py index 372f29ba..0bbfbf6c 100644 --- a/src/zimscraperlib/image/transformation.py +++ b/src/zimscraperlib/image/transformation.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import io diff --git a/src/zimscraperlib/image/utils.py b/src/zimscraperlib/image/utils.py index 712705f0..240ae947 100644 --- a/src/zimscraperlib/image/utils.py +++ b/src/zimscraperlib/image/utils.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pathlib diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index ad6878c8..a20ccec2 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pathlib @@ -43,7 +42,7 @@ def handle_user_provided_file( else: source = pathlib.Path(source).expanduser().resolve() if not source.exists(): - raise IOError(f"{source} could not be found.") + raise OSError(f"{source} could not be found.") if nocopy: return source diff --git a/src/zimscraperlib/logging.py b/src/zimscraperlib/logging.py index b2f2996e..5b7abb11 100644 --- a/src/zimscraperlib/logging.py +++ b/src/zimscraperlib/logging.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import io @@ -76,5 +75,5 @@ def nicer_args_join(args: Iterable) -> str: """slightly better concateated list of subprocess args for display""" nargs = args[0:1] for arg in args[1:]: - nargs.append(arg if arg.startswith("-") else '"{}"'.format(arg)) + nargs.append(arg if arg.startswith("-") else f'"{arg}"') return " ".join(nargs) diff --git a/src/zimscraperlib/types.py b/src/zimscraperlib/types.py index d1aabd41..b3993382 100644 --- a/src/zimscraperlib/types.py +++ b/src/zimscraperlib/types.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ File extensions to MIME-Type mapping diff --git a/src/zimscraperlib/video/config.py b/src/zimscraperlib/video/config.py index d15553ab..5097f21c 100644 --- a/src/zimscraperlib/video/config.py +++ b/src/zimscraperlib/video/config.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index ec6726d1..815cd052 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu @@ -34,14 +33,15 @@ def reencode( + [f"file:{tmp_path}"] ) logger.debug( - f"Encode {src_path} -> {dst_path} " f"video format = {dst_path.suffix}" + f"Encode {src_path} -> {dst_path} video format = {dst_path.suffix}" ) logger.debug(nicer_args_join(args)) ffmpeg = subprocess.run( args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + check=False, ) if not failsafe: ffmpeg.check_returncode() diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index 4fb7328d..de505f95 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu from .config import Config diff --git a/src/zimscraperlib/video/probing.py b/src/zimscraperlib/video/probing.py index 15b98a50..5d7a16d1 100644 --- a/src/zimscraperlib/video/probing.py +++ b/src/zimscraperlib/video/probing.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu @@ -30,7 +29,7 @@ def get_media_info(src_path): args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, check=False, ) result = ffprobe.stdout.strip().split("\n") diff --git a/src/zimscraperlib/zim/__init__.py b/src/zimscraperlib/zim/__init__.py index 44a9a601..80bee44a 100644 --- a/src/zimscraperlib/zim/__init__.py +++ b/src/zimscraperlib/zim/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ ZIM file creation tools diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 01c317a1..0b5813c4 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ ZIM Archive helper diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index a87a24e6..0817d7a3 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ ZIM Creator helper @@ -78,7 +77,6 @@ def mimetype_for( class Creator(libzim.writer.Creator): - """libzim.writer.Creator subclass Note: due to the lack of a cancel() method in the libzim itself, it is not possible diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index f047a8c6..2844577c 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ zimwriterfs-like tools to convert a build folder into a ZIM @@ -99,7 +98,7 @@ def add_redirects_to_zim( zim_file.add_redirect(source_url, target_url, title) if redirects_file: - with open(redirects_file, "r") as fh: + with open(redirects_file) as fh: for line in fh.readlines(): namespace, path, title, target_url = re.match( r"^(.)\t(.+)\t(.*)\t(.+)$", line @@ -144,11 +143,11 @@ def make_zim_file( # sanity checks if not build_dir.exists() or not build_dir.is_dir(): - raise IOError(f"Incorrect build_dir: {build_dir}") + raise OSError(f"Incorrect build_dir: {build_dir}") illustration_path = build_dir / illustration if not illustration_path.exists() or not illustration_path.is_file(): - raise IOError(f"Incorrect illustration: {illustration} ({illustration_path})") + raise OSError(f"Incorrect illustration: {illustration} ({illustration_path})") with open(illustration_path, "rb") as fh: illustration_data = fh.read() diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index 20eff63f..f4cf19b8 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu @@ -113,7 +112,7 @@ def __init__(self, url: str, **kwargs): url, byte_stream=io.BytesIO(), only_first_block=True ) except Exception as exc: - raise IOError(f"Unable to access URL at {url}: {exc}") + raise OSError(f"Unable to access URL at {url}: {exc}") # HTML content will be indexed. # we proxy the content in the Item to prevent double-download of the resource diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index 6fa74e0f..08b3fc48 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu """ libzim Providers accepting a `ref` arg to keep it away from garbage collection diff --git a/tests/conftest.py b/tests/conftest.py index a6d8be2b..c5e78583 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pathlib @@ -139,7 +138,7 @@ def small_zim_file(tmpdir_factory): dst = tmpdir_factory.mktemp("data").join("small.zim") stream_file( - "https://github.com/openzim/zim-testing-suite/raw/v0.3/data/nons/" "small.zim", + "https://github.com/openzim/zim-testing-suite/raw/v0.3/data/nons/small.zim", dst, ) return dst diff --git a/tests/download/test_download.py b/tests/download/test_download.py index a2c123d9..6e3635a6 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import concurrent.futures diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index 76524f42..bca52521 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import magic diff --git a/tests/html/conftest.py b/tests/html/conftest.py index f0f7fb22..63e12990 100644 --- a/tests/html/conftest.py +++ b/tests/html/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pytest diff --git a/tests/html/test_html.py b/tests/html/test_html.py index 320566ef..2a43e7ca 100644 --- a/tests/html/test_html.py +++ b/tests/html/test_html.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pytest diff --git a/tests/i18n/test_i18n.py b/tests/i18n/test_i18n.py index b3adad74..bb78d8a5 100644 --- a/tests/i18n/test_i18n.py +++ b/tests/i18n/test_i18n.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import locale diff --git a/tests/image/test_image.py b/tests/image/test_image.py index 5032e921..4f352d31 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import inspect diff --git a/tests/inputs/test_inputs.py b/tests/inputs/test_inputs.py index 197ce760..9377ca4c 100644 --- a/tests/inputs/test_inputs.py +++ b/tests/inputs/test_inputs.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pathlib diff --git a/tests/logging/conftest.py b/tests/logging/conftest.py index 1dd1eb05..c91f00a3 100644 --- a/tests/logging/conftest.py +++ b/tests/logging/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import io diff --git a/tests/logging/test_logging.py b/tests/logging/test_logging.py index e207d4c1..01932695 100644 --- a/tests/logging/test_logging.py +++ b/tests/logging/test_logging.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import logging @@ -19,7 +18,7 @@ def assert_message_console(logger, console, level, expected): def assert_message_file(logger, fpath, level, expected): msg = f"a {level} message" getattr(logger, level)(msg) - with open(fpath, "r") as file: + with open(fpath) as file: file.seek(0) if expected: assert msg in file.read() diff --git a/tests/ogvjs/conftest.py b/tests/ogvjs/conftest.py index dee80ab6..20d75054 100644 --- a/tests/ogvjs/conftest.py +++ b/tests/ogvjs/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pytest diff --git a/tests/ogvjs/test_ogvjs.py b/tests/ogvjs/test_ogvjs.py index fe9087f2..b6d666e2 100644 --- a/tests/ogvjs/test_ogvjs.py +++ b/tests/ogvjs/test_ogvjs.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import shutil @@ -59,7 +58,8 @@ def test_missing_param(): sys.executable, "-m", "zimscraperlib.fix_ogvjs_dist", - ] + ], + check=False, ) assert script.returncode == 1 @@ -76,11 +76,12 @@ def test_fix_ogvjs_dist(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): "zimscraperlib.fix_ogvjs_dist", str(tmp_path), ], - universal_newlines=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + check=False, ) assert script.returncode == 0 - with open(tmp_path / "videojs-ogvjs.js", "r") as fh: + with open(tmp_path / "videojs-ogvjs.js") as fh: assert "webm" in fh.read() diff --git a/tests/types/test_types.py b/tests/types/test_types.py index 3809be11..2ee602d1 100644 --- a/tests/types/test_types.py +++ b/tests/types/test_types.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pytest diff --git a/tests/video/conftest.py b/tests/video/conftest.py index 93f6fca0..d2b45de9 100644 --- a/tests/video/conftest.py +++ b/tests/video/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pathlib diff --git a/tests/video/test_video.py b/tests/video/test_video.py index ca8ed2fe..8591dc73 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import inspect diff --git a/tests/zim/conftest.py b/tests/zim/conftest.py index d73b37f4..f5a7f59b 100644 --- a/tests/zim/conftest.py +++ b/tests/zim/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pytest diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index 580e9004..c2f7781f 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import pytest diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index 41be535e..99360c81 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import shutil @@ -123,7 +122,7 @@ def test_make_zim_file_no_file_on_error(tmp_path, png_image, build_data): print("Program exiting") """ - py = subprocess.run([sys.executable, "-c", pycode]) + py = subprocess.run([sys.executable, "-c", pycode], check=False) # returncode will be either 0 or -11, depending on garbage collection # in scrapers, we want to be able to fail on errors and absolutely don't want to # create a ZIM file, so SEGFAULT on exit it (somewhat) OK diff --git a/tests/zim/test_libkiwix.py b/tests/zim/test_libkiwix.py index 1e5a2e24..02105348 100644 --- a/tests/zim/test_libkiwix.py +++ b/tests/zim/test_libkiwix.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import io diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 151a906f..a2f3be40 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu import base64 From b6f25e52f27b84cd7d4e6442b16da61e19483f40 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 16:11:05 +0100 Subject: [PATCH 064/335] Automatic ruff fixes with unsafe rules --- pyproject.toml | 2 ++ src/zimscraperlib/__init__.py | 4 ++-- src/zimscraperlib/download.py | 2 +- src/zimscraperlib/html.py | 2 +- src/zimscraperlib/image/convertion.py | 8 ++++---- src/zimscraperlib/image/optimization.py | 6 +++--- src/zimscraperlib/image/transformation.py | 4 ++-- src/zimscraperlib/inputs.py | 12 ++++++++---- src/zimscraperlib/logging.py | 2 +- src/zimscraperlib/uri.py | 4 ++-- src/zimscraperlib/video/encoding.py | 4 ++-- src/zimscraperlib/video/presets.py | 2 +- src/zimscraperlib/zim/__init__.py | 15 ++++++++++----- src/zimscraperlib/zim/archive.py | 4 ++-- src/zimscraperlib/zim/creator.py | 16 ++++++++++------ src/zimscraperlib/zim/filesystem.py | 12 ++++++------ src/zimscraperlib/zim/items.py | 9 +++++++-- src/zimscraperlib/zim/metadata.py | 6 +++--- src/zimscraperlib/zim/providers.py | 2 +- 19 files changed, 68 insertions(+), 48 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2c5c0a6c..b3cd7cd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -182,6 +182,8 @@ ignore = [ "S603", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # To be fixed once more recent Python versions are available + "UP006", "UP007" ] unfixable = [ # Don't touch unused imports diff --git a/src/zimscraperlib/__init__.py b/src/zimscraperlib/__init__.py index 4bb2781e..1ff4f461 100644 --- a/src/zimscraperlib/__init__.py +++ b/src/zimscraperlib/__init__.py @@ -4,8 +4,8 @@ import logging as stdlogging import os -from .constants import NAME -from .logging import getLogger +from zimscraperlib.constants import NAME +from zimscraperlib.logging import getLogger debug = os.getenv("ZIMSCRAPERLIB_DEBUG") logger = getLogger(NAME, level=stdlogging.DEBUG if debug else stdlogging.INFO) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 42a05309..370d26ff 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -12,7 +12,7 @@ import requests import yt_dlp as youtube_dl -from . import logger +from zimscraperlib import logger class YoutubeDownloader: diff --git a/src/zimscraperlib/html.py b/src/zimscraperlib/html.py index 42d02100..3ccc8711 100644 --- a/src/zimscraperlib/html.py +++ b/src/zimscraperlib/html.py @@ -8,7 +8,7 @@ from bs4 import BeautifulSoup -from .types import ARTICLE_MIME +from zimscraperlib.types import ARTICLE_MIME def find_title_in(content: Union[str, BinaryIO, TextIO], mime_type: str) -> str: diff --git a/src/zimscraperlib/image/convertion.py b/src/zimscraperlib/image/convertion.py index 6c7d5327..01a36c3f 100644 --- a/src/zimscraperlib/image/convertion.py +++ b/src/zimscraperlib/image/convertion.py @@ -6,10 +6,10 @@ import PIL -from ..constants import ALPHA_NOT_SUPPORTED -from .probing import format_for -from .transformation import resize_image -from .utils import save_image +from zimscraperlib.constants import ALPHA_NOT_SUPPORTED +from zimscraperlib.image.probing import format_for +from zimscraperlib.image.transformation import resize_image +from zimscraperlib.image.utils import save_image def convert_image( diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 094ff663..a394ba3a 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -35,9 +35,9 @@ from optimize_images.img_dynamic_quality import jpeg_dynamic_quality from PIL import Image -from .convertion import convert_image -from .probing import format_for -from .utils import save_image +from zimscraperlib.image.convertion import convert_image +from zimscraperlib.image.probing import format_for +from zimscraperlib.image.utils import save_image def ensure_matches( diff --git a/src/zimscraperlib/image/transformation.py b/src/zimscraperlib/image/transformation.py index 0bbfbf6c..1b604654 100644 --- a/src/zimscraperlib/image/transformation.py +++ b/src/zimscraperlib/image/transformation.py @@ -8,8 +8,8 @@ import PIL from resizeimage import resizeimage -from ..constants import ALPHA_NOT_SUPPORTED -from .utils import save_image +from zimscraperlib.constants import ALPHA_NOT_SUPPORTED +from zimscraperlib.image.utils import save_image def resize_image( diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index a20ccec2..a388d71b 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -6,10 +6,14 @@ import tempfile from typing import Optional, Tuple, Union -from . import logger -from .constants import MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH -from .constants import MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH -from .download import stream_file +from zimscraperlib import logger +from zimscraperlib.constants import ( + MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH, +) +from zimscraperlib.constants import ( + MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH, +) +from zimscraperlib.download import stream_file def handle_user_provided_file( diff --git a/src/zimscraperlib/logging.py b/src/zimscraperlib/logging.py index 5b7abb11..9f2b1c3f 100644 --- a/src/zimscraperlib/logging.py +++ b/src/zimscraperlib/logging.py @@ -8,7 +8,7 @@ from logging.handlers import RotatingFileHandler from typing import Iterable, Optional -from .constants import NAME +from zimscraperlib.constants import NAME DEFAULT_FORMAT = "[%(name)s::%(asctime)s] %(levelname)s:%(message)s" VERBOSE_DEPENDENCIES = ["urllib3", "PIL", "boto3", "botocore", "s3transfer"] diff --git a/src/zimscraperlib/uri.py b/src/zimscraperlib/uri.py index c0fdb60d..f21b5bd3 100644 --- a/src/zimscraperlib/uri.py +++ b/src/zimscraperlib/uri.py @@ -3,8 +3,8 @@ import urllib.parse from typing import Union -from . import logger -from .misc import first +from zimscraperlib import logger +from zimscraperlib.misc import first def rebuild_uri( diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index 815cd052..f0a39a52 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -7,8 +7,8 @@ import subprocess import tempfile -from .. import logger -from ..logging import nicer_args_join +from zimscraperlib import logger +from zimscraperlib.logging import nicer_args_join def reencode( diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index de505f95..b6b97c1a 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu -from .config import Config +from zimscraperlib.video.config import Config preset_type = "video" diff --git a/src/zimscraperlib/zim/__init__.py b/src/zimscraperlib/zim/__init__.py index 80bee44a..5f3f44bc 100644 --- a/src/zimscraperlib/zim/__init__.py +++ b/src/zimscraperlib/zim/__init__.py @@ -11,11 +11,16 @@ from libzim.writer import Blob -from .archive import Archive -from .creator import Creator -from .filesystem import make_zim_file -from .items import Item, StaticItem, URLItem -from .providers import FileLikeProvider, FileProvider, StringProvider, URLProvider +from zimscraperlib.zim.archive import Archive +from zimscraperlib.zim.creator import Creator +from zimscraperlib.zim.filesystem import make_zim_file +from zimscraperlib.zim.items import Item, StaticItem, URLItem +from zimscraperlib.zim.providers import ( + FileLikeProvider, + FileProvider, + StringProvider, + URLProvider, +) __all__ = [ "Archive", diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 0b5813c4..cca2527c 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -15,8 +15,8 @@ import libzim.search # Query, Searcher import libzim.suggestion # SuggestionSearcher -from ._libkiwix import convertTags, parseMimetypeCounter -from .items import Item +from zimscraperlib.zim._libkiwix import convertTags, parseMimetypeCounter +from zimscraperlib.zim.items import Item class Archive(libzim.reader.Archive): diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 0817d7a3..c4878c7e 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -25,16 +25,20 @@ import libzim.writer -from ..constants import ( +from zimscraperlib.constants import ( DEFAULT_DEV_ZIM_METADATA, FRONT_ARTICLE_MIMETYPES, MANDATORY_ZIM_METADATA_KEYS, ) -from ..filesystem import delete_callback, get_content_mimetype, get_file_mimetype -from ..i18n import is_valid_iso_639_3 -from ..types import get_mime_for_name -from .items import StaticItem -from .metadata import ( +from zimscraperlib.filesystem import ( + delete_callback, + get_content_mimetype, + get_file_mimetype, +) +from zimscraperlib.i18n import is_valid_iso_639_3 +from zimscraperlib.types import get_mime_for_name +from zimscraperlib.zim.items import StaticItem +from zimscraperlib.zim.metadata import ( validate_counter, validate_date, validate_description, diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 2844577c..6e7eb279 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -31,12 +31,12 @@ import re from typing import Optional, Sequence, Tuple -from .. import logger -from ..filesystem import get_file_mimetype -from ..html import find_title_in_file -from ..types import get_mime_for_name -from .creator import Creator -from .items import StaticItem +from zimscraperlib import logger +from zimscraperlib.filesystem import get_file_mimetype +from zimscraperlib.html import find_title_in_file +from zimscraperlib.types import get_mime_for_name +from zimscraperlib.zim.creator import Creator +from zimscraperlib.zim.items import StaticItem class FileItem(StaticItem): diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index f4cf19b8..c5633444 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -13,8 +13,13 @@ import libzim.writer -from ..download import stream_file -from .providers import FileLikeProvider, FileProvider, StringProvider, URLProvider +from zimscraperlib.download import stream_file +from zimscraperlib.zim.providers import ( + FileLikeProvider, + FileProvider, + StringProvider, + URLProvider, +) class Item(libzim.writer.Item): diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index bedcddd6..6ffc7421 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -3,15 +3,15 @@ from collections.abc import Iterable as IterableT from typing import Any, Iterable, Union -from ..constants import ( +from zimscraperlib.constants import ( ILLUSTRATIONS_METADATA_RE, MANDATORY_ZIM_METADATA_KEYS, MAXIMUM_DESCRIPTION_METADATA_LENGTH, MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH, RECOMMENDED_MAX_TITLE_LENGTH, ) -from ..i18n import is_valid_iso_639_3 -from ..image.probing import is_valid_image +from zimscraperlib.i18n import is_valid_iso_639_3 +from zimscraperlib.image.probing import is_valid_image def validate_required_values(name: str, value: Any): diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index 08b3fc48..c08a3e89 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -16,7 +16,7 @@ import libzim.writer import requests -from ..download import _get_retry_adapter, stream_file +from zimscraperlib.download import _get_retry_adapter, stream_file class FileProvider(libzim.writer.FileProvider): From 3f50609767fe882c2352291f3e82c11a981b4db2 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 16:18:55 +0100 Subject: [PATCH 065/335] Add all needed noqa statements - to be removed in the future, one by one --- src/zimscraperlib/download.py | 12 +++--- src/zimscraperlib/fix_ogvjs_dist.py | 6 +-- src/zimscraperlib/i18n.py | 8 ++-- src/zimscraperlib/image/convertion.py | 2 +- src/zimscraperlib/image/optimization.py | 30 +++++++------- src/zimscraperlib/image/presets.py | 24 +++++------ src/zimscraperlib/image/probing.py | 13 ++++-- src/zimscraperlib/image/transformation.py | 10 +++-- src/zimscraperlib/inputs.py | 2 +- src/zimscraperlib/logging.py | 6 +-- src/zimscraperlib/uri.py | 20 ++++----- src/zimscraperlib/video/config.py | 10 ++--- src/zimscraperlib/video/encoding.py | 9 +++- src/zimscraperlib/video/presets.py | 10 ++--- src/zimscraperlib/video/probing.py | 2 +- src/zimscraperlib/zim/_libkiwix.py | 38 +++++++++-------- src/zimscraperlib/zim/archive.py | 2 +- src/zimscraperlib/zim/creator.py | 50 +++++++++++++---------- src/zimscraperlib/zim/filesystem.py | 24 +++++------ src/zimscraperlib/zim/items.py | 4 +- src/zimscraperlib/zim/metadata.py | 6 +-- src/zimscraperlib/zim/providers.py | 2 +- tests/download/test_download.py | 10 ++--- tests/filesystem/test_filesystem.py | 4 +- tests/html/conftest.py | 2 +- tests/html/test_html.py | 4 +- tests/image/test_image.py | 8 ++-- tests/logging/test_logging.py | 10 ++--- tests/ogvjs/test_ogvjs.py | 4 +- tests/zim/test_archive.py | 2 +- tests/zim/test_libkiwix.py | 4 +- tests/zim/test_zim_creator.py | 14 +++---- 32 files changed, 189 insertions(+), 163 deletions(-) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 370d26ff..ed3643f7 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -42,7 +42,7 @@ def download( self, url: str, options: Optional[Dict], - wait: Optional[bool] = True, + wait: Optional[bool] = True, # noqa: FBT002 ) -> Union[bool, Future]: """Downloads video using initialized executor. @@ -63,8 +63,8 @@ def download( class YoutubeConfig(dict): - options = {} - defaults = { + options = {} # noqa: RUF012 + defaults = { # noqa: RUF012 "writethumbnail": True, "write_all_thumbnails": True, "writesubtitles": True, @@ -105,14 +105,14 @@ def get_options( class BestWebm(YoutubeConfig): - options = { + options = { # noqa: RUF012 "preferredcodec": "webm", "format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best", } class BestMp4(YoutubeConfig): - options = { + options = { # noqa: RUF012 "preferredcodec": "mp4", "format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best", } @@ -171,7 +171,7 @@ def stream_file( byte_stream: Optional[io.BytesIO] = None, block_size: Optional[int] = 1024, proxies: Optional[dict] = None, - only_first_block: Optional[bool] = False, + only_first_block: Optional[bool] = False, # noqa: FBT002 max_retries: Optional[int] = 5, headers: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None, diff --git a/src/zimscraperlib/fix_ogvjs_dist.py b/src/zimscraperlib/fix_ogvjs_dist.py index 7bfea969..00d1b5e2 100755 --- a/src/zimscraperlib/fix_ogvjs_dist.py +++ b/src/zimscraperlib/fix_ogvjs_dist.py @@ -34,9 +34,9 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]): def run(): - if len(sys.argv) < 2: - print(f"Usage: {sys.argv[0]} ") - print( + if len(sys.argv) < 2: # noqa: PLR2004 + print(f"Usage: {sys.argv[0]} ") # noqa: T201 + print( # noqa: T201 "\t\tpath to your folder containing " "ogvjs/videojs/videojs-ogvjs." ) diff --git a/src/zimscraperlib/i18n.py b/src/zimscraperlib/i18n.py index 87b51fdf..90991944 100644 --- a/src/zimscraperlib/i18n.py +++ b/src/zimscraperlib/i18n.py @@ -13,7 +13,7 @@ ISO_LEVELS = ["1", "2b", "2t", "3", "5"] -class NotFound(ValueError): +class NotFound(ValueError): # noqa: N818 pass @@ -126,7 +126,9 @@ def update_with_macro(lang_data: Dict, macro_data: Dict): return lang_data -def get_language_details(query: str, failsafe: Optional[bool] = False) -> Dict: +def get_language_details( + query: str, failsafe: Optional[bool] = False # noqa: FBT002 +) -> Dict: """language details dict from query. Raises NotFound or return `und` language details if failsafe @@ -142,7 +144,7 @@ def get_language_details(query: str, failsafe: Optional[bool] = False) -> Dict: """ - if query.isalpha() and (2 <= len(query) <= 3): + if query.isalpha() and (2 <= len(query) <= 3): # noqa: PLR2004 # possibility of iso-639 code adjusted_query = query native_query = query diff --git a/src/zimscraperlib/image/convertion.py b/src/zimscraperlib/image/convertion.py index 01a36c3f..e1460d52 100644 --- a/src/zimscraperlib/image/convertion.py +++ b/src/zimscraperlib/image/convertion.py @@ -30,7 +30,7 @@ def convert_image( fmt = format_for(dst) with PIL.Image.open(src) as image: if image.mode == "RGBA" and fmt in ALPHA_NOT_SUPPORTED or colorspace: - image = image.convert(colorspace or "RGB") + image = image.convert(colorspace or "RGB") # noqa: PLW2901 save_image(image, dst, fmt, **params) diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index a394ba3a..935a5be2 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -53,12 +53,12 @@ def ensure_matches( def optimize_png( src: Union[pathlib.Path, io.BytesIO], dst: Optional[pathlib.Path] = None, - reduce_colors: Optional[bool] = False, + reduce_colors: Optional[bool] = False, # noqa: FBT002 max_colors: Optional[int] = 256, - fast_mode: Optional[bool] = True, - remove_transparency: Optional[bool] = False, + fast_mode: Optional[bool] = True, # noqa: FBT002 + remove_transparency: Optional[bool] = False, # noqa: FBT002 background_color: Optional[Tuple[int, int, int]] = (255, 255, 255), - **options, + **options, # noqa: ARG001 ) -> Union[pathlib.Path, io.BytesIO]: """method to optimize PNG files using a pure python external optimizer @@ -101,9 +101,9 @@ def optimize_jpeg( src: Union[pathlib.Path, io.BytesIO], dst: Optional[pathlib.Path] = None, quality: Optional[int] = 85, - fast_mode: Optional[bool] = True, - keep_exif: Optional[bool] = True, - **options, + fast_mode: Optional[bool] = True, # noqa: FBT002 + keep_exif: Optional[bool] = True, # noqa: FBT002 + **options, # noqa: ARG001 ) -> Union[pathlib.Path, io.BytesIO]: """method to optimize JPEG files using a pure python external optimizer quality: JPEG quality (integer between 1 and 100) @@ -130,7 +130,7 @@ def optimize_jpeg( had_exif = True # only use progressive if file size is bigger - use_progressive_jpg = orig_size > 10240 # 10KiB + use_progressive_jpg = orig_size > 10240 # 10KiB # noqa: PLR2004 if fast_mode: quality_setting = quality @@ -168,10 +168,10 @@ def optimize_jpeg( def optimize_webp( src: Union[pathlib.Path, io.BytesIO], dst: Optional[pathlib.Path] = None, - lossless: Optional[bool] = False, + lossless: Optional[bool] = False, # noqa: FBT002 quality: Optional[int] = 60, method: Optional[int] = 6, - **options, + **options, # noqa: ARG001 ) -> Union[pathlib.Path, io.BytesIO]: """method to optimize WebP using Pillow options lossless: Whether to use lossless compression (boolean) @@ -213,10 +213,10 @@ def optimize_gif( dst: pathlib.Path, optimize_level: Optional[int] = 1, lossiness: Optional[int] = None, - interlace: Optional[bool] = True, - no_extensions: Optional[bool] = True, + interlace: Optional[bool] = True, # noqa: FBT002 + no_extensions: Optional[bool] = True, # noqa: FBT002 max_colors: Optional[int] = None, - **options, + **options, # noqa: ARG001 ) -> pathlib.Path: """method to optimize GIFs using gifsicle >= 1.92 optimize_level: Optimization level; @@ -264,8 +264,8 @@ def optimize_gif( def optimize_image( src: pathlib.Path, dst: pathlib.Path, - delete_src: Optional[bool] = False, - convert: Optional[Union[bool, str]] = False, + delete_src: Optional[bool] = False, # noqa: FBT002 + convert: Optional[Union[bool, str]] = False, # noqa: FBT002 **options, ) -> bool: """Optimize image, automatically selecting correct optimizer diff --git a/src/zimscraperlib/image/presets.py b/src/zimscraperlib/image/presets.py index 230fb299..5e4ebb38 100644 --- a/src/zimscraperlib/image/presets.py +++ b/src/zimscraperlib/image/presets.py @@ -18,7 +18,7 @@ class WebpLow: ext = "webp" mimetype = f"{preset_type}/webp" - options = { + options = { # noqa: RUF012 "lossless": False, "quality": 40, "method": 6, @@ -37,7 +37,7 @@ class WebpMedium: ext = "webp" mimetype = f"{preset_type}/webp" - options = { + options = { # noqa: RUF012 "lossless": False, "quality": 50, "method": 6, @@ -56,7 +56,7 @@ class WebpHigh: ext = "webp" mimetype = f"{preset_type}/webp" - options = { + options = { # noqa: RUF012 "lossless": False, "quality": 90, "method": 6, @@ -77,7 +77,7 @@ class GifLow: ext = "gif" mimetype = f"{preset_type}/gif" - options = { + options = { # noqa: RUF012 "optimize_level": 3, "max_colors": 256, "lossiness": 80, @@ -100,7 +100,7 @@ class GifMedium: ext = "gif" mimetype = f"{preset_type}/gif" - options = { + options = { # noqa: RUF012 "optimize_level": 3, "lossiness": 20, "no_extensions": True, @@ -122,7 +122,7 @@ class GifHigh: ext = "gif" mimetype = f"{preset_type}/gif" - options = { + options = { # noqa: RUF012 "optimize_level": 2, "lossiness": None, "no_extensions": True, @@ -141,7 +141,7 @@ class PngLow: ext = "png" mimetype = f"{preset_type}/png" - options = { + options = { # noqa: RUF012 "reduce_colors": True, "remove_transparency": False, "max_colors": 256, @@ -160,7 +160,7 @@ class PngMedium: ext = "png" mimetype = f"{preset_type}/png" - options = { + options = { # noqa: RUF012 "reduce_colors": False, "remove_transparency": False, "fast_mode": False, @@ -178,7 +178,7 @@ class PngHigh: ext = "png" mimetype = f"{preset_type}/png" - options = { + options = { # noqa: RUF012 "reduce_colors": False, "remove_transparency": False, "fast_mode": True, @@ -197,7 +197,7 @@ class JpegLow: ext = "png" mimetype = f"{preset_type}/png" - options = { + options = { # noqa: RUF012 "quality": 45, "keep_exif": False, "fast_mode": True, @@ -216,7 +216,7 @@ class JpegMedium: ext = "jpg" mimetype = f"{preset_type}/jpeg" - options = { + options = { # noqa: RUF012 "quality": 65, "keep_exif": False, "fast_mode": True, @@ -235,7 +235,7 @@ class JpegHigh: ext = "jpg" mimetype = f"{preset_type}/jpeg" - options = { + options = { # noqa: RUF012 "quality": 80, "keep_exif": True, "fast_mode": True, diff --git a/src/zimscraperlib/image/probing.py b/src/zimscraperlib/image/probing.py index 42d2277c..d17b89b1 100644 --- a/src/zimscraperlib/image/probing.py +++ b/src/zimscraperlib/image/probing.py @@ -12,7 +12,7 @@ def get_colors( - src: pathlib.Path, use_palette: Optional[bool] = True + src: pathlib.Path, use_palette: Optional[bool] = True # noqa: FBT002 ) -> Tuple[str, str]: """(main, secondary) HTML color codes from an image path""" @@ -22,7 +22,9 @@ def rgb_to_hex(r: int, g: int, b: int) -> str: def solarize(r: int, g: int, b: int) -> Tuple[int, int, int]: # calculate solarized color for main - h, l, s = colorsys.rgb_to_hls(float(r) / 256, float(g) / 256, float(b) / 256) + h, l, s = colorsys.rgb_to_hls( # noqa: E741 + float(r) / 256, float(g) / 256, float(b) / 256 + ) r2, g2, b2 = (int(x * 256) for x in colorsys.hls_to_rgb(h, 0.95, s)) return r2, g2, b2 @@ -48,13 +50,16 @@ def is_hex_color(text: str) -> bool: return re.search(r"^#(?:[0-9a-fA-F]{3}){1,2}$", text) -def format_for(src: Union[pathlib.Path, io.BytesIO], from_suffix: bool = True) -> str: +def format_for( + src: Union[pathlib.Path, io.BytesIO], + from_suffix: bool = True, # noqa: FBT001, FBT002 +) -> str: """Pillow format of a given filename, either Pillow-detected or from suffix""" if not from_suffix: with PIL.Image.open(src) as img: return img.format - from PIL.Image import EXTENSION as ext_fmt_map + from PIL.Image import EXTENSION as ext_fmt_map # noqa: N811 from PIL.Image import init as init_pil init_pil() diff --git a/src/zimscraperlib/image/transformation.py b/src/zimscraperlib/image/transformation.py index 1b604654..93d461c9 100644 --- a/src/zimscraperlib/image/transformation.py +++ b/src/zimscraperlib/image/transformation.py @@ -18,7 +18,7 @@ def resize_image( height: Optional[int] = None, dst: Optional[Union[pathlib.Path, io.BytesIO]] = None, method: Optional[str] = "width", - allow_upscaling: Optional[bool] = True, + allow_upscaling: Optional[bool] = True, # noqa: FBT002 **params: Optional[dict], ) -> None: """resize an image to requested dimensions @@ -34,9 +34,13 @@ def resize_image( if allow_upscaling: height_width_ratio = float(image.size[1]) / float(image.size[0]) if image.size[0] < width: - image = image.resize((width, int(width * height_width_ratio))) + image = image.resize( # noqa: PLW2901 + (width, int(width * height_width_ratio)) + ) if height and image.size[1] < height: - image = image.resize((int(height / height_width_ratio), height)) + image = image.resize( # noqa: PLW2901 + (int(height / height_width_ratio), height) + ) # resize using the requested method if method == "width": diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index a388d71b..4ab9e6ef 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -20,7 +20,7 @@ def handle_user_provided_file( source: Optional[Union[pathlib.Path, str]] = None, dest: Optional[pathlib.Path] = None, in_dir: Optional[pathlib.Path] = None, - nocopy: bool = False, + nocopy: bool = False, # noqa: FBT001, FBT002 ) -> Union[pathlib.Path, None]: """path to downloaded or copied a user provided file (URL or path) diff --git a/src/zimscraperlib/logging.py b/src/zimscraperlib/logging.py index 9f2b1c3f..8ef0e54c 100644 --- a/src/zimscraperlib/logging.py +++ b/src/zimscraperlib/logging.py @@ -14,17 +14,17 @@ VERBOSE_DEPENDENCIES = ["urllib3", "PIL", "boto3", "botocore", "s3transfer"] -def getLogger( +def getLogger( # noqa: N802 name: str, level: Optional[int] = logging.INFO, console: Optional[io.TextIOBase] = sys.stdout, log_format: Optional[str] = DEFAULT_FORMAT, - file: Optional[pathlib.Path] = False, + file: Optional[pathlib.Path] = False, # noqa: FBT002 file_level: Optional[int] = None, file_format: Optional[str] = None, file_max: Optional[int] = 2**20, file_nb_backup: Optional[int] = 1, - deps_level: Optional[int] = logging.WARNING, + deps_level: Optional[int] = logging.WARNING, # noqa: ARG001 additional_deps: Optional[Iterable] = None, ): """configured logger for most usages diff --git a/src/zimscraperlib/uri.py b/src/zimscraperlib/uri.py index f21b5bd3..7fa43b96 100644 --- a/src/zimscraperlib/uri.py +++ b/src/zimscraperlib/uri.py @@ -9,16 +9,16 @@ def rebuild_uri( uri: urllib.parse.ParseResult, - scheme: str = None, - username: str = None, - password: str = None, - hostname: str = None, - port: Union[str, int] = None, - path: str = None, - params: str = None, - query: str = None, - fragment: str = None, - failsafe: bool = False, + scheme: str = None, # noqa: RUF013 + username: str = None, # noqa: RUF013 + password: str = None, # noqa: RUF013 + hostname: str = None, # noqa: RUF013 + port: Union[str, int] = None, # noqa: RUF013 + path: str = None, # noqa: RUF013 + params: str = None, # noqa: RUF013 + query: str = None, # noqa: RUF013 + fragment: str = None, # noqa: RUF013 + failsafe: bool = False, # noqa: FBT001, FBT002 ) -> urllib.parse.ParseResult: """new ParseResult named tuple from uri with requested part updated""" try: diff --git a/src/zimscraperlib/video/config.py b/src/zimscraperlib/video/config.py index 5097f21c..6fb4b6a0 100644 --- a/src/zimscraperlib/video/config.py +++ b/src/zimscraperlib/video/config.py @@ -6,9 +6,9 @@ class Config(dict): VERSION = 1 ext = "dat" mimetype = "application/data" - options = {} - defaults = {"-max_muxing_queue_size": "9999"} - mapping = { + options = {} # noqa: RUF012 + defaults = {"-max_muxing_queue_size": "9999"} # noqa: RUF012 + mapping = { # noqa: RUF012 "video_codec": "-codec:v", "audio_codec": "-codec:a", "max_video_bitrate": "-maxrate", @@ -128,8 +128,8 @@ def quantizer_scale_range(self, value): if ( isinstance(qmin, int) and isinstance(qmax, int) - and -1 <= qmin <= 69 - and -1 <= qmax <= 1024 + and -1 <= qmin <= 69 # noqa: PLR2004 + and -1 <= qmax <= 1024 # noqa: PLR2004 ): self["-qmin"] = str(qmin) self["-qmax"] = str(qmax) diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index f0a39a52..79100f3a 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -12,7 +12,12 @@ def reencode( - src_path, dst_path, ffmpeg_args, delete_src=False, with_process=False, failsafe=True + src_path, + dst_path, + ffmpeg_args, + delete_src=False, # noqa: FBT002 + with_process=False, # noqa: FBT002 + failsafe=True, # noqa: FBT002 ): """Runs ffmpeg with given ffmpeg_args @@ -28,7 +33,7 @@ def reencode( with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = pathlib.Path(tmp_dir).joinpath(f"video.tmp{dst_path.suffix}") args = ( - ["ffmpeg", "-y", "-i", f"file:{src_path}"] + ["ffmpeg", "-y", "-i", f"file:{src_path}"] # noqa: RUF005 + ffmpeg_args + [f"file:{tmp_path}"] ) diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index b6b97c1a..31622a67 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -18,7 +18,7 @@ class VoiceMp3Low(Config): ext = "mp3" mimetype = "audio/mp3" - options = { + options = { # noqa: RUF012 "-vn": "", # remove video stream "-codec:a": "mp3", # audio codec "-ar": "44100", # audio sampling rate @@ -39,7 +39,7 @@ class VideoWebmLow(Config): ext = "webm" mimetype = f"{preset_type}/webm" - options = { + options = { # noqa: RUF012 "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset "-b:v": "300k", # target video bitrate @@ -67,7 +67,7 @@ class VideoMp4Low(Config): ext = "mp4" mimetype = f"{preset_type}/mp4" - options = { + options = { # noqa: RUF012 "-codec:v": "h264", # video codec "-b:v": "300k", # target video bitrate "-maxrate": "300k", # max video bitrate @@ -92,7 +92,7 @@ class VideoWebmHigh(Config): ext = "webm" mimetype = f"{preset_type}/webm" - options = { + options = { # noqa: RUF012 "-codec:v": "libvpx", # video codec "-codec:a": "libvorbis", # audio codec "-crf": "25", # constant quality, lower value gives better qual and larger size @@ -110,7 +110,7 @@ class VideoMp4High(Config): ext = "mp4" mimetype = f"{preset_type}/mp4" - options = { + options = { # noqa: RUF012 "-codec:v": "h264", # video codec "-codec:a": "aac", # audio codec "-crf": "20", # constant quality, lower value gives better qual and larger size diff --git a/src/zimscraperlib/video/probing.py b/src/zimscraperlib/video/probing.py index 5d7a16d1..fbad0859 100644 --- a/src/zimscraperlib/video/probing.py +++ b/src/zimscraperlib/video/probing.py @@ -25,7 +25,7 @@ def get_media_info(src_path): "-of", "csv", ] - ffprobe = subprocess.run( + ffprobe = subprocess.run( # noqa: UP022 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index d71db702..40f0d823 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -38,27 +38,29 @@ def getline(src: io.StringIO, delim: Optional[bool] = None) -> Tuple[bool, str]: return char == "", output -def readFullMimetypeAndCounterString(src: io.StringIO) -> Tuple[bool, str]: +def readFullMimetypeAndCounterString( # noqa: N802 + src: io.StringIO, +) -> Tuple[bool, str]: """read a single mimetype-and-counter string from source Returns whether the source is EOF and the extracted string (or empty one)""" params = "" - eof, mtcStr = getline(src, ";") + eof, mtcStr = getline(src, ";") # noqa: N806 if mtcStr.find("=") == -1: - while params.count("=") != 2: + while params.count("=") != 2: # noqa: PLR2004 eof, params = getline(src, ";") - if params.count("=") == 2: - mtcStr += ";" + params + if params.count("=") == 2: # noqa: PLR2004 + mtcStr += ";" + params # noqa: N806 if eof: break return eof, mtcStr -def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: +def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: # noqa: N802 """MimetypeAndCounter from a single mimetype-and-counter string""" k: int = string.rfind("=") if k != len(string) - 1: - mimeType = string[:k] + mimeType = string[:k] # noqa: N806 counter = string[k + 1 :] if counter: try: @@ -68,15 +70,15 @@ def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: return MimetypeAndCounter("", 0) -def parseMimetypeCounter( - counterData: str, +def parseMimetypeCounter( # noqa: N802 + counterData: str, # noqa: N803 ) -> CounterMap: """Mapping of MIME types with count for each from ZIM Counter metadata string""" - counters = dict() + counters = dict() # noqa: C408 ss = io.StringIO(counterData) eof = False while not eof: - eof, mtcStr = readFullMimetypeAndCounterString(ss) + eof, mtcStr = readFullMimetypeAndCounterString(ss) # noqa: N806 mtc = parseASingleMimetypeCounter(mtcStr) if mtc.mimetype: counters.update([mtc]) @@ -84,19 +86,19 @@ def parseMimetypeCounter( return counters -def convertTags(tags_str: str) -> List[str]: +def convertTags(tags_str: str) -> List[str]: # noqa: N802 """List of tags expanded with libkiwix's additional hints for pic/vid/det/index""" tags = tags_str.split(";") - tagsList = [] - picSeen = vidSeen = detSeen = indexSeen = False + tagsList = [] # noqa: N806 + picSeen = vidSeen = detSeen = indexSeen = False # noqa: N806 for tag in tags: # not upstream if not tag: continue - picSeen |= tag == "nopic" or tag.startswith("_pictures:") - vidSeen |= tag == "novid" or tag.startswith("_videos:") - detSeen |= tag == "nodet" or tag.startswith("_details:") - indexSeen |= tag.startswith("_ftindex") + picSeen |= tag == "nopic" or tag.startswith("_pictures:") # noqa: N806 + vidSeen |= tag == "novid" or tag.startswith("_videos:") # noqa: N806 + detSeen |= tag == "nodet" or tag.startswith("_details:") # noqa: N806 + indexSeen |= tag.startswith("_ftindex") # noqa: N806 if tag == "nopic": tagsList.append("_pictures:no") diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index cca2527c..a9b46bd8 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -39,7 +39,7 @@ def metadata(self) -> Dict[str, str]: def tags(self): return self.get_tags() - def get_tags(self, libkiwix: bool = False) -> List[str]: + def get_tags(self, libkiwix: bool = False) -> List[str]: # noqa: FBT001, FBT002 """List of ZIM tags, optionnaly expanded with libkiwix's hints""" try: tags_meta = self.get_text_metadata("Tags") diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index c4878c7e..16e39a8a 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -101,11 +101,11 @@ def __init__( filename: pathlib.Path, main_path: str, compression: Optional[str] = None, - workaround_nocancel: Optional[bool] = True, - ignore_duplicates: Optional[bool] = False, + workaround_nocancel: Optional[bool] = True, # noqa: FBT002 + ignore_duplicates: Optional[bool] = False, # noqa: FBT002 ): super().__init__(filename=filename) - self._metadata = dict() + self._metadata = dict() # noqa: C408 self.__indexing_configured = False self.can_finish = True @@ -121,7 +121,9 @@ def __init__( self.workaround_nocancel = workaround_nocancel self.ignore_duplicates = ignore_duplicates - def config_indexing(self, indexing: bool, language: Optional[str] = None): + def config_indexing( + self, indexing: bool, language: Optional[str] = None # noqa: FBT001 + ): """Toggle full-text and title indexing of entries Uses Language metadata's value (or "") if not set""" @@ -133,7 +135,11 @@ def config_indexing(self, indexing: bool, language: Optional[str] = None): return self def start(self): - if not all([self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS]): + if not all( + [ # noqa: C419 + self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS + ] # noqa: C419, RUF100 + ): raise ValueError("Mandatory metadata are not all set.") for name, value in self._metadata.items(): @@ -188,21 +194,21 @@ def add_metadata( def config_metadata( self, *, - Name: str, - Language: str, - Title: str, - Description: str, - LongDescription: Optional[str] = None, - Creator: str, - Publisher: str, - Date: Union[datetime.datetime, datetime.date, str], - Illustration_48x48_at_1: bytes, - Tags: Optional[Union[Iterable[str], str]] = None, - Scraper: Optional[str] = None, - Flavour: Optional[str] = None, - Source: Optional[str] = None, - License: Optional[str] = None, - Relation: Optional[str] = None, + Name: str, # noqa: N803 + Language: str, # noqa: N803 + Title: str, # noqa: N803 + Description: str, # noqa: N803 + LongDescription: Optional[str] = None, # noqa: N803 + Creator: str, # noqa: N803 + Publisher: str, # noqa: N803 + Date: Union[datetime.datetime, datetime.date, str], # noqa: N803 + Illustration_48x48_at_1: bytes, # noqa: N803 + Tags: Optional[Union[Iterable[str], str]] = None, # noqa: N803 + Scraper: Optional[str] = None, # noqa: N803 + Flavour: Optional[str] = None, # noqa: N803 + Source: Optional[str] = None, # noqa: N803 + License: Optional[str] = None, # noqa: N803 + Relation: Optional[str] = None, # noqa: N803 **extras: str, ): """Sets all mandatory Metadata as well as standard and any other text ones""" @@ -243,7 +249,7 @@ def add_item_for( mimetype: Optional[str] = None, is_front: Optional[bool] = None, should_compress: Optional[bool] = None, - delete_fpath: Optional[bool] = False, + delete_fpath: Optional[bool] = False, # noqa: FBT002 duplicate_ok: Optional[bool] = None, callback: Optional[Union[callable, Tuple[callable, Any]]] = None, ): @@ -357,7 +363,7 @@ def add_redirect( self.can_finish = False # pragma: no cover raise - def finish(self, exc_type=None, exc_val=None, exc_tb=None): + def finish(self, exc_type=None, exc_val=None, exc_tb=None): # noqa: ARG002 """Triggers finalization of ZIM creation and create final ZIM file.""" if not getattr(self, "can_finish", False): return diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 6e7eb279..9b0f3d38 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -116,20 +116,20 @@ def make_zim_file( illustration: str, title: str, description: str, - date: datetime.date = None, + date: datetime.date = None, # noqa: RUF013 language: str = "eng", creator: str = "-", publisher="-", - tags: Sequence[str] = None, - source: str = None, - flavour: str = None, - scraper: str = None, - long_description: str = None, - without_fulltext_index: bool = False, - redirects: Sequence[Tuple[str, str, str]] = None, - redirects_file: pathlib.Path = None, - rewrite_links: bool = True, - workaround_nocancel: bool = True, + tags: Sequence[str] = None, # noqa: RUF013 + source: str = None, # noqa: RUF013 + flavour: str = None, # noqa: RUF013 + scraper: str = None, # noqa: RUF013 + long_description: str = None, # noqa: RUF013 + without_fulltext_index: bool = False, # noqa: FBT001, FBT002, ARG001 + redirects: Sequence[Tuple[str, str, str]] = None, # noqa: RUF013 + redirects_file: pathlib.Path = None, # noqa: RUF013 + rewrite_links: bool = True, # noqa: FBT001, FBT002, ARG001 + workaround_nocancel: bool = True, # noqa: FBT001, FBT002 ): """Creates a zimwriterfs-like ZIM file at {fpath} from {build_dir} @@ -160,7 +160,7 @@ def make_zim_file( "Name": name, "Title": title, "Description": description, - "Date": date or datetime.date.today(), + "Date": date or datetime.date.today(), # noqa: DTZ011 "Language": language, "Creator": creator, "Publisher": publisher, diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index c5633444..b83983e0 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -46,7 +46,7 @@ def get_mimetype(self) -> str: return getattr(self, "mimetype", "") def get_hints(self) -> dict: - return getattr(self, "hints", dict()) + return getattr(self, "hints", dict()) # noqa: C408 class StaticItem(Item): @@ -117,7 +117,7 @@ def __init__(self, url: str, **kwargs): url, byte_stream=io.BytesIO(), only_first_block=True ) except Exception as exc: - raise OSError(f"Unable to access URL at {url}: {exc}") + raise OSError(f"Unable to access URL at {url}: {exc}") # noqa: B904 # HTML content will be indexed. # we proxy the content in the Item to prevent double-download of the resource diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index 6ffc7421..1a6bfe8c 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -55,7 +55,7 @@ def validate_date(name: str, value: Union[datetime.datetime, datetime.date, str] try: datetime.date(**{k: int(v) for k, v in match.groupdict().items()}) except Exception as exc: - raise ValueError(f"Invalid {name} format: {exc}") + raise ValueError(f"Invalid {name} format: {exc}") # noqa: B904 def validate_language(name: str, value: Union[Iterable[str], str]): @@ -68,7 +68,7 @@ def validate_language(name: str, value: Union[Iterable[str], str]): raise ValueError(f"{code} is not ISO-639-3.") -def validate_counter(name: str, value: str): +def validate_counter(name: str, value: str): # noqa: ARG001 """ensures Counter metadata is not manually set""" if name == "Counter": raise ValueError(f"{name} cannot be set. libzim sets it.") @@ -93,7 +93,7 @@ def validate_tags(name: str, value: Union[Iterable[str], str]): """ensures Tags metadata is either one or a list of strings""" if name == "Tags" and ( not isinstance(value, IterableT) - or not all([isinstance(tag, str) for tag in value]) + or not all([isinstance(tag, str) for tag in value]) # noqa: C419 ): raise ValueError(f"Invalid type(s) for {name}") diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index c08a3e89..8140cc09 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -23,7 +23,7 @@ class FileProvider(libzim.writer.FileProvider): def __init__( self, filepath: pathlib.Path, - size: Optional[int] = None, + size: Optional[int] = None, # noqa: ARG002 ref: Optional[object] = None, ): super().__init__(filepath) diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 6e3635a6..c49b65de 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -20,7 +20,7 @@ def assert_downloaded_file(url, file): assert file.exists() # our google test urls dont support HEAD - req = requests.get(url) + req = requests.get(url) # noqa: S113 # we test against binary response: Content-Length not accurate as gzip-encoded assert file.stat().st_size == len(req.content) @@ -34,7 +34,7 @@ def get_dest_file(tmp_path): return tmp_path.joinpath("favicon.ico") -def test_missing_dest(tmp_path): +def test_missing_dest(tmp_path): # noqa: ARG001 with pytest.raises(requests.exceptions.ConnectionError): stream_file(url="http://some_url", byte_stream=io.BytesIO()) @@ -97,7 +97,7 @@ def test_stream_to_bytes(valid_https_url): byte_stream = io.BytesIO() size, ret = stream_file(url=valid_https_url, byte_stream=byte_stream) assert_headers(ret) - assert byte_stream.read() == requests.get(valid_https_url).content + assert byte_stream.read() == requests.get(valid_https_url).content # noqa: S113 @pytest.mark.slow @@ -166,9 +166,9 @@ def test_youtube_download_nowait(tmp_path): @pytest.mark.slow -def test_youtube_download_error(tmp_path): +def test_youtube_download_error(tmp_path): # noqa: ARG001 yt_downloader = YoutubeDownloader(threads=1) - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 yt_downloader.download("11", BestMp4.get_options()) yt_downloader.shutdown() diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index bca52521..9da91a1b 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -28,7 +28,7 @@ def test_content_mimetype_fallback(monkeypatch, undecodable_byte_stream): assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" # mock then so we keep coverage on systems where magic works - def raising_magic(*args): + def raising_magic(*args): # noqa: ARG001 raise UnicodeDecodeError("nocodec", b"", 0, 1, "noreason") monkeypatch.setattr(magic, "detect_from_content", raising_magic) @@ -47,7 +47,7 @@ def test_delete_callback(tmp_path): class Store: called = 0 - def cb(*args): + def cb(*args): # noqa: ARG001 Store.called += 1 fpath = tmp_path.joinpath("my-file") diff --git a/tests/html/conftest.py b/tests/html/conftest.py index 63e12990..b87ac0f1 100644 --- a/tests/html/conftest.py +++ b/tests/html/conftest.py @@ -22,4 +22,4 @@ def html_page(): -""" +""" # noqa: RUF001 diff --git a/tests/html/test_html.py b/tests/html/test_html.py index 2a43e7ca..988f428a 100644 --- a/tests/html/test_html.py +++ b/tests/html/test_html.py @@ -15,7 +15,7 @@ def test_find_title(tmp_path, html_page): # find title in example HTML assert ( find_title_in(html_page, "text/html") - == "Kiwix lets you access free knowledge – even offline" + == "Kiwix lets you access free knowledge – even offline" # noqa: RUF001 ) # make sure non-HTML returns no title assert find_title_in(html_page, "text/plain") == "" @@ -28,7 +28,7 @@ def test_find_title(tmp_path, html_page): fh.write(html_page) assert ( find_title_in_file(fpath, "text/html") - == "Kiwix lets you access free knowledge – even offline" + == "Kiwix lets you access free knowledge – even offline" # noqa: RUF001 ) # make sure non-HTML returns no title (from file) assert find_title_in_file(fpath, "text/plain") == "" diff --git a/tests/image/test_image.py b/tests/image/test_image.py index 4f352d31..e8327d96 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -270,7 +270,7 @@ def test_change_image_format( assert dst_image.format == dst_fmt -def test_change_image_format_defaults(png_image, jpg_image, tmp_path): +def test_change_image_format_defaults(png_image, jpg_image, tmp_path): # noqa: ARG001 # PNG to JPEG (loosing alpha) dst = tmp_path.joinpath(f"{png_image.stem}.jpg") convert_image(png_image, dst) @@ -535,13 +535,13 @@ def test_optimize_webp_gif_failure(tmp_path, webp_image, gif_image): dst = tmp_path.joinpath("image.img") # webp - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 optimize_webp(webp_image, dst, lossless="bad") assert not dst.exists() # gif dst.touch() # fake temp file created during optim (actually fails before) - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 optimize_gif(gif_image, dst, optimize_level="bad") assert not dst.exists() @@ -549,7 +549,7 @@ def test_optimize_webp_gif_failure(tmp_path, webp_image, gif_image): def test_wrong_extension_optim(tmp_path, png_image): dst = tmp_path.joinpath("image.jpg") shutil.copy(png_image, dst) - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 optimize_jpeg(dst, dst) diff --git a/tests/logging/test_logging.py b/tests/logging/test_logging.py index 01932695..3e7421d3 100644 --- a/tests/logging/test_logging.py +++ b/tests/logging/test_logging.py @@ -100,14 +100,14 @@ def test_debug_level_file(random_id, tmp_path): assert_message_file(logger, log_file, "debug", True) -def test_info_level_file(random_id, console, tmp_path): +def test_info_level_file(random_id, console, tmp_path): # noqa: ARG001 log_file = tmp_path / "test.log" logger = getLogger(name=random_id, file=log_file, file_level=logging.INFO) assert_message_file(logger, log_file, "debug", False) assert_message_file(logger, log_file, "info", True) -def test_warning_level_file(random_id, console, tmp_path): +def test_warning_level_file(random_id, console, tmp_path): # noqa: ARG001 log_file = tmp_path / "test.log" logger = getLogger(name=random_id, file=log_file, file_level=logging.WARNING) assert_message_file(logger, log_file, "debug", False) @@ -115,7 +115,7 @@ def test_warning_level_file(random_id, console, tmp_path): assert_message_file(logger, log_file, "warning", True) -def test_error_level_file(random_id, console, tmp_path): +def test_error_level_file(random_id, console, tmp_path): # noqa: ARG001 log_file = tmp_path / "test.log" logger = getLogger(name=random_id, file=log_file, file_level=logging.ERROR) assert_message_file(logger, log_file, "debug", False) @@ -124,7 +124,7 @@ def test_error_level_file(random_id, console, tmp_path): assert_message_file(logger, log_file, "error", True) -def test_critical_level_file(random_id, console, tmp_path): +def test_critical_level_file(random_id, console, tmp_path): # noqa: ARG001 log_file = tmp_path / "test.log" logger = getLogger(name=random_id, file=log_file, file_level=logging.CRITICAL) assert_message_file(logger, log_file, "debug", False) @@ -134,7 +134,7 @@ def test_critical_level_file(random_id, console, tmp_path): assert_message_file(logger, log_file, "critical", True) -def test_level_fallback(random_id, console, tmp_path): +def test_level_fallback(random_id, console, tmp_path): # noqa: ARG001 log_file = tmp_path / "test.log" logger = getLogger(name=random_id, file=log_file, level=logging.CRITICAL) assert_message_file(logger, log_file, "debug", False) diff --git a/tests/ogvjs/test_ogvjs.py b/tests/ogvjs/test_ogvjs.py index b6d666e2..d9ca8543 100644 --- a/tests/ogvjs/test_ogvjs.py +++ b/tests/ogvjs/test_ogvjs.py @@ -47,7 +47,9 @@ def prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): @pytest.mark.installed def test_installed_script(): kwargs = {"universal_newlines": True, "stdout": subprocess.PIPE} - script = subprocess.run(["fix_ogvjs_dist"], **kwargs) # nosec + script = subprocess.run( # noqa: PLW1510 + ["fix_ogvjs_dist"], **kwargs # noqa: S607 + ) # nosec assert script.returncode == 1 assert script.stdout.strip().startswith("Usage: ") diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index c2f7781f..31fac6f1 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -97,7 +97,7 @@ def test_get_tags(small_zim_file, real_zim_file): assert zim.tags == zim.get_tags() -def test_libkiwix_convertTags(): +def test_libkiwix_convertTags(): # noqa: N802 assert convertTags("") == [ "_ftindex:no", "_pictures:yes", diff --git a/tests/zim/test_libkiwix.py b/tests/zim/test_libkiwix.py index 02105348..fce72af5 100644 --- a/tests/zim/test_libkiwix.py +++ b/tests/zim/test_libkiwix.py @@ -6,7 +6,7 @@ import pytest from zimscraperlib.zim._libkiwix import getline -from zimscraperlib.zim._libkiwix import parseMimetypeCounter as parse +from zimscraperlib.zim._libkiwix import parseMimetypeCounter as parse # noqa: N813 empty = {} @@ -74,6 +74,6 @@ def test_getline(): ("text/html=50;;foo", {"text/html": 50}), ], ) -def test_counter_parsing(counterStr, counterMap): +def test_counter_parsing(counterStr, counterMap): # noqa: N803 # https://github.com/kiwix/libkiwix/blob/master/test/counterParsing.cpp assert parse(counterStr) == counterMap diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index a2f3be40..46d832a2 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -166,7 +166,7 @@ def test_add_item_for_delete_fail(tmp_path, png_image): shutil.copyfile(png_image, local_path) def remove_source(item): - print("##########", "remove_source") + print("##########", "remove_source") # noqa: T201 os.remove(item.filepath) with Creator(fpath, "welcome").config_dev_metadata() as creator: @@ -352,7 +352,7 @@ def test_urlprovider_nolength(tmp_path, png_image_url, png_image): # create and start an http server without Content-Length support server_fpath = tmp_path / "httpd.py" - port = random.randint(10000, 20000) + port = random.randint(10000, 20000) # noqa: S311 server_code = """ from http.server import BaseHTTPRequestHandler, HTTPServer @@ -385,7 +385,7 @@ def do_GET(self): with tempfile.TemporaryDirectory() as tmp_dir, Creator( fpath, "" ).config_dev_metadata() as creator: - tmp_dir = pathlib.Path(tmp_dir) + tmp_dir = pathlib.Path(tmp_dir) # noqa: PLW2901 creator.add_item( URLItem( url=f"http://localhost:{port}/hoho.png", @@ -442,7 +442,7 @@ def test_callback_and_remove(tmp_path, html_file): class Store: called = 0 - def cb(*args): + def cb(*args): # noqa: ARG001 Store.called += 1 # duplicate test file as we'll want to remove twice @@ -579,10 +579,10 @@ def test_config_metadata(tmp_path, png_image): ("Title", "X" * 30, True), ("Title", "X" * 31, False), ("Date", 4, False), - ("Date", datetime.datetime.now(), True), - ("Date", datetime.datetime(1969, 12, 31, 23, 59), True), + ("Date", datetime.datetime.now(), True), # noqa: DTZ005 + ("Date", datetime.datetime(1969, 12, 31, 23, 59), True), # noqa: DTZ001 ("Date", datetime.date(1969, 12, 31), True), - ("Date", datetime.date.today(), True), + ("Date", datetime.date.today(), True), # noqa: DTZ011 ("Date", "1969-12-31", True), ("Date", "1969-13-31", False), ("Date", "2023/02/29", False), From 2baf95fa2d048726f970711baf52d21ebd423866 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 16:36:44 +0100 Subject: [PATCH 066/335] Add all needed pyright:ignore statements - to be removed in the future, one by one --- src/zimscraperlib/download.py | 24 +++++--- src/zimscraperlib/html.py | 8 +-- src/zimscraperlib/i18n.py | 12 ++-- src/zimscraperlib/image/convertion.py | 10 +-- src/zimscraperlib/image/optimization.py | 44 +++++++------- src/zimscraperlib/image/probing.py | 8 ++- src/zimscraperlib/image/transformation.py | 9 ++- src/zimscraperlib/image/utils.py | 9 ++- src/zimscraperlib/logging.py | 22 ++++--- src/zimscraperlib/types.py | 6 +- src/zimscraperlib/uri.py | 32 +++++----- src/zimscraperlib/video/config.py | 2 +- src/zimscraperlib/zim/__init__.py | 2 +- src/zimscraperlib/zim/_libkiwix.py | 8 ++- src/zimscraperlib/zim/archive.py | 6 +- src/zimscraperlib/zim/creator.py | 28 +++++---- src/zimscraperlib/zim/filesystem.py | 22 +++---- src/zimscraperlib/zim/items.py | 2 +- src/zimscraperlib/zim/metadata.py | 7 ++- src/zimscraperlib/zim/providers.py | 10 +-- tests/download/test_download.py | 10 +-- tests/image/test_image.py | 74 +++++++++++++---------- tests/video/test_video.py | 4 +- tests/zim/test_libkiwix.py | 11 ++-- tests/zim/test_zim_creator.py | 41 ++++++++----- 25 files changed, 240 insertions(+), 171 deletions(-) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index ed3643f7..103cd5db 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -52,14 +52,16 @@ def download( Returns download result of future (wait=False)""" - future = self.executor.submit(self._run_youtube_dl, url, options) + future = self.executor.submit( + self._run_youtube_dl, url, options # pyright: ignore + ) if not wait: return future if not future.exception(): # return the result - return future.result() + return future.result() # pyright: ignore # raise the exception - raise future.exception() + raise future.exception() # pyright: ignore class YoutubeConfig(dict): @@ -137,8 +139,10 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None: ) -def _get_retry_adapter(max_retries: Optional[int] = 5) -> requests.adapters.BaseAdapter: - retries = requests.packages.urllib3.util.retry.Retry( +def _get_retry_adapter( + max_retries: Optional[int] = 5, +) -> requests.adapters.BaseAdapter: # pyright: ignore + retries = requests.packages.urllib3.util.retry.Retry( # pyright: ignore total=max_retries, # total number of retries connect=max_retries, # connection errors read=max_retries, # read errors @@ -155,7 +159,7 @@ def _get_retry_adapter(max_retries: Optional[int] = 5) -> requests.adapters.Base ], # force retry on the following codes ) - return requests.adapters.HTTPAdapter(max_retries=retries) + return requests.adapters.HTTPAdapter(max_retries=retries) # pyright: ignore def get_session(max_retries: Optional[int] = 5) -> requests.Session: @@ -175,7 +179,7 @@ def stream_file( max_retries: Optional[int] = 5, headers: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None, -) -> tuple[int, requests.structures.CaseInsensitiveDict]: +) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore """Stream data from a URL to either a BytesIO object or a file Arguments - fpath - Path of the file where data is sent @@ -211,7 +215,7 @@ def stream_file( for data in resp.iter_content(block_size): total_downloaded += len(data) - fp.write(data) + fp.write(data) # pyright: ignore # stop downloading/reading if we're just testing first block if only_first_block: @@ -220,7 +224,7 @@ def stream_file( logger.debug(f"Downloaded {total_downloaded} bytes from {url}") if fpath: - fp.close() + fp.close() # pyright: ignore else: - fp.seek(0) + fp.seek(0) # pyright: ignore return total_downloaded, resp.headers diff --git a/src/zimscraperlib/html.py b/src/zimscraperlib/html.py index 3ccc8711..c87dcf32 100644 --- a/src/zimscraperlib/html.py +++ b/src/zimscraperlib/html.py @@ -18,7 +18,7 @@ def find_title_in(content: Union[str, BinaryIO, TextIO], mime_type: str) -> str: if mime_type != ARTICLE_MIME: return "" try: - return BeautifulSoup(content, "lxml").find("title").text + return BeautifulSoup(content, "lxml").find("title").text # pyright: ignore except Exception: return "" @@ -44,15 +44,15 @@ def find_language_in(content: Union[str, BinaryIO, TextIO], mime_type: str) -> s for key in keylist: node = soup.find(nodename) if node: - if not node.has_attr(key): + if not node.has_attr(key): # pyright: ignore continue if ( nodename == "meta" - and not node.attrs.get("http-equiv", "").lower() + and not node.attrs.get("http-equiv", "").lower() # pyright: ignore == "content-language" ): continue - return node.attrs[key] + return node.attrs[key] # pyright: ignore return "" diff --git a/src/zimscraperlib/i18n.py b/src/zimscraperlib/i18n.py index 90991944..fdff9904 100644 --- a/src/zimscraperlib/i18n.py +++ b/src/zimscraperlib/i18n.py @@ -102,7 +102,9 @@ def find_language_names( lang_data = get_language_details(query, failsafe=True) or {} try: query_locale = babel.Locale.parse(query) - return query_locale.get_display_name(), query_locale.get_display_name("en") + return query_locale.get_display_name(), query_locale.get_display_name( + "en" + ) # pyright: ignore except (babel.UnknownLocaleError, TypeError, ValueError, AttributeError): pass @@ -110,7 +112,9 @@ def find_language_names( for iso_level in [f"iso-639-{lang_}" for lang_ in reversed(ISO_LEVELS)]: try: query_locale = babel.Locale.parse(lang_data.get(iso_level)) - return query_locale.get_display_name(), query_locale.get_display_name("en") + return query_locale.get_display_name(), query_locale.get_display_name( + "en" + ) # pyright: ignore except (babel.UnknownLocaleError, TypeError, ValueError, AttributeError): pass default = lang_data.get("english", query) @@ -166,10 +170,10 @@ def get_language_details( lang_data, macro_data = get_iso_lang_data(adjusted_query) except NotFound as exc: if failsafe: - return None + return None # pyright: ignore raise exc - iso_data = update_with_macro(lang_data, macro_data) + iso_data = update_with_macro(lang_data, macro_data) # pyright: ignore native_name, english_name = find_language_names(native_query, iso_data) iso_data.update( { diff --git a/src/zimscraperlib/image/convertion.py b/src/zimscraperlib/image/convertion.py index e1460d52..31674236 100644 --- a/src/zimscraperlib/image/convertion.py +++ b/src/zimscraperlib/image/convertion.py @@ -25,10 +25,12 @@ def convert_image( to RGB. ex: RGB, ARGB, CMYK (and other PIL colorspaces)""" colorspace = params.get("colorspace") # requested colorspace - fmt = params.pop("fmt").upper() if "fmt" in params else None # requested format + fmt = ( + params.pop("fmt").upper() if "fmt" in params else None # pyright: ignore + ) # requested format if not fmt: fmt = format_for(dst) - with PIL.Image.open(src) as image: + with PIL.Image.open(src) as image: # pyright: ignore if image.mode == "RGBA" and fmt in ALPHA_NOT_SUPPORTED or colorspace: image = image.convert(colorspace or "RGB") # noqa: PLW2901 save_image(image, dst, fmt, **params) @@ -39,13 +41,13 @@ def create_favicon(src: pathlib.Path, dst: pathlib.Path) -> None: if dst.suffix != ".ico": raise ValueError("favicon extension must be ICO") - img = PIL.Image.open(src) + img = PIL.Image.open(src) # pyright: ignore w, h = img.size # resize image to square first if w != h: size = min([w, h]) resized = dst.parent.joinpath(f"{src.stem}.tmp.{src.suffix}") resize_image(src, size, size, resized, "contain") - img = PIL.Image.open(resized) + img = PIL.Image.open(resized) # pyright: ignore # now convert to ICO save_image(img, dst, "ICO") diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 935a5be2..431c7c79 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -76,25 +76,25 @@ def optimize_png( if remove_transparency is True (tuple containing RGB values) values: (255, 255, 255) | (221, 121, 108) | (XX, YY, ZZ)""" - ensure_matches(src, "PNG") + ensure_matches(src, "PNG") # pyright: ignore img = Image.open(src) if remove_transparency: - img = remove_alpha(img, background_color) + img = remove_alpha(img, background_color) # pyright: ignore if reduce_colors: - img, _, _ = do_reduce_colors(img, max_colors) + img, _, _ = do_reduce_colors(img, max_colors) # pyright: ignore if not fast_mode and img.mode == "P": img, _ = rebuild_palette(img) if dst is None: - dst = io.BytesIO() - img.save(dst, optimize=True, format="PNG") + dst = io.BytesIO() # pyright: ignore + img.save(dst, optimize=True, format="PNG") # pyright: ignore if isinstance(dst, io.BytesIO): dst.seek(0) - return dst + return dst # pyright: ignore def optimize_jpeg( @@ -114,7 +114,7 @@ def optimize_jpeg( get dynamic quality value to ensure better compression values: True | False""" - ensure_matches(src, "JPEG") + ensure_matches(src, "JPEG") # pyright: ignore img = Image.open(src) orig_size = ( @@ -138,10 +138,10 @@ def optimize_jpeg( quality_setting, _ = jpeg_dynamic_quality(img) if dst is None: - dst = io.BytesIO() + dst = io.BytesIO() # pyright: ignore img.save( - dst, + dst, # pyright: ignore quality=quality_setting, optimize=True, progressive=use_progressive_jpg, @@ -157,12 +157,14 @@ def optimize_jpeg( str(src.resolve()) if isinstance(src, pathlib.Path) else src.getvalue() ), image=( - str(dst.resolve()) if isinstance(dst, pathlib.Path) else dst.getvalue() + str(dst.resolve()) + if isinstance(dst, pathlib.Path) + else dst.getvalue() # pyright: ignore ), new_file=dst, ) - return dst + return dst # pyright: ignore def optimize_webp( @@ -186,7 +188,7 @@ def optimize_webp( refer to the link for more details https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp""" - ensure_matches(src, "WEBP") + ensure_matches(src, "WEBP") # pyright: ignore params = { "lossless": lossless, "quality": quality, @@ -195,17 +197,17 @@ def optimize_webp( webp_image = Image.open(src) if dst is None: - dst = io.BytesIO() - webp_image.save(dst, format="WEBP", **params) - dst.seek(0) + dst = io.BytesIO() # pyright: ignore + webp_image.save(dst, format="WEBP", **params) # pyright: ignore + dst.seek(0) # pyright: ignore else: try: - save_image(webp_image, dst, fmt="WEBP", **params) + save_image(webp_image, dst, fmt="WEBP", **params) # pyright: ignore except Exception as exc: - if src.resolve() != dst.resolve() and dst.exists(): + if src.resolve() != dst.resolve() and dst.exists(): # pyright: ignore dst.unlink() # pragma: nocover raise exc - return dst + return dst # pyright: ignore def optimize_gif( @@ -267,7 +269,7 @@ def optimize_image( delete_src: Optional[bool] = False, # noqa: FBT002 convert: Optional[Union[bool, str]] = False, # noqa: FBT002 **options, -) -> bool: +) -> bool: # pyright: ignore """Optimize image, automatically selecting correct optimizer delete_src: whether to remove src file upon success (boolean) @@ -281,12 +283,12 @@ def optimize_image( # if requested, convert src to requested format into dst path if convert and src_format != dst_format: src_format = dst_format = convert if isinstance(convert, str) else dst_format - convert_image(src, dst, fmt=src_format) + convert_image(src, dst, fmt=src_format) # pyright: ignore src_img = pathlib.Path(dst) else: src_img = pathlib.Path(src) - { + { # pyright: ignore "JPEG": optimize_jpeg, "PNG": optimize_png, "GIF": optimize_gif, diff --git a/src/zimscraperlib/image/probing.py b/src/zimscraperlib/image/probing.py index d17b89b1..ac3badca 100644 --- a/src/zimscraperlib/image/probing.py +++ b/src/zimscraperlib/image/probing.py @@ -47,7 +47,7 @@ def solarize(r: int, g: int, b: int) -> Tuple[int, int, int]: def is_hex_color(text: str) -> bool: """whether supplied text is a valid hex-formated color code""" - return re.search(r"^#(?:[0-9a-fA-F]{3}){1,2}$", text) + return re.search(r"^#(?:[0-9a-fA-F]{3}){1,2}$", text) # pyright: ignore def format_for( @@ -57,13 +57,15 @@ def format_for( """Pillow format of a given filename, either Pillow-detected or from suffix""" if not from_suffix: with PIL.Image.open(src) as img: - return img.format + return img.format # pyright: ignore from PIL.Image import EXTENSION as ext_fmt_map # noqa: N811 from PIL.Image import init as init_pil init_pil() - return ext_fmt_map[src.suffix] # might raise KeyError on unknown extension + return ext_fmt_map[ + src.suffix # pyright: ignore + ] # might raise KeyError on unknown extension def is_valid_image( diff --git a/src/zimscraperlib/image/transformation.py b/src/zimscraperlib/image/transformation.py index 93d461c9..4db56cc6 100644 --- a/src/zimscraperlib/image/transformation.py +++ b/src/zimscraperlib/image/transformation.py @@ -25,7 +25,7 @@ def resize_image( methods: width, height, cover, thumbnail allow upscaling: upscale image first, preserving aspect ratio if required""" - with PIL.Image.open(src) as image: + with PIL.Image.open(src) as image: # pyright: ignore # preserve image format as resize() does not transmit it into new object image_format = image.format image_mode = image.mode @@ -58,4 +58,9 @@ def resize_image( if dst is None and isinstance(src, io.BytesIO): src.seek(0) - save_image(resized, dst if dst is not None else src, image_format, **params) + save_image( + resized, + dst if dst is not None else src, # pyright: ignore + image_format, + **params, + ) diff --git a/src/zimscraperlib/image/utils.py b/src/zimscraperlib/image/utils.py index 240ae947..2568492d 100644 --- a/src/zimscraperlib/image/utils.py +++ b/src/zimscraperlib/image/utils.py @@ -8,9 +8,12 @@ def save_image( - src: Image, dst: pathlib.Path, fmt: Optional[str] = None, **params: Optional[dict] + src: Image, # pyright: ignore + dst: pathlib.Path, + fmt: Optional[str] = None, + **params: Optional[dict], ) -> None: """PIL.Image.save() wrapper setting default parameters""" - args = {"JPEG": {"quality": 100}, "PNG": {}}.get(fmt, {}) + args = {"JPEG": {"quality": 100}, "PNG": {}}.get(fmt, {}) # pyright: ignore args.update(params or {}) - src.save(dst, fmt, **args) + src.save(dst, fmt, **args) # pyright: ignore diff --git a/src/zimscraperlib/logging.py b/src/zimscraperlib/logging.py index 8ef0e54c..4a0f1881 100644 --- a/src/zimscraperlib/logging.py +++ b/src/zimscraperlib/logging.py @@ -17,9 +17,9 @@ def getLogger( # noqa: N802 name: str, level: Optional[int] = logging.INFO, - console: Optional[io.TextIOBase] = sys.stdout, + console: Optional[io.TextIOBase] = sys.stdout, # pyright: ignore log_format: Optional[str] = DEFAULT_FORMAT, - file: Optional[pathlib.Path] = False, # noqa: FBT002 + file: Optional[pathlib.Path] = False, # noqa: FBT002 # pyright: ignore file_level: Optional[int] = None, file_format: Optional[str] = None, file_max: Optional[int] = 2**20, @@ -43,11 +43,11 @@ def getLogger( # noqa: N802 additional_deps = [] # align zimscraperlib logging level to that of scraper - logging.Logger(NAME).setLevel(level) + logging.Logger(NAME).setLevel(level) # pyright: ignore # set arbitrary level for some known verbose dependencies # prevents them from polluting logs - for logger_name in set(VERBOSE_DEPENDENCIES + additional_deps): + for logger_name in set(VERBOSE_DEPENDENCIES + additional_deps): # pyright: ignore logging.getLogger(logger_name).setLevel(logging.WARNING) logger = logging.Logger(name) @@ -57,15 +57,17 @@ def getLogger( # noqa: N802 if console: console_handler = logging.StreamHandler(console) console_handler.setFormatter(logging.Formatter(log_format)) - console_handler.setLevel(level) + console_handler.setLevel(level) # pyright: ignore logger.addHandler(console_handler) if file: - file_handler = RotatingFileHandler( - file, maxBytes=file_max, backupCount=file_nb_backup + file_handler = RotatingFileHandler( # pyright: ignore + file, + maxBytes=file_max, # pyright: ignore + backupCount=file_nb_backup, # pyright: ignore ) file_handler.setFormatter(logging.Formatter(file_format or log_format)) - file_handler.setLevel(file_level or level) + file_handler.setLevel(file_level or level) # pyright: ignore logger.addHandler(file_handler) return logger @@ -73,7 +75,7 @@ def getLogger( # noqa: N802 def nicer_args_join(args: Iterable) -> str: """slightly better concateated list of subprocess args for display""" - nargs = args[0:1] - for arg in args[1:]: + nargs = args[0:1] # pyright: ignore + for arg in args[1:]: # pyright: ignore nargs.append(arg if arg.startswith("-") else f'"{arg}"') return " ".join(nargs) diff --git a/src/zimscraperlib/types.py b/src/zimscraperlib/types.py index b3993382..de142111 100644 --- a/src/zimscraperlib/types.py +++ b/src/zimscraperlib/types.py @@ -50,9 +50,11 @@ def get_mime_for_name( filename = pathlib.Path(filename) if not filename.suffix: return no_ext_to - return mimetypes.guess_type(f"{filename.stem}{filename.suffix}")[0] or fallback + return ( + mimetypes.guess_type(f"{filename.stem}{filename.suffix}")[0] or fallback + ) # pyright: ignore except Exception: - return fallback + return fallback # pyright: ignore def init_types(): diff --git a/src/zimscraperlib/uri.py b/src/zimscraperlib/uri.py index 7fa43b96..96ce4708 100644 --- a/src/zimscraperlib/uri.py +++ b/src/zimscraperlib/uri.py @@ -9,31 +9,31 @@ def rebuild_uri( uri: urllib.parse.ParseResult, - scheme: str = None, # noqa: RUF013 - username: str = None, # noqa: RUF013 - password: str = None, # noqa: RUF013 - hostname: str = None, # noqa: RUF013 - port: Union[str, int] = None, # noqa: RUF013 - path: str = None, # noqa: RUF013 - params: str = None, # noqa: RUF013 - query: str = None, # noqa: RUF013 - fragment: str = None, # noqa: RUF013 + scheme: str = None, # noqa: RUF013 # pyright: ignore + username: str = None, # noqa: RUF013 # pyright: ignore + password: str = None, # noqa: RUF013 # pyright: ignore + hostname: str = None, # noqa: RUF013 # pyright: ignore + port: Union[str, int] = None, # noqa: RUF013 # pyright: ignore + path: str = None, # noqa: RUF013 # pyright: ignore + params: str = None, # noqa: RUF013 # pyright: ignore + query: str = None, # noqa: RUF013 # pyright: ignore + fragment: str = None, # noqa: RUF013 # pyright: ignore failsafe: bool = False, # noqa: FBT001, FBT002 ) -> urllib.parse.ParseResult: """new ParseResult named tuple from uri with requested part updated""" try: - username = first(username, uri.username, "") - password = first(password, uri.password, "") - hostname = first(hostname, uri.hostname, "") - port = first(port, uri.port, "") + username = first(username, uri.username, "") # pyright: ignore + password = first(password, uri.password, "") # pyright: ignore + hostname = first(hostname, uri.hostname, "") # pyright: ignore + port = first(port, uri.port, "") # pyright: ignore netloc = ( f"{username}{':' if password else ''}{password}" f"{'@' if username or password else ''}{hostname}" f"{':' if port else ''}{port}" ) - return urllib.parse.urlparse( - urllib.parse.urlunparse( - ( + return urllib.parse.urlparse( # pyright: ignore + urllib.parse.urlunparse( # pyright: ignore + ( # pyright: ignore first(scheme, uri.scheme), netloc, first(path, uri.path), diff --git a/src/zimscraperlib/video/config.py b/src/zimscraperlib/video/config.py index 6fb4b6a0..f2f8da9f 100644 --- a/src/zimscraperlib/video/config.py +++ b/src/zimscraperlib/video/config.py @@ -110,7 +110,7 @@ def buffersize(self, value): @property def video_scale(self): # remove "scale='" and "'" and return the value in between - return self.get("-vf")[7:-1] if self.get("-vf") else None + return self.get("-vf")[7:-1] if self.get("-vf") else None # pyright: ignore @video_scale.setter def video_scale(self, value): diff --git a/src/zimscraperlib/zim/__init__.py b/src/zimscraperlib/zim/__init__.py index 5f3f44bc..df282f3a 100644 --- a/src/zimscraperlib/zim/__init__.py +++ b/src/zimscraperlib/zim/__init__.py @@ -9,7 +9,7 @@ zim.items: item to add to creator zim.archive: read ZIM files, accessing or searching its content""" -from libzim.writer import Blob +from libzim.writer import Blob # pyright: ignore from zimscraperlib.zim.archive import Archive from zimscraperlib.zim.creator import Creator diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index 40f0d823..af1eec5c 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -17,7 +17,9 @@ from typing import Dict, List, Optional, Tuple MimetypeAndCounter = namedtuple("MimetypeAndCounter", ["mimetype", "value"]) -CounterMap = Dict[type(MimetypeAndCounter.mimetype), type(MimetypeAndCounter.value)] +CounterMap = Dict[ + type(MimetypeAndCounter.mimetype), type(MimetypeAndCounter.value) # pyright: ignore +] def getline(src: io.StringIO, delim: Optional[bool] = None) -> Tuple[bool, str]: @@ -45,10 +47,10 @@ def readFullMimetypeAndCounterString( # noqa: N802 Returns whether the source is EOF and the extracted string (or empty one)""" params = "" - eof, mtcStr = getline(src, ";") # noqa: N806 + eof, mtcStr = getline(src, ";") # noqa: N806 # pyright: ignore if mtcStr.find("=") == -1: while params.count("=") != 2: # noqa: PLR2004 - eof, params = getline(src, ";") + eof, params = getline(src, ";") # pyright: ignore if params.count("=") == 2: # noqa: PLR2004 mtcStr += ";" + params # noqa: N806 if eof: diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index a9b46bd8..62768589 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -11,9 +11,9 @@ from typing import Dict, Iterable, List, Optional -import libzim.reader -import libzim.search # Query, Searcher -import libzim.suggestion # SuggestionSearcher +import libzim.reader # pyright: ignore +import libzim.search # Query, Searcher # pyright: ignore +import libzim.suggestion # SuggestionSearcher # pyright: ignore from zimscraperlib.zim._libkiwix import convertTags, parseMimetypeCounter from zimscraperlib.zim.items import Item diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 16e39a8a..44d2d1e2 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -23,7 +23,7 @@ import weakref from typing import Any, Callable, Iterable, Optional, Tuple, Union -import libzim.writer +import libzim.writer # pyright: ignore from zimscraperlib.constants import ( DEFAULT_DEV_ZIM_METADATA, @@ -68,7 +68,9 @@ def mimetype_for( """mimetype as provided or guessed from fpath, path or content""" if not mimetype: mimetype = ( - get_file_mimetype(fpath) if fpath else get_content_mimetype(content[:2048]) + get_file_mimetype(fpath) + if fpath + else get_content_mimetype(content[:2048]) # pyright: ignore ) # try to guess more-defined mime if it's text if ( @@ -171,16 +173,16 @@ def validate_metadata( See https://wiki.openzim.org/wiki/Metadata""" validate_required_values(name, value) - validate_standard_str_types(name, value) + validate_standard_str_types(name, value) # pyright: ignore - validate_title(name, value) - validate_date(name, value) - validate_language(name, value) - validate_counter(name, value) - validate_description(name, value) - validate_longdescription(name, value) - validate_tags(name, value) - validate_illustrations(name, value) + validate_title(name, value) # pyright: ignore + validate_date(name, value) # pyright: ignore + validate_language(name, value) # pyright: ignore + validate_counter(name, value) # pyright: ignore + validate_description(name, value) # pyright: ignore + validate_longdescription(name, value) # pyright: ignore + validate_tags(name, value) # pyright: ignore + validate_illustrations(name, value) # pyright: ignore def add_metadata( self, @@ -251,7 +253,9 @@ def add_item_for( should_compress: Optional[bool] = None, delete_fpath: Optional[bool] = False, # noqa: FBT002 duplicate_ok: Optional[bool] = None, - callback: Optional[Union[callable, Tuple[callable, Any]]] = None, + callback: Optional[ + Union[callable, Tuple[callable, Any]] # pyright: ignore + ] = None, ): """Add a File or content at a specified path and get its path diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 9b0f3d38..e0f3d6a2 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -46,8 +46,8 @@ def __init__( self, root: pathlib.Path, filepath: pathlib.Path, - ): - super().__init__(root=root, filepath=filepath) + ): # pyright: ignore + super().__init__(root=root, filepath=filepath) # pyright: ignore # first look inside the file's magic headers self.mimetype = get_file_mimetype(self.filepath) # most web-specific files are plain text. In this case, use extension @@ -102,7 +102,7 @@ def add_redirects_to_zim( for line in fh.readlines(): namespace, path, title, target_url = re.match( r"^(.)\t(.+)\t(.*)\t(.+)$", line - ).groups() + ).groups() # pyright: ignore if namespace.strip(): path = f"{namespace.strip()}/{path}" zim_file.add_redirect(path, target_url, title) @@ -116,18 +116,18 @@ def make_zim_file( illustration: str, title: str, description: str, - date: datetime.date = None, # noqa: RUF013 + date: datetime.date = None, # noqa: RUF013 # pyright: ignore language: str = "eng", creator: str = "-", publisher="-", - tags: Sequence[str] = None, # noqa: RUF013 - source: str = None, # noqa: RUF013 - flavour: str = None, # noqa: RUF013 - scraper: str = None, # noqa: RUF013 - long_description: str = None, # noqa: RUF013 + tags: Sequence[str] = None, # noqa: RUF013 # pyright: ignore + source: str = None, # noqa: RUF013 # pyright: ignore + flavour: str = None, # noqa: RUF013 # pyright: ignore + scraper: str = None, # noqa: RUF013 # pyright: ignore + long_description: str = None, # noqa: RUF013 # pyright: ignore without_fulltext_index: bool = False, # noqa: FBT001, FBT002, ARG001 - redirects: Sequence[Tuple[str, str, str]] = None, # noqa: RUF013 - redirects_file: pathlib.Path = None, # noqa: RUF013 + redirects: Sequence[Tuple[str, str, str]] = None, # noqa: RUF013 # pyright: ignore + redirects_file: pathlib.Path = None, # noqa: RUF013 # pyright: ignore rewrite_links: bool = True, # noqa: FBT001, FBT002, ARG001 workaround_nocancel: bool = True, # noqa: FBT001, FBT002 ): diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index b83983e0..6ea51a4a 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -11,7 +11,7 @@ import urllib.parse from typing import Dict, Union -import libzim.writer +import libzim.writer # pyright: ignore from zimscraperlib.download import stream_file from zimscraperlib.zim.providers import ( diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index 1a6bfe8c..7962bd28 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -53,7 +53,12 @@ def validate_date(name: str, value: Union[datetime.datetime, datetime.date, str] elif isinstance(value, str): match = re.match(r"(?P\d{4})-(?P\d{2})-(?P\d{2})", value) try: - datetime.date(**{k: int(v) for k, v in match.groupdict().items()}) + datetime.date( + **{ + k: int(v) + for k, v in match.groupdict().items() # pyright: ignore + } + ) except Exception as exc: raise ValueError(f"Invalid {name} format: {exc}") # noqa: B904 diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index 8140cc09..913403fa 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -13,7 +13,7 @@ import pathlib from typing import Optional, Union -import libzim.writer +import libzim.writer # pyright: ignore import requests from zimscraperlib.download import _get_retry_adapter, stream_file @@ -58,10 +58,12 @@ def __init__( self.fileobj.seek(0, io.SEEK_SET) def get_size(self) -> int: - return self.size + return self.size # pyright: ignore def gen_blob(self) -> libzim.writer.Blob: - yield libzim.writer.Blob(self.fileobj.getvalue()) # pragma: nocover + yield libzim.writer.Blob( + self.fileobj.getvalue() # pyright: ignore + ) # pragma: nocover class URLProvider(libzim.writer.ContentProvider): @@ -91,7 +93,7 @@ def get_size_of(url) -> Union[int, None]: return None def get_size(self) -> int: - return self.size + return self.size # pyright: ignore def gen_blob(self) -> libzim.writer.Blob: # pragma: nocover for chunk in self.resp.iter_content(10 * 1024): diff --git a/tests/download/test_download.py b/tests/download/test_download.py index c49b65de..c8a0f6dc 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -26,7 +26,9 @@ def assert_downloaded_file(url, file): def assert_headers(returned_headers): - assert isinstance(returned_headers, requests.structures.CaseInsensitiveDict) + assert isinstance( + returned_headers, requests.structures.CaseInsensitiveDict # pyright: ignore + ) assert returned_headers["Content-Type"] == "image/x-icon" @@ -155,12 +157,12 @@ def test_youtube_download_nowait(tmp_path): BestMp4.get_options(target_dir=tmp_path), wait=False, ) - assert future.running() + assert future.running() # pyright: ignore assert not yt_downloader.executor._shutdown done, not_done = concurrent.futures.wait( - [future], return_when=concurrent.futures.ALL_COMPLETED + [future], return_when=concurrent.futures.ALL_COMPLETED # pyright: ignore ) - assert future.exception() is None + assert future.exception() is None # pyright: ignore assert len(done) == 1 assert len(not_done) == 0 diff --git a/tests/image/test_image.py b/tests/image/test_image.py index e8327d96..0abaf916 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -93,7 +93,7 @@ def test_is_hex_color(value, valid): def test_colors_noimage(): with pytest.raises(FileNotFoundError): - get_colors("nofile.here") + get_colors("nofile.here") # pyright: ignore def test_colors_png_nopalette(png_image): @@ -118,11 +118,13 @@ def test_colors_jpg_palette(jpg_image): ) def test_save_image(png_image, jpg_image, tmp_path, fmt, params): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) - img = Image.open(src) + img = Image.open(src) # pyright: ignore if params: - save_image(img, dst, "JPEG" if fmt == "jpg" else fmt, **params) + save_image( + img, dst, "JPEG" if fmt == "jpg" else fmt, **params # pyright: ignore + ) else: - save_image(img, dst, "JPEG" if fmt == "jpg" else fmt) + save_image(img, dst, "JPEG" if fmt == "jpg" else fmt) # pyright: ignore assert pathlib.Path(dst).exists() @@ -134,7 +136,7 @@ def test_resize_thumbnail(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 100, 50 - resize_image(src, width, height, dst=dst, method="thumbnail") + resize_image(src, width, height, dst=dst, method="thumbnail") # pyright: ignore tw, th = get_image_size(dst) assert tw <= width assert th <= height @@ -148,8 +150,8 @@ def test_resize_bytestream(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) # copy image content into a bytes stream - img = io.BytesIO() - with open(src, "rb") as srch: + img = io.BytesIO() # pyright: ignore + with open(src, "rb") as srch: # pyright: ignore img.write(srch.read()) # resize in place (no dst) @@ -168,7 +170,7 @@ def test_resize_width(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 100, 50 - resize_image(src, width, height, dst=dst, method="width") + resize_image(src, width, height, dst=dst, method="width") # pyright: ignore tw, _ = get_image_size(dst) assert tw == width @@ -181,7 +183,7 @@ def test_resize_height(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 100, 50 - resize_image(src, width, height, dst=dst, method="height") + resize_image(src, width, height, dst=dst, method="height") # pyright: ignore _, th = get_image_size(dst) assert th == height @@ -194,7 +196,7 @@ def test_resize_crop(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 5, 50 - resize_image(src, width, height, dst=dst, method="crop") + resize_image(src, width, height, dst=dst, method="crop") # pyright: ignore tw, th = get_image_size(dst) assert tw == width assert th == height @@ -208,7 +210,7 @@ def test_resize_cover(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 5, 50 - resize_image(src, width, height, dst=dst, method="cover") + resize_image(src, width, height, dst=dst, method="cover") # pyright: ignore tw, th = get_image_size(dst) assert tw == width assert th == height @@ -222,7 +224,7 @@ def test_resize_contain(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 5, 50 - resize_image(src, width, height, dst=dst, method="contain") + resize_image(src, width, height, dst=dst, method="contain") # pyright: ignore tw, th = get_image_size(dst) assert tw <= width assert th <= height @@ -236,7 +238,7 @@ def test_resize_upscale(png_image, jpg_image, tmp_path, fmt): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image, jpg_image=jpg_image) width, height = 500, 1000 - resize_image(src, width, height, dst=dst, method="cover") + resize_image(src, width, height, dst=dst, method="cover") # pyright: ignore tw, th = get_image_size(dst) assert tw == width assert th == height @@ -251,7 +253,14 @@ def test_resize_small_image_error(png_image, jpg_image, tmp_path, fmt): width, height = 500, 1000 with pytest.raises(ImageSizeError): - resize_image(src, width, height, dst=dst, method="cover", allow_upscaling=False) + resize_image( + src, # pyright: ignore + width, + height, + dst=dst, + method="cover", + allow_upscaling=False, + ) @pytest.mark.parametrize( @@ -263,7 +272,7 @@ def test_change_image_format( ): src, _ = get_src_dst(tmp_path, src_fmt, png_image=png_image, jpg_image=jpg_image) dst = tmp_path / f"out.{dst_fmt.lower()}" - convert_image(src, dst, fmt=dst_fmt, colorspace=colorspace) + convert_image(src, dst, fmt=dst_fmt, colorspace=colorspace) # pyright: ignore dst_image = Image.open(dst) if colorspace: assert dst_image.mode == colorspace @@ -292,7 +301,7 @@ def test_change_image_format_defaults(png_image, jpg_image, tmp_path): # noqa: def test_create_favicon(png_image2, jpg_image, tmp_path, fmt, exp_size): src, dst = get_src_dst(tmp_path, fmt, png_image=png_image2, jpg_image=jpg_image) dst = dst.parent.joinpath("favicon.ico") - create_favicon(src, dst) + create_favicon(src, dst) # pyright: ignore im = Image.open(dst) assert im.format == "ICO" @@ -308,7 +317,7 @@ def test_create_favicon_square(square_png_image, square_jpg_image, tmp_path, fmt tmp_path, fmt, png_image=square_png_image, jpg_image=square_jpg_image ) dst = dst.parent.joinpath("favicon.ico") - create_favicon(src, dst) + create_favicon(src, dst) # pyright: ignore im = Image.open(dst) assert im.format == "ICO" @@ -324,7 +333,7 @@ def test_wrong_extension(square_png_image, square_jpg_image, tmp_path, fmt): tmp_path, fmt, png_image=square_png_image, jpg_image=square_jpg_image ) with pytest.raises(ValueError): - create_favicon(src, dst) + create_favicon(src, dst) # pyright: ignore @pytest.mark.parametrize( @@ -342,8 +351,8 @@ def test_optimize_image_default( gif_image=gif_image, webp_image=webp_image, ) - optimize_image(src, dst, delete_src=False) - assert os.path.getsize(dst) < os.path.getsize(src) + optimize_image(src, dst, delete_src=False) # pyright: ignore + assert os.path.getsize(dst) < os.path.getsize(src) # pyright: ignore def test_optimize_image_del_src(png_image, tmp_path): @@ -459,15 +468,17 @@ def test_preset( gif_image=gif_image, webp_image=webp_image, ) - optimize_image(src, dst, delete_src=False, **preset.options) - assert os.path.getsize(dst) < os.path.getsize(src) + optimize_image(src, dst, delete_src=False, **preset.options) # pyright: ignore + assert os.path.getsize(dst) < os.path.getsize(src) # pyright: ignore if fmt in ["jpg", "webp", "png"]: - image_bytes = "" - with open(src, "rb") as fl: + image_bytes = "" # pyright: ignore + with open(src, "rb") as fl: # pyright: ignore image_bytes = fl.read() byte_stream = io.BytesIO(image_bytes) - dst_bytes = get_optimization_method(fmt)(src=byte_stream, **preset.options) + dst_bytes = get_optimization_method(fmt)( + src=byte_stream, **preset.options + ) # pyright: ignore assert dst_bytes.getbuffer().nbytes < byte_stream.getbuffer().nbytes @@ -496,8 +507,9 @@ def test_jpeg_exif_preserve(jpg_exif_image, tmp_path): with open(jpg_exif_image, "rb") as fl: src_bytes = fl.read() optimized_img = optimize_jpeg(src=io.BytesIO(src_bytes)) - assert piexif.load(optimized_img.getvalue())["Exif"] and ( - piexif.load(src_bytes)["Exif"] == piexif.load(optimized_img.getvalue())["Exif"] + assert piexif.load(optimized_img.getvalue())["Exif"] and ( # pyright: ignore + piexif.load(src_bytes)["Exif"] + == piexif.load(optimized_img.getvalue())["Exif"] # pyright: ignore ) @@ -528,7 +540,7 @@ def test_format_for( gif_image=gif_image, webp_image=webp_image, ) - assert format_for(src) == expected + assert format_for(src) == expected # pyright: ignore def test_optimize_webp_gif_failure(tmp_path, webp_image, gif_image): @@ -536,13 +548,13 @@ def test_optimize_webp_gif_failure(tmp_path, webp_image, gif_image): # webp with pytest.raises(Exception): # noqa: B017 - optimize_webp(webp_image, dst, lossless="bad") + optimize_webp(webp_image, dst, lossless="bad") # pyright: ignore assert not dst.exists() # gif dst.touch() # fake temp file created during optim (actually fails before) with pytest.raises(Exception): # noqa: B017 - optimize_gif(gif_image, dst, optimize_level="bad") + optimize_gif(gif_image, dst, optimize_level="bad") # pyright: ignore assert not dst.exists() @@ -560,7 +572,7 @@ def test_is_valid_image(png_image, png_image2, jpg_image, font): assert is_valid_image(png_image, "PNG", (48, 48)) assert not is_valid_image(png_image2, "PNG", (48, 48)) assert not is_valid_image(b"", "PNG") - assert not is_valid_image(34, "PNG") + assert not is_valid_image(34, "PNG") # pyright: ignore assert not is_valid_image(font, "PNG") with open(png_image, "rb") as fh: assert is_valid_image(fh.read(), "PNG", (48, 48)) diff --git a/tests/video/test_video.py b/tests/video/test_video.py index 8591dc73..31035273 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -84,7 +84,7 @@ def test_config_build_from(): assert idx != -1 assert args[idx + 1] == str(getattr(config, attr)) video_scale = config.video_scale - qmin, qmax = config.quantizer_scale_range + qmin, qmax = config.quantizer_scale_range # pyright: ignore assert args.index("-qmin") != -1 and args[args.index("-qmin") + 1] == str(qmin) assert args.index("-qmax") != -1 and args[args.index("-qmax") + 1] == str(qmax) assert ( @@ -414,7 +414,7 @@ def test_reencode_return_ffmpeg_output( with_process=return_output, ) if return_output: - success, process = ret + success, process = ret # pyright: ignore assert success assert len(process.stdout) > 0 else: diff --git a/tests/zim/test_libkiwix.py b/tests/zim/test_libkiwix.py index fce72af5..98ea8781 100644 --- a/tests/zim/test_libkiwix.py +++ b/tests/zim/test_libkiwix.py @@ -19,10 +19,13 @@ def test_geline_nodelim(): def test_getline(): ins = io.StringIO("text/javascript=8;text/html=3;application/warc-headers=28364;") - assert getline(ins, ";") == (False, "text/javascript=8") - assert getline(ins, ";") == (False, "text/html=3") - assert getline(ins, ";") == (False, "application/warc-headers=28364") - assert getline(ins, ";") == (True, "") + assert getline(ins, ";") == (False, "text/javascript=8") # pyright: ignore + assert getline(ins, ";") == (False, "text/html=3") # pyright: ignore + assert getline(ins, ";") == ( # pyright: ignore + False, + "application/warc-headers=28364", + ) + assert getline(ins, ";") == (True, "") # pyright: ignore @pytest.mark.parametrize( diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 46d832a2..6b96adc4 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -14,7 +14,7 @@ import time import pytest -from libzim.writer import Compression +from libzim.writer import Compression # pyright: ignore from zimscraperlib.constants import ( DEFAULT_DEV_ZIM_METADATA, @@ -51,7 +51,7 @@ def test_zim_creator(tmp_path, png_image, html_file, html_str): with open(png_image, "rb") as fh: png_data = fh.read() with Creator(fpath, main_path).config_dev_metadata( - Tags=tags, Illustration_48x48_at_1=png_data + Tags=tags, Illustration_48x48_at_1=png_data # pyright: ignore ) as creator: # verbatim HTML from string creator.add_item_for("welcome", "wel", content=html_str, is_front=True) @@ -119,7 +119,7 @@ def test_noindexlanguage(tmp_path): creator = Creator(fpath, "welcome").config_dev_metadata(Language="bam") creator.config_indexing(False) with creator as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) + creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore creator.add_item_for("index", "Index", content="-", mimetype="text/html") reader = Archive(fpath) @@ -171,7 +171,11 @@ def remove_source(item): with Creator(fpath, "welcome").config_dev_metadata() as creator: creator.add_item( - StaticItem(filepath=local_path, path="index", callback=remove_source), + StaticItem( + filepath=local_path, # pyright: ignore + path="index", # pyright: ignore + callback=remove_source, # pyright: ignore + ), callback=(delete_callback, local_path), ) assert not local_path.exists() @@ -185,18 +189,18 @@ def test_compression(tmp_path): with Creator( tmp_path / "test.zim", "welcome", compression="zstd" ).config_dev_metadata() as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) + creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore with Creator( - fpath, "welcome", compression=Compression.zstd + fpath, "welcome", compression=Compression.zstd # pyright: ignore ).config_dev_metadata() as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) + creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore def test_double_finish(tmp_path): fpath = tmp_path / "test.zim" with Creator(fpath, "welcome").config_dev_metadata() as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) + creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore # ensure we can finish an already finished creator creator.finish() @@ -216,7 +220,11 @@ def test_sourcefile_removal(tmp_path, html_file): # copy html to folder src_path = pathlib.Path(tmpdir.name, "source.html") shutil.copyfile(html_file, src_path) - creator.add_item(StaticItem(filepath=src_path, path=src_path.name, ref=tmpdir)) + creator.add_item( + StaticItem( + filepath=src_path, path=src_path.name, ref=tmpdir # pyright: ignore + ) + ) del tmpdir assert not src_path.exists() @@ -234,7 +242,7 @@ def test_sourcefile_removal_std(tmp_path, html_file): StaticItem( filepath=paths[-1], path=paths[-1].name, - mimetype="text/html", + mimetype="text/html", # pyright: ignore ), callback=(delete_callback, paths[-1]), ) @@ -323,7 +331,9 @@ def test_filelikeprovider_nosize(tmp_path, png_image_url): fpath = tmp_path / "test.zim" with Creator(fpath, "").config_dev_metadata() as creator: - creator.add_item(FileLikeProviderItem(fileobj=fileobj, path="one.png")) + creator.add_item( + FileLikeProviderItem(fileobj=fileobj, path="one.png") # pyright: ignore + ) zim = Archive(fpath) assert bytes(zim.get_item("one.png").content) == fileobj.getvalue() @@ -337,7 +347,9 @@ def test_urlprovider(tmp_path, png_image_url): fpath = tmp_path / "test.zim" with Creator(fpath, "").config_dev_metadata() as creator: - creator.add_item(SpecialURLProviderItem(url=png_image_url, path="one.png")) + creator.add_item( + SpecialURLProviderItem(url=png_image_url, path="one.png") # pyright: ignore + ) zim = Archive(fpath) assert bytes(zim.get_item("one.png").content) == file_bytes @@ -400,7 +412,8 @@ def do_GET(self): creator.add_item( SpecialURLProviderItem( - url=f"http://localhost:{port}/home.png", mimetype="image/png" + url=f"http://localhost:{port}/home.png", # pyright: ignore + mimetype="image/png", # pyright: ignore ) ) finally: @@ -493,7 +506,7 @@ def test_without_metadata(tmp_path): def test_check_metadata(tmp_path): with pytest.raises(ValueError, match="Counter cannot be set"): - Creator(tmp_path, "").config_dev_metadata(Counter=1).start() + Creator(tmp_path, "").config_dev_metadata(Counter=1).start() # pyright: ignore with pytest.raises(ValueError, match="Description is too long."): Creator(tmp_path, "").config_dev_metadata(Description="T" * 90).start() From c9cf3431ab181bd51583776eb1765bb0424bbb65 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 16:43:28 +0100 Subject: [PATCH 067/335] Add 3.12 support and standardize CI --- .github/workflows/Publish.yaml | 28 ++++++++++++++++ .github/workflows/QA.yaml | 34 +++++++++++++++++++ .github/workflows/Tests.yaml | 60 ++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 54 ------------------------------ .github/workflows/release.yml | 34 ------------------- pyproject.toml | 2 +- 6 files changed, 123 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/Publish.yaml create mode 100644 .github/workflows/QA.yaml create mode 100644 .github/workflows/Tests.yaml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/Publish.yaml b/.github/workflows/Publish.yaml new file mode 100644 index 00000000..b7e1859b --- /dev/null +++ b/.github/workflows/Publish.yaml @@ -0,0 +1,28 @@ +name: Build and upload to PyPI + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-22.04 + permissions: + id-token: write # mandatory for PyPI trusted publishing + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Build packages + run: | + pip install -U pip build + python -m build --sdist --wheel + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1.8 diff --git a/.github/workflows/QA.yaml b/.github/workflows/QA.yaml new file mode 100644 index 00000000..48ccee5a --- /dev/null +++ b/.github/workflows/QA.yaml @@ -0,0 +1,34 @@ +name: QA + +on: + pull_request: + push: + branches: + - main + +jobs: + check-qa: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[lint,scripts,test,check] + + - name: Check black formatting + run: inv lint-black + + - name: Check ruff + run: inv lint-ruff + + - name: Check pyright + run: inv check-pyright diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml new file mode 100644 index 00000000..1d437788 --- /dev/null +++ b/.github/workflows/Tests.yaml @@ -0,0 +1,60 @@ +name: Tests + +on: + pull_request: + push: + branches: + - main + +jobs: + run-tests: + strategy: + matrix: + os: [ubuntu-22.04] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + + - name: install ffmpeg and gifsicle + run: sudo apt update && sudo apt install ffmpeg gifsicle + + - name: add required locales for tests + run: sudo locale-gen fr_FR.UTF-8 pt_BR.UTF-8 && sudo update-locale + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[test,scripts] + + - name: Run the tests + run: inv coverage --args "-vvv" + + - name: Upload coverage report to codecov + if: matrix.python == '3.12' + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + build_python: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Ensure we can build Python targets + run: | + pip install -U pip build + python3 -m build --sdist --wheel diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 953a6867..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: CI - -on: [push, pull_request] - -jobs: - unit-tests: - runs-on: ubuntu-22.04 - strategy: - max-parallel: 3 - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - steps: - - name: install ffmpeg and gifsicle - run: sudo apt update && sudo apt install ffmpeg gifsicle - - name: add required locales for tests - run: sudo locale-gen fr_FR.UTF-8 pt_BR.UTF-8 && sudo update-locale - - uses: actions/checkout@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: pyvar - run: echo "pyversion=${{ matrix.python-version }}" >> $GITHUB_ENV - - name: set image name - run: echo "toxpyv=py${pyversion//\./}" >> $GITHUB_ENV - - name: verify envname - run : echo $toxpyv - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install tox - - name: Test with tox - run: tox -e $toxpyv - - qa: - runs-on: ubuntu-22.04 - steps: - - name: install ffmpeg and gifsicle - run: sudo apt update && sudo apt install ffmpeg gifsicle - - name: add required locales for tests - run: sudo locale-gen fr_FR.UTF-8 pt_BR.UTF-8 && sudo update-locale - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: "3.8" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U tox - - name: Run tox - run: tox -e py38,black,isort,flake8,coverage - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index d72c5d8d..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: release -on: - release: - types: [published] - tags: - - v* - -env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - -jobs: - release: - environment: release - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - - name: Build sdist and wheel - run: | - pip install --upgrade pip build wheel - python3 -m build - - - name: Push release to PyPI - if: github.event_name == 'release' - run: | - pip install --upgrade twine - twine check dist/* - twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml index b3cd7cd8..83d02756 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "zimscraperlib" -requires-python = ">=3.8,<3.12" +requires-python = ">=3.8,<3.13" description = "Collection of python tools to re-use common code across scrapers" readme = "README.md" dependencies = [ From 5a0feca9d6af381451d891f73db445e4ca29e3b1 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 6 Feb 2024 18:36:00 +0100 Subject: [PATCH 068/335] Migrate from iso-639 to iso639-lang library, former one is not maintained anymore --- pyproject.toml | 2 +- src/zimscraperlib/i18n.py | 51 +++++++++++++++++++++++++++------------ tests/i18n/test_i18n.py | 6 ++--- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 83d02756..b304a71f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires-python = ">=3.8,<3.13" description = "Collection of python tools to re-use common code across scrapers" readme = "README.md" dependencies = [ - "iso-639==0.4.5", + "iso639-lang==2.2.3", "requests==2.31.0", "colorthief==0.2.1", "python-resize-image==1.1.20", diff --git a/src/zimscraperlib/i18n.py b/src/zimscraperlib/i18n.py index fdff9904..0c9e1948 100644 --- a/src/zimscraperlib/i18n.py +++ b/src/zimscraperlib/i18n.py @@ -8,7 +8,8 @@ from typing import Dict, Optional, Tuple, Union import babel -from iso639 import languages as iso639_languages +import iso639 +import iso639.exceptions ISO_LEVELS = ["1", "2b", "2t", "3", "5"] @@ -67,27 +68,45 @@ def get_iso_lang_data(lang: str) -> Tuple[Dict, Union[Dict, None]]: iso_types = [] - for code_type in [f"part{lang_}" for lang_ in ISO_LEVELS] + ["name"]: - try: - iso639_languages.get(**{code_type: lang}) - iso_types.append(code_type) - except KeyError: - pass - - if not iso_types: - raise NotFound("Not a valid iso language name/code") - - language = iso639_languages.get(**{iso_types[0]: lang}) + try: + isolang = iso639.Lang(lang) + except ( + iso639.exceptions.InvalidLanguageValue, + iso639.exceptions.DeprecatedLanguageValue, + ) as exc: + raise NotFound("Not a valid iso language name/code") from exc + + def replace_types(new_type: str) -> str: + # convert new iso_types from iso639-lang Pypi package to old iso_types from + # iso-639 package, since we were returning these values for a long time + if new_type == "pt1": + return "part1" + elif new_type == "pt2b": + return "part2b" + elif new_type == "pt2t": + return "part2t" + elif new_type == "pt3": + return "part3" + elif new_type == "pt5": + return "part5" + return new_type + + for code_type in [f"pt{lang_}" for lang_ in ISO_LEVELS] + ["name"]: + # the `if` condition below is a bit hackish but it is the only way to know + # if the passed value is matching a code type or not with new python-i639 + # library and we do not expect weird things to happen here + if str(getattr(isolang, code_type)).lower() == lang.lower(): + iso_types.append(replace_types(code_type)) lang_data = { - f"iso-639-{lang_}": getattr(language, f"part{lang_}") for lang_ in ISO_LEVELS + f"iso-639-{lang_}": getattr(isolang, f"pt{lang_}") for lang_ in ISO_LEVELS } - lang_data.update({"english": language.name, "iso_types": iso_types}) + lang_data.update({"english": isolang.name, "iso_types": iso_types}) - if language.macro: + if isolang.macro(): return ( lang_data, - get_iso_lang_data(language.macro)[0], + get_iso_lang_data(isolang.macro().name)[0], ) # first item in the returned tuple return lang_data, None diff --git a/tests/i18n/test_i18n.py b/tests/i18n/test_i18n.py index bb78d8a5..75a7226c 100644 --- a/tests/i18n/test_i18n.py +++ b/tests/i18n/test_i18n.py @@ -96,11 +96,11 @@ def test_selocale_unsupported(tmp_path): { "iso-639-1": "", "iso-639-2b": "afa", - "iso-639-2t": "afa", + "iso-639-2t": "", "iso-639-3": "", "iso-639-5": "afa", "english": "Afro-Asiatic languages", - "iso_types": ["part2b", "part2t", "part5"], + "iso_types": ["part2b", "part5"], "querytype": "purecode", "query": "afa", "native": "Afro-Asiatic languages", @@ -111,7 +111,7 @@ def test_selocale_unsupported(tmp_path): { "iso-639-1": "", "iso-639-2b": "afa", - "iso-639-2t": "afa", + "iso-639-2t": "", "iso-639-3": "", "iso-639-5": "afa", "english": "Afro-Asiatic languages", From f3d6e0907c0820775fe36b55e2f54d935fa618c5 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 8 Feb 2024 10:04:21 +0100 Subject: [PATCH 069/335] Run all tests and install fix_ogvjs_dist script --- .github/workflows/Tests.yaml | 2 +- pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 1d437788..7db6f910 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -35,7 +35,7 @@ jobs: pip install -e .[test,scripts] - name: Run the tests - run: inv coverage --args "-vvv" + run: inv coverage --args "--runslow --runinstalled -vvv" - name: Upload coverage report to codecov if: matrix.python == '3.12' diff --git a/pyproject.toml b/pyproject.toml index b304a71f..ee641539 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,9 @@ dev = [ "zimscraperlib[check]", ] +[project.scripts] +fix_ogvjs_dist = "zimscraperlib.fix_ogvjs_dist:run" + [tool.hatch.version] path = "src/zimscraperlib/__about__.py" From 07755c34146f40839e67ed447030881be58a85f3 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 8 Feb 2024 11:28:48 +0100 Subject: [PATCH 070/335] Fix nocover pragmas --- src/zimscraperlib/fix_ogvjs_dist.py | 2 +- src/zimscraperlib/image/optimization.py | 4 ++-- src/zimscraperlib/zim/archive.py | 2 +- src/zimscraperlib/zim/providers.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/zimscraperlib/fix_ogvjs_dist.py b/src/zimscraperlib/fix_ogvjs_dist.py index 00d1b5e2..6c0358e5 100755 --- a/src/zimscraperlib/fix_ogvjs_dist.py +++ b/src/zimscraperlib/fix_ogvjs_dist.py @@ -34,7 +34,7 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]): def run(): - if len(sys.argv) < 2: # noqa: PLR2004 + if len(sys.argv) < 2: # noqa: PLR2004 # pragma: no cover print(f"Usage: {sys.argv[0]} ") # noqa: T201 print( # noqa: T201 "\t\tpath to your folder containing " diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 431c7c79..80c9ac73 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -205,7 +205,7 @@ def optimize_webp( save_image(webp_image, dst, fmt="WEBP", **params) # pyright: ignore except Exception as exc: if src.resolve() != dst.resolve() and dst.exists(): # pyright: ignore - dst.unlink() # pragma: nocover + dst.unlink() # pragma: no cover raise exc return dst # pyright: ignore @@ -256,7 +256,7 @@ def optimize_gif( # remove dst if gifsicle failed and src is different from dst if gifsicle.returncode != 0 and src.resolve() != dst.resolve() and dst.exists(): - dst.unlink() # pragma: nocover + dst.unlink() # pragma: no cover # raise error if unsuccessful gifsicle.check_returncode() diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 62768589..7d69ee9a 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -43,7 +43,7 @@ def get_tags(self, libkiwix: bool = False) -> List[str]: # noqa: FBT001, FBT002 """List of ZIM tags, optionnaly expanded with libkiwix's hints""" try: tags_meta = self.get_text_metadata("Tags") - except RuntimeError: # pragma: nocover + except RuntimeError: # pragma: no cover tags_meta = "" if libkiwix: diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index 913403fa..ff23b1eb 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -61,9 +61,9 @@ def get_size(self) -> int: return self.size # pyright: ignore def gen_blob(self) -> libzim.writer.Blob: - yield libzim.writer.Blob( + yield libzim.writer.Blob( # pragma: no cover self.fileobj.getvalue() # pyright: ignore - ) # pragma: nocover + ) class URLProvider(libzim.writer.ContentProvider): @@ -95,7 +95,7 @@ def get_size_of(url) -> Union[int, None]: def get_size(self) -> int: return self.size # pyright: ignore - def gen_blob(self) -> libzim.writer.Blob: # pragma: nocover + def gen_blob(self) -> libzim.writer.Blob: for chunk in self.resp.iter_content(10 * 1024): if chunk: yield libzim.writer.Blob(chunk) From e1c87afe94765dcbbdfec64e757a99ff380e62c6 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 8 Feb 2024 13:46:47 +0100 Subject: [PATCH 071/335] Fix changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c849d881..13f3a4db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120 +- Suuport for Python 3.12, drop Python 3.7 #118 +- Replace "iso-369" iso639-lang by "iso639-lang" library ## [3.2.0] - 2023-12-16 From dcded70ada76b8677e4fa4532a6069c0228fa425 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 14:23:32 +0100 Subject: [PATCH 072/335] Fix many qa issues and one typing issue --- pyproject.toml | 2 ++ src/zimscraperlib/misc.py | 4 ++-- src/zimscraperlib/uri.py | 20 +++++++++--------- src/zimscraperlib/video/encoding.py | 13 +++++++----- src/zimscraperlib/video/probing.py | 5 ++--- src/zimscraperlib/zim/_libkiwix.py | 32 ++++++++++++++--------------- src/zimscraperlib/zim/creator.py | 8 ++------ src/zimscraperlib/zim/items.py | 4 ++-- src/zimscraperlib/zim/metadata.py | 4 ++-- tests/download/test_download.py | 17 ++++++++++----- tests/html/conftest.py | 4 ++-- tests/html/test_html.py | 4 ++-- tests/image/test_image.py | 14 ++++++++----- tests/logging/test_logging.py | 10 ++++----- tests/ogvjs/test_ogvjs.py | 4 +--- tests/zim/test_archive.py | 2 +- tests/zim/test_libkiwix.py | 6 +++--- tests/zim/test_zim_creator.py | 1 - 18 files changed, 81 insertions(+), 73 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee641539..8003a58e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -206,6 +206,8 @@ ban-relative-imports = "all" [tool.ruff.lint.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] +# _libkiwix mimics libkiwix C++ code, names obey C++ conventions +"src/zimscraperlib/zim/_libkiwix.py" = ["N802", "N803", "N806"] [tool.pytest.ini_options] minversion = "7.3" diff --git a/src/zimscraperlib/misc.py b/src/zimscraperlib/misc.py index 55f32d8c..6cba114d 100644 --- a/src/zimscraperlib/misc.py +++ b/src/zimscraperlib/misc.py @@ -1,8 +1,8 @@ """ Miscelaneous utils""" -from typing import Iterable +from typing import Optional -def first(*args: Iterable[object]) -> object: +def first(*args: Optional[object]) -> object: """first non-None value from *args ; fallback to empty string""" return next((item for item in args if item is not None), "") diff --git a/src/zimscraperlib/uri.py b/src/zimscraperlib/uri.py index 96ce4708..b4906473 100644 --- a/src/zimscraperlib/uri.py +++ b/src/zimscraperlib/uri.py @@ -1,7 +1,7 @@ """ URI handling module""" import urllib.parse -from typing import Union +from typing import Optional, Union from zimscraperlib import logger from zimscraperlib.misc import first @@ -9,15 +9,15 @@ def rebuild_uri( uri: urllib.parse.ParseResult, - scheme: str = None, # noqa: RUF013 # pyright: ignore - username: str = None, # noqa: RUF013 # pyright: ignore - password: str = None, # noqa: RUF013 # pyright: ignore - hostname: str = None, # noqa: RUF013 # pyright: ignore - port: Union[str, int] = None, # noqa: RUF013 # pyright: ignore - path: str = None, # noqa: RUF013 # pyright: ignore - params: str = None, # noqa: RUF013 # pyright: ignore - query: str = None, # noqa: RUF013 # pyright: ignore - fragment: str = None, # noqa: RUF013 # pyright: ignore + scheme: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + hostname: Optional[str] = None, + port: Optional[Union[str, int]] = None, + path: Optional[str] = None, + params: Optional[str] = None, + query: Optional[str] = None, + fragment: Optional[str] = None, failsafe: bool = False, # noqa: FBT001, FBT002 ) -> urllib.parse.ParseResult: """new ParseResult named tuple from uri with requested part updated""" diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index 79100f3a..e8b86d7b 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -32,11 +32,14 @@ def reencode( with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = pathlib.Path(tmp_dir).joinpath(f"video.tmp{dst_path.suffix}") - args = ( - ["ffmpeg", "-y", "-i", f"file:{src_path}"] # noqa: RUF005 - + ffmpeg_args - + [f"file:{tmp_path}"] - ) + args = [ + "ffmpeg", + "-y", + "-i", + f"file:{src_path}", + *ffmpeg_args, + f"file:{tmp_path}", + ] logger.debug( f"Encode {src_path} -> {dst_path} video format = {dst_path.suffix}" ) diff --git a/src/zimscraperlib/video/probing.py b/src/zimscraperlib/video/probing.py index fbad0859..9c35fc0b 100644 --- a/src/zimscraperlib/video/probing.py +++ b/src/zimscraperlib/video/probing.py @@ -25,10 +25,9 @@ def get_media_info(src_path): "-of", "csv", ] - ffprobe = subprocess.run( # noqa: UP022 + ffprobe = subprocess.run( args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, text=True, check=False, ) diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index af1eec5c..1a6a298c 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -40,29 +40,29 @@ def getline(src: io.StringIO, delim: Optional[bool] = None) -> Tuple[bool, str]: return char == "", output -def readFullMimetypeAndCounterString( # noqa: N802 +def readFullMimetypeAndCounterString( src: io.StringIO, ) -> Tuple[bool, str]: """read a single mimetype-and-counter string from source Returns whether the source is EOF and the extracted string (or empty one)""" params = "" - eof, mtcStr = getline(src, ";") # noqa: N806 # pyright: ignore + eof, mtcStr = getline(src, ";") # pyright: ignore if mtcStr.find("=") == -1: while params.count("=") != 2: # noqa: PLR2004 eof, params = getline(src, ";") # pyright: ignore if params.count("=") == 2: # noqa: PLR2004 - mtcStr += ";" + params # noqa: N806 + mtcStr += ";" + params if eof: break return eof, mtcStr -def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: # noqa: N802 +def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: """MimetypeAndCounter from a single mimetype-and-counter string""" k: int = string.rfind("=") if k != len(string) - 1: - mimeType = string[:k] # noqa: N806 + mimeType = string[:k] counter = string[k + 1 :] if counter: try: @@ -72,15 +72,15 @@ def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: # noqa: N80 return MimetypeAndCounter("", 0) -def parseMimetypeCounter( # noqa: N802 - counterData: str, # noqa: N803 +def parseMimetypeCounter( + counterData: str, ) -> CounterMap: """Mapping of MIME types with count for each from ZIM Counter metadata string""" - counters = dict() # noqa: C408 + counters = {} ss = io.StringIO(counterData) eof = False while not eof: - eof, mtcStr = readFullMimetypeAndCounterString(ss) # noqa: N806 + eof, mtcStr = readFullMimetypeAndCounterString(ss) mtc = parseASingleMimetypeCounter(mtcStr) if mtc.mimetype: counters.update([mtc]) @@ -88,19 +88,19 @@ def parseMimetypeCounter( # noqa: N802 return counters -def convertTags(tags_str: str) -> List[str]: # noqa: N802 +def convertTags(tags_str: str) -> List[str]: """List of tags expanded with libkiwix's additional hints for pic/vid/det/index""" tags = tags_str.split(";") - tagsList = [] # noqa: N806 - picSeen = vidSeen = detSeen = indexSeen = False # noqa: N806 + tagsList = [] + picSeen = vidSeen = detSeen = indexSeen = False for tag in tags: # not upstream if not tag: continue - picSeen |= tag == "nopic" or tag.startswith("_pictures:") # noqa: N806 - vidSeen |= tag == "novid" or tag.startswith("_videos:") # noqa: N806 - detSeen |= tag == "nodet" or tag.startswith("_details:") # noqa: N806 - indexSeen |= tag.startswith("_ftindex") # noqa: N806 + picSeen |= tag == "nopic" or tag.startswith("_pictures:") + vidSeen |= tag == "novid" or tag.startswith("_videos:") + detSeen |= tag == "nodet" or tag.startswith("_details:") + indexSeen |= tag.startswith("_ftindex") if tag == "nopic": tagsList.append("_pictures:no") diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 44d2d1e2..6ab905db 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -107,7 +107,7 @@ def __init__( ignore_duplicates: Optional[bool] = False, # noqa: FBT002 ): super().__init__(filename=filename) - self._metadata = dict() # noqa: C408 + self._metadata = {} self.__indexing_configured = False self.can_finish = True @@ -137,11 +137,7 @@ def config_indexing( return self def start(self): - if not all( - [ # noqa: C419 - self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS - ] # noqa: C419, RUF100 - ): + if not all(self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS): raise ValueError("Mandatory metadata are not all set.") for name, value in self._metadata.items(): diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index 6ea51a4a..a3d20d74 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -46,7 +46,7 @@ def get_mimetype(self) -> str: return getattr(self, "mimetype", "") def get_hints(self) -> dict: - return getattr(self, "hints", dict()) # noqa: C408 + return getattr(self, "hints", {}) class StaticItem(Item): @@ -117,7 +117,7 @@ def __init__(self, url: str, **kwargs): url, byte_stream=io.BytesIO(), only_first_block=True ) except Exception as exc: - raise OSError(f"Unable to access URL at {url}: {exc}") # noqa: B904 + raise OSError(f"Unable to access URL at {url}: {exc}") from None # HTML content will be indexed. # we proxy the content in the Item to prevent double-download of the resource diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index 7962bd28..66e92f13 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -60,7 +60,7 @@ def validate_date(name: str, value: Union[datetime.datetime, datetime.date, str] } ) except Exception as exc: - raise ValueError(f"Invalid {name} format: {exc}") # noqa: B904 + raise ValueError(f"Invalid {name} format: {exc}") from None def validate_language(name: str, value: Union[Iterable[str], str]): @@ -98,7 +98,7 @@ def validate_tags(name: str, value: Union[Iterable[str], str]): """ensures Tags metadata is either one or a list of strings""" if name == "Tags" and ( not isinstance(value, IterableT) - or not all([isinstance(tag, str) for tag in value]) # noqa: C419 + or not all(isinstance(tag, str) for tag in value) ): raise ValueError(f"Invalid type(s) for {name}") diff --git a/tests/download/test_download.py b/tests/download/test_download.py index c8a0f6dc..6eef0756 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -4,9 +4,11 @@ import concurrent.futures import io import pathlib +import re import pytest import requests +from yt_dlp import DownloadError from zimscraperlib.download import ( BestMp4, @@ -16,11 +18,13 @@ stream_file, ) +DEFAULT_REQUEST_TIMEOUT = 60 + def assert_downloaded_file(url, file): assert file.exists() # our google test urls dont support HEAD - req = requests.get(url) # noqa: S113 + req = requests.get(url, timeout=DEFAULT_REQUEST_TIMEOUT) # we test against binary response: Content-Length not accurate as gzip-encoded assert file.stat().st_size == len(req.content) @@ -36,7 +40,7 @@ def get_dest_file(tmp_path): return tmp_path.joinpath("favicon.ico") -def test_missing_dest(tmp_path): # noqa: ARG001 +def test_missing_dest(): with pytest.raises(requests.exceptions.ConnectionError): stream_file(url="http://some_url", byte_stream=io.BytesIO()) @@ -99,7 +103,10 @@ def test_stream_to_bytes(valid_https_url): byte_stream = io.BytesIO() size, ret = stream_file(url=valid_https_url, byte_stream=byte_stream) assert_headers(ret) - assert byte_stream.read() == requests.get(valid_https_url).content # noqa: S113 + assert ( + byte_stream.read() + == requests.get(valid_https_url, timeout=DEFAULT_REQUEST_TIMEOUT).content + ) @pytest.mark.slow @@ -168,9 +175,9 @@ def test_youtube_download_nowait(tmp_path): @pytest.mark.slow -def test_youtube_download_error(tmp_path): # noqa: ARG001 +def test_youtube_download_error(): yt_downloader = YoutubeDownloader(threads=1) - with pytest.raises(Exception): # noqa: B017 + with pytest.raises(DownloadError, match=re.escape("is not a valid URL")): yt_downloader.download("11", BestMp4.get_options()) yt_downloader.shutdown() diff --git a/tests/html/conftest.py b/tests/html/conftest.py index b87ac0f1..f31cbcef 100644 --- a/tests/html/conftest.py +++ b/tests/html/conftest.py @@ -14,7 +14,7 @@ def html_page(): - Kiwix lets you access free knowledge – even offline + Kiwix lets you access free knowledge - even offline Date: Fri, 9 Feb 2024 14:26:06 +0100 Subject: [PATCH 073/335] --runinstalled is not necessary anymore with bootstrap convention, we are always installed --- .github/workflows/Tests.yaml | 2 +- tests/conftest.py | 12 ------------ tests/ogvjs/test_ogvjs.py | 1 - 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 7db6f910..4b5d00c6 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -35,7 +35,7 @@ jobs: pip install -e .[test,scripts] - name: Run the tests - run: inv coverage --args "--runslow --runinstalled -vvv" + run: inv coverage --args "--runslow -vvv" - name: Upload coverage report to codecov if: matrix.python == '3.12' diff --git a/tests/conftest.py b/tests/conftest.py index c5e78583..bee8e890 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,28 +10,16 @@ def pytest_addoption(parser): parser.addoption( "--runslow", action="store_true", default=False, help="run slow tests" ) - parser.addoption( - "--runinstalled", - action="store_true", - default=False, - help="run tests checking for installed features", - ) def pytest_configure(config): config.addinivalue_line("markers", "slow: mark test as slow to run") - config.addinivalue_line( - "markers", "installed: mark test as testing installed features" - ) def pytest_collection_modifyitems(config, items): skip_slow = pytest.mark.skip(reason="need --runslow option to run") - skip_installed = pytest.mark.skip(reason="need --runinstalled option to run") for item in items: - if "installed" in item.keywords and not config.getoption("--runinstalled"): - item.add_marker(skip_installed) if "slow" in item.keywords and not config.getoption("--runslow"): item.add_marker(skip_slow) diff --git a/tests/ogvjs/test_ogvjs.py b/tests/ogvjs/test_ogvjs.py index 65692a38..b272d4de 100644 --- a/tests/ogvjs/test_ogvjs.py +++ b/tests/ogvjs/test_ogvjs.py @@ -44,7 +44,6 @@ def prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): tmp_path.joinpath(member).rename(tmp_path.joinpath("videojs-ogvjs.js")) -@pytest.mark.installed def test_installed_script(): kwargs = {"universal_newlines": True, "stdout": subprocess.PIPE} script = subprocess.run(["/usr/bin/env", "fix_ogvjs_dist"], **kwargs, check=False) From 4e5c216224a83ff2eb5372af23961f99fca419d9 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 14:54:34 +0100 Subject: [PATCH 074/335] Add back a lost # pragma: no cover --- src/zimscraperlib/zim/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index ff23b1eb..a698b4f5 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -95,7 +95,7 @@ def get_size_of(url) -> Union[int, None]: def get_size(self) -> int: return self.size # pyright: ignore - def gen_blob(self) -> libzim.writer.Blob: + def gen_blob(self) -> libzim.writer.Blob: # pragma: no cover for chunk in self.resp.iter_content(10 * 1024): if chunk: yield libzim.writer.Blob(chunk) From dccaa66cfd0211d75cd1f17baa14971cf102926d Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 16:10:12 +0100 Subject: [PATCH 075/335] Rewrite fix_ogvjs_dist and its tests to remove tests redundancies + ensure proper coverage --- src/zimscraperlib/fix_ogvjs_dist.py | 15 ++++---- tests/ogvjs/test_ogvjs.py | 54 ++++++++++++++++------------- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/zimscraperlib/fix_ogvjs_dist.py b/src/zimscraperlib/fix_ogvjs_dist.py index 6c0358e5..72cb8972 100755 --- a/src/zimscraperlib/fix_ogvjs_dist.py +++ b/src/zimscraperlib/fix_ogvjs_dist.py @@ -7,7 +7,7 @@ import logging import pathlib import sys -from typing import Union +from typing import List, Union logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -33,16 +33,17 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]): logger.info("all done.") -def run(): - if len(sys.argv) < 2: # noqa: PLR2004 # pragma: no cover - print(f"Usage: {sys.argv[0]} ") # noqa: T201 +def run(args: List[str] = sys.argv): + if len(args) < 2: # noqa: PLR2004 + print(f"Usage: {args[0]} ") # noqa: T201 print( # noqa: T201 "\t\tpath to your folder containing " "ogvjs/videojs/videojs-ogvjs." ) - sys.exit(1) - return sys.exit(fix_source_dir(sys.argv[1])) + return 1 + fix_source_dir(args[1]) + return 0 if __name__ == "__main__": - run() + sys.exit(run()) diff --git a/tests/ogvjs/test_ogvjs.py b/tests/ogvjs/test_ogvjs.py index b272d4de..94d4ada0 100644 --- a/tests/ogvjs/test_ogvjs.py +++ b/tests/ogvjs/test_ogvjs.py @@ -3,12 +3,12 @@ import shutil import subprocess -import sys import zipfile import pytest from zimscraperlib.download import save_large_file +from zimscraperlib.fix_ogvjs_dist import run def prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): @@ -44,43 +44,49 @@ def prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): tmp_path.joinpath(member).rename(tmp_path.joinpath("videojs-ogvjs.js")) -def test_installed_script(): - kwargs = {"universal_newlines": True, "stdout": subprocess.PIPE} - script = subprocess.run(["/usr/bin/env", "fix_ogvjs_dist"], **kwargs, check=False) - assert script.returncode == 1 - assert script.stdout.strip().startswith("Usage: ") - - -def test_missing_param(): +def test_ogvjs_installed_script_missing_param(): + # run from installed script to check real conditions script = subprocess.run( - [ - sys.executable, - "-m", - "zimscraperlib.fix_ogvjs_dist", - ], + ["/usr/bin/env", "fix_ogvjs_dist"], + text=True, + capture_output=True, check=False, ) assert script.returncode == 1 + assert script.stdout.strip().startswith("Usage: ") + + +def test_ogvjs_from_code_missing_params(): + # run from code to mesure coverage easily + + assert run(["fix_ogvjs_dist"]) == 1 @pytest.mark.slow -def test_fix_ogvjs_dist(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): +def test_ogvjs_installed_script_ok(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): + # run from installed script to check real conditions + prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url) - # run to fix it from source (using installed script name) script = subprocess.run( - [ - sys.executable, - "-m", - "zimscraperlib.fix_ogvjs_dist", - str(tmp_path), - ], + ["/usr/bin/env", "fix_ogvjs_dist", str(tmp_path)], text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + capture_output=True, check=False, ) assert script.returncode == 0 with open(tmp_path / "videojs-ogvjs.js") as fh: assert "webm" in fh.read() + + +@pytest.mark.slow +def test_ogvjs_from_code_ok(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url): + # run from code to mesure coverage easily + + prepare_ogvjs_folder(tmp_path, videojs_url, ogvjs_url, videojs_ogvjs_url) + + assert run(["fix_ogvjs_dist", str(tmp_path)]) == 0 + + with open(tmp_path / "videojs-ogvjs.js") as fh: + assert "webm" in fh.read() From a2f4650a7441a7b06a653d25b4bb2f81ffb5249c Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:02:27 +0100 Subject: [PATCH 076/335] Increase test coverage of zim/metadata.py --- tests/zim/test_metadata.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/zim/test_metadata.py diff --git a/tests/zim/test_metadata.py b/tests/zim/test_metadata.py new file mode 100644 index 00000000..c4d96255 --- /dev/null +++ b/tests/zim/test_metadata.py @@ -0,0 +1,32 @@ +import re +from typing import Iterable, Union + +import pytest + +from zimscraperlib.zim import metadata + + +@pytest.mark.parametrize( + "name, value", + [ + ("Language", "fra"), + ("Language", "fra,eng"), + ("Language", ["fra", "eng"]), + ("Other", "not_an_iso_639_3_code"), + ], +) +def test_validate_language_valid(name: str, value: Union[Iterable[str], str]): + metadata.validate_language(name, value) + + +@pytest.mark.parametrize( + "name, value", + [ + ("Language", "fr"), + ("Language", "fra;eng"), + ("Language", "fra, eng"), + ], +) +def test_validate_language_invalid(name: str, value: Union[Iterable[str], str]): + with pytest.raises(ValueError, match=re.escape("is not ISO-639-3")): + metadata.validate_language(name, value) From b588a7924b06bcac4f995ddbfdad107f6117cce8 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:04:09 +0100 Subject: [PATCH 077/335] Increase test coverage of download.py --- pyproject.toml | 1 + tests/download/test_download.py | 83 ++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8003a58e..f7ebd753 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ check = [ ] test = [ "pytest==8.0.0", + "pytest-mock==3.12.0", "coverage==7.4.1", ] dev = [ diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 6eef0756..874e45f9 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -58,7 +58,7 @@ def test_no_output_supplied(valid_http_url): stream_file(url=valid_http_url) -def test_first_block_download(valid_http_url): +def test_first_block_download_default_session(valid_http_url): byte_stream = io.BytesIO() size, ret = stream_file( url=valid_http_url, byte_stream=byte_stream, only_first_block=True @@ -70,6 +70,29 @@ def test_first_block_download(valid_http_url): assert len(byte_stream.read()) <= expected +def test_first_block_download_custom_session(mocker, valid_http_url): + byte_stream = io.BytesIO() + custom_session = mocker.Mock(spec=requests.Session) + + expected_response = requests.Response() + expected_response.status_code = 200 + expected_response.raw = io.BytesIO(b"Whatever\n") + custom_session.get.return_value = expected_response + + mocker.patch("requests.Session") + stream_file( + url=valid_http_url, + byte_stream=byte_stream, + only_first_block=True, + session=custom_session, + ) + # check that custom session has been used + custom_session.get.assert_called_once_with( + valid_http_url, stream=True, proxies=None, headers=None + ) + requests.Session.assert_not_called() # pyright: ignore + + @pytest.mark.slow def test_user_agent(): ua = "zimscraperlib-test" @@ -190,3 +213,61 @@ def test_youtube_download_contextmanager(tmp_path): ) assert yt_downloader.executor._shutdown assert tmp_path.joinpath("video.mp4").exists() # videmo doesn't offer webm + + +@pytest.fixture +def target_dir() -> pathlib.Path: + return pathlib.Path("adir1") + + +@pytest.fixture +def filepath() -> pathlib.Path: + return pathlib.Path("adir2/afile") + + +@pytest.fixture +def custom_outtmpl() -> str: + return "custom.%(ext)s" + + +def test_get_options_target_dir(target_dir): + options = BestWebm.get_options(target_dir=target_dir) + assert options["outtmpl"] == "adir1/video.%(ext)s" + + +def test_get_options_filepath(filepath): + options = BestWebm.get_options(filepath=filepath) + assert options["outtmpl"] == "adir2/afile" + + +def test_get_options_target_dir_filepath(target_dir, filepath): + options = BestWebm.get_options(target_dir=target_dir, filepath=filepath) + assert options["outtmpl"] == "adir1/adir2/afile" + + +def test_get_options_override_outtmpl_no_other_vars(custom_outtmpl): + original = BestWebm.get_options() + overriden = BestWebm.get_options(outtmpl=custom_outtmpl) + assert "outtmpl" in original + assert "outtmpl" in overriden + for key, value in original.items(): + if key != "outtmpl": + assert overriden[key] == value + else: + assert overriden[key] == custom_outtmpl + + +def test_get_options_override_outtmpl_other_vars(target_dir, filepath, custom_outtmpl): + original = BestWebm.get_options(target_dir=target_dir, filepath=filepath) + overriden = BestWebm.get_options( + target_dir=target_dir, + filepath=filepath, + outtmpl=custom_outtmpl, + ) + assert "outtmpl" in original + assert "outtmpl" in overriden + for key, value in original.items(): + if key != "outtmpl": + assert overriden[key] == value + else: + assert overriden[key] == custom_outtmpl From c399f6f82fef122f2ae4293be3fb3900c8354028 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:07:46 +0100 Subject: [PATCH 078/335] Increase test coverage of filesystem.py --- tests/filesystem/test_filesystem.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index 9da91a1b..71daba63 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -43,7 +43,7 @@ def test_mime_overrides(svg_image): assert get_content_mimetype(fh.read(64)) == expected_mime -def test_delete_callback(tmp_path): +def test_delete_callback_with_cb(tmp_path): class Store: called = 0 @@ -59,3 +59,13 @@ def cb(*args): # noqa: ARG001 assert not fpath.exists() assert Store.called assert Store.called == 1 + + +def test_delete_callback_without_cb(tmp_path): + fpath = tmp_path.joinpath("my-file") + with open(fpath, "w") as fh: + fh.write("content") + + delete_callback(fpath) + + assert not fpath.exists() From fbe613fea7f81c7b8c5c5c15de37b94e6ac1e8b8 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:29:58 +0100 Subject: [PATCH 079/335] Increase test coverage of image/optimization.py --- tests/image/test_image.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/image/test_image.py b/tests/image/test_image.py index a0041a22..8352cfe1 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -582,3 +582,15 @@ def test_is_valid_image(png_image, png_image2, jpg_image, font): assert is_valid_image(fh.read(), "PNG", (48, 48)) fh.seek(0) assert is_valid_image(io.BytesIO(fh.read()), "PNG", (48, 48)) + + +def test_optimize_gif_no_optimize_level(gif_image, tmp_path): + optimize_gif(gif_image, tmp_path / "out.gif", delete_src=False, optimize_level=None) + + +def test_optimize_gif_no_no_extensions(gif_image, tmp_path): + optimize_gif(gif_image, tmp_path / "out.gif", delete_src=False, no_extensions=None) + + +def test_optimize_gif_no_interlace(gif_image, tmp_path): + optimize_gif(gif_image, tmp_path / "out.gif", delete_src=False, interlace=None) From 6c620224bcca75f06f73d5ebe5a732a6dc1dfbc0 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:30:55 +0100 Subject: [PATCH 080/335] Increase test coverage of logging.py --- tests/logging/test_logging.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/logging/test_logging.py b/tests/logging/test_logging.py index 4367a3c5..235e66b4 100644 --- a/tests/logging/test_logging.py +++ b/tests/logging/test_logging.py @@ -142,3 +142,14 @@ def test_level_fallback(random_id, tmp_path): assert_message_file(logger, log_file, "warning", False) assert_message_file(logger, log_file, "error", False) assert_message_file(logger, log_file, "critical", True) + + +def test_no_output(random_id): + logger = getLogger(name=random_id, console=None, file=None) + logger.error("error") + + +def test_additional_deps(random_id): + assert logging.getLogger("something").level == logging.NOTSET + getLogger(name=random_id, additional_deps=["something"], console=None, file=None) + assert logging.getLogger("something").level == logging.WARNING From 75c7241db7c008b29a626c5668981636a38bba59 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:36:31 +0100 Subject: [PATCH 081/335] Increase test coverage of zim/_libkiwix.py + remove useless if branch --- src/zimscraperlib/zim/_libkiwix.py | 9 ++++----- tests/zim/test_libkiwix.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index 1a6a298c..117b5b20 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -64,11 +64,10 @@ def parseASingleMimetypeCounter(string: str) -> MimetypeAndCounter: if k != len(string) - 1: mimeType = string[:k] counter = string[k + 1 :] - if counter: - try: - return MimetypeAndCounter(mimeType, int(counter)) - except ValueError: - pass # value is not castable to int + try: + return MimetypeAndCounter(mimeType, int(counter)) + except ValueError: + pass # value is not castable to int return MimetypeAndCounter("", 0) diff --git a/tests/zim/test_libkiwix.py b/tests/zim/test_libkiwix.py index 39912c64..02fa218c 100644 --- a/tests/zim/test_libkiwix.py +++ b/tests/zim/test_libkiwix.py @@ -69,6 +69,7 @@ def test_getline(): ), ("text/html", empty), ("text/html=", empty), + ("text/html=0", {"text/html": 0}), ("text/html=foo", empty), ("text/html=123foo", empty), ("text/html=50;foo", {"text/html": 50}), From 40cadd463e34f5e8ae98b24ab0bdb1bd2fe31e72 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 17:44:20 +0100 Subject: [PATCH 082/335] Increase test coverage of zim/archive.py --- tests/zim/test_archive.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index d39c4a66..9ec27309 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -39,7 +39,14 @@ def test_get_item(small_zim_file): def test_suggestions(small_zim_file): with Archive(small_zim_file) as zim: assert zim.get_suggestions_count("test") == 1 - assert "main.html" in list(zim.get_suggestions("test")) + assert list(zim.get_suggestions("test")) == ["main.html"] + + +def test_suggestions_end_index(small_zim_file): + with Archive(small_zim_file) as zim: + assert zim.get_suggestions_count("test") == 1 + assert len(list(zim.get_suggestions("test", end=0))) == 0 + assert list(zim.get_suggestions("test", end=1)) == ["main.html"] def test_search_no_fti(small_zim_file): @@ -61,6 +68,15 @@ def test_search(real_zim_file): assert "A/Diesel_emissions_scandal" in list(zim.get_search_results("test")) +@pytest.mark.slow +def test_search_end_index(real_zim_file): + with Archive(real_zim_file) as zim: + assert list(zim.get_search_results("test", end=0)) == [] + assert "A/Diesel_emissions_scandal" in list( + zim.get_search_results("test", end=1) + ) + + def test_counters(small_zim_file): with Archive(small_zim_file) as zim: assert zim.counters == {"image/png": 1, "text/html": 1} From f9f0b9c965dff69135666da3a4fd08936fa7e9f9 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 18:00:33 +0100 Subject: [PATCH 083/335] Increase test coverage of zim/filesystem.py --- tests/zim/test_fs.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index 99360c81..348a68ad 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -58,7 +58,13 @@ def test_make_zim_file_fail_noillustration(build_data): assert not build_data["fpath"].exists() -def test_make_zim_file_working(build_data, png_image): +@pytest.mark.parametrize( + "with_redirects, with_redirects_file", + [(True, True), (True, False), (False, True), (False, False)], +) +def test_make_zim_file_working( + build_data, png_image, with_redirects, with_redirects_file +): build_data["build_dir"].mkdir() # add an image @@ -73,11 +79,20 @@ def test_make_zim_file_working(build_data, png_image): with open(build_data["build_dir"] / "app.js", "w") as fh: fh.write("console.log(window);") + if not with_redirects: + build_data.pop("redirects") + if not with_redirects_file: + build_data.pop("redirects_file") make_zim_file(**build_data) assert build_data["fpath"].exists() reader = Archive(build_data["fpath"]) - # welcome (actual) and two redirs - assert reader.entry_count == 8 # includes redirect + expected_entry_count = 4 + if with_redirects: + expected_entry_count += 1 + if with_redirects_file: + expected_entry_count += 3 + + assert reader.entry_count == expected_entry_count assert reader.get_item("style.css").mimetype == "text/css" assert reader.get_item("app.js").mimetype in ( From e018d5df35f5d49539429ea19d836a23a35ca462 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 9 Feb 2024 21:04:24 +0100 Subject: [PATCH 084/335] Remove pytest.raise match condition because message changed in Python 3.10 --- tests/image/test_image.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/image/test_image.py b/tests/image/test_image.py index 8352cfe1..28041737 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -549,9 +549,7 @@ def test_optimize_webp_gif_failure(tmp_path, webp_image, gif_image): dst = tmp_path.joinpath("image.img") # webp - with pytest.raises( - TypeError, match=re.escape("an integer is required (got type str)") - ): + with pytest.raises(TypeError): optimize_webp(webp_image, dst, lossless="bad") # pyright: ignore assert not dst.exists() From 0568e645291be94d4a6dacde7a46d7136d8c6c67 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 09:42:55 +0100 Subject: [PATCH 085/335] Fix options typing + add handling for incorrect outtmpl type --- src/zimscraperlib/download.py | 12 +++++++----- src/zimscraperlib/image/presets.py | 26 ++++++++++++++------------ src/zimscraperlib/video/config.py | 10 +++++++--- src/zimscraperlib/video/presets.py | 12 +++++++----- tests/download/test_download.py | 10 ++++++++++ 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 103cd5db..329400fe 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -7,7 +7,7 @@ import pathlib import subprocess from concurrent.futures import Future, ThreadPoolExecutor -from typing import Dict, Optional, Union +from typing import ClassVar, Dict, Optional, Union import requests import yt_dlp as youtube_dl @@ -65,8 +65,8 @@ def download( class YoutubeConfig(dict): - options = {} # noqa: RUF012 - defaults = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {} + defaults: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "writethumbnail": True, "write_all_thumbnails": True, "writesubtitles": True, @@ -94,6 +94,8 @@ def get_options( ): if "outtmpl" not in options: outtmpl = cls.options.get("outtmpl", cls.defaults["outtmpl"]) + if not isinstance(outtmpl, str): + raise ValueError(f"outtmpl must be a a str, {outtmpl.__class__} found") if filepath: outtmpl = str(filepath) # send output to target_dir @@ -107,14 +109,14 @@ def get_options( class BestWebm(YoutubeConfig): - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "preferredcodec": "webm", "format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best", } class BestMp4(YoutubeConfig): - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "preferredcodec": "mp4", "format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best", } diff --git a/src/zimscraperlib/image/presets.py b/src/zimscraperlib/image/presets.py index 5e4ebb38..8eab6991 100644 --- a/src/zimscraperlib/image/presets.py +++ b/src/zimscraperlib/image/presets.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from typing import ClassVar, Dict, Optional, Union + """ presets for ImageOptimizer in zimscraperlib.image.optimization module """ preset_type = "image" @@ -18,7 +20,7 @@ class WebpLow: ext = "webp" mimetype = f"{preset_type}/webp" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "lossless": False, "quality": 40, "method": 6, @@ -37,7 +39,7 @@ class WebpMedium: ext = "webp" mimetype = f"{preset_type}/webp" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "lossless": False, "quality": 50, "method": 6, @@ -56,7 +58,7 @@ class WebpHigh: ext = "webp" mimetype = f"{preset_type}/webp" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "lossless": False, "quality": 90, "method": 6, @@ -77,7 +79,7 @@ class GifLow: ext = "gif" mimetype = f"{preset_type}/gif" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "optimize_level": 3, "max_colors": 256, "lossiness": 80, @@ -100,7 +102,7 @@ class GifMedium: ext = "gif" mimetype = f"{preset_type}/gif" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "optimize_level": 3, "lossiness": 20, "no_extensions": True, @@ -122,7 +124,7 @@ class GifHigh: ext = "gif" mimetype = f"{preset_type}/gif" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "optimize_level": 2, "lossiness": None, "no_extensions": True, @@ -141,7 +143,7 @@ class PngLow: ext = "png" mimetype = f"{preset_type}/png" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "reduce_colors": True, "remove_transparency": False, "max_colors": 256, @@ -160,7 +162,7 @@ class PngMedium: ext = "png" mimetype = f"{preset_type}/png" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "reduce_colors": False, "remove_transparency": False, "fast_mode": False, @@ -178,7 +180,7 @@ class PngHigh: ext = "png" mimetype = f"{preset_type}/png" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "reduce_colors": False, "remove_transparency": False, "fast_mode": True, @@ -197,7 +199,7 @@ class JpegLow: ext = "png" mimetype = f"{preset_type}/png" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "quality": 45, "keep_exif": False, "fast_mode": True, @@ -216,7 +218,7 @@ class JpegMedium: ext = "jpg" mimetype = f"{preset_type}/jpeg" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "quality": 65, "keep_exif": False, "fast_mode": True, @@ -235,7 +237,7 @@ class JpegHigh: ext = "jpg" mimetype = f"{preset_type}/jpeg" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "quality": 80, "keep_exif": True, "fast_mode": True, diff --git a/src/zimscraperlib/video/config.py b/src/zimscraperlib/video/config.py index f2f8da9f..82d8ed54 100644 --- a/src/zimscraperlib/video/config.py +++ b/src/zimscraperlib/video/config.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from typing import ClassVar, Dict, Optional, Union + class Config(dict): VERSION = 1 ext = "dat" mimetype = "application/data" - options = {} # noqa: RUF012 - defaults = {"-max_muxing_queue_size": "9999"} # noqa: RUF012 - mapping = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {} + defaults: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + "-max_muxing_queue_size": "9999" + } + mapping: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "video_codec": "-codec:v", "audio_codec": "-codec:a", "max_video_bitrate": "-maxrate", diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index 31622a67..da55da2d 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from typing import ClassVar, Dict, Optional, Union + from zimscraperlib.video.config import Config preset_type = "video" @@ -18,7 +20,7 @@ class VoiceMp3Low(Config): ext = "mp3" mimetype = "audio/mp3" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-vn": "", # remove video stream "-codec:a": "mp3", # audio codec "-ar": "44100", # audio sampling rate @@ -39,7 +41,7 @@ class VideoWebmLow(Config): ext = "webm" mimetype = f"{preset_type}/webm" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset "-b:v": "300k", # target video bitrate @@ -67,7 +69,7 @@ class VideoMp4Low(Config): ext = "mp4" mimetype = f"{preset_type}/mp4" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "h264", # video codec "-b:v": "300k", # target video bitrate "-maxrate": "300k", # max video bitrate @@ -92,7 +94,7 @@ class VideoWebmHigh(Config): ext = "webm" mimetype = f"{preset_type}/webm" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-codec:a": "libvorbis", # audio codec "-crf": "25", # constant quality, lower value gives better qual and larger size @@ -110,7 +112,7 @@ class VideoMp4High(Config): ext = "mp4" mimetype = f"{preset_type}/mp4" - options = { # noqa: RUF012 + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "h264", # video codec "-codec:a": "aac", # audio codec "-crf": "20", # constant quality, lower value gives better qual and larger size diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 874e45f9..2c10093f 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -5,6 +5,7 @@ import io import pathlib import re +from typing import ClassVar, Dict, Optional, Union import pytest import requests @@ -230,6 +231,15 @@ def custom_outtmpl() -> str: return "custom.%(ext)s" +class WrongOuttmplType(BestWebm): + options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {"outtmpl": 123} + + +def test_get_options_wrong_outtmpl_type(): + with pytest.raises(ValueError): + WrongOuttmplType.get_options() + + def test_get_options_target_dir(target_dir): options = BestWebm.get_options(target_dir=target_dir) assert options["outtmpl"] == "adir1/video.%(ext)s" From b6db70833de60868543599a9da9e1c8a9ee30225 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 09:51:58 +0100 Subject: [PATCH 086/335] Skip test temporarily --- tests/download/test_download.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 2c10093f..bc37c34f 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -94,6 +94,7 @@ def test_first_block_download_custom_session(mocker, valid_http_url): requests.Session.assert_not_called() # pyright: ignore +@pytest.mark.skip(reason="name resolution issue for useragent.fr in Github Actions") @pytest.mark.slow def test_user_agent(): ua = "zimscraperlib-test" From f342cc27b5e55bcbcdd635101a68b30ee874f392 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 11:01:05 +0100 Subject: [PATCH 087/335] Restore Python dependencies ranges + widen the libzim range --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7ebd753..779791ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,16 +8,16 @@ requires-python = ">=3.8,<3.13" description = "Collection of python tools to re-use common code across scrapers" readme = "README.md" dependencies = [ - "iso639-lang==2.2.3", - "requests==2.31.0", + "iso639-lang>=2.2.3,<3.0", + "requests>=2.25.1,<3.0", "colorthief==0.2.1", - "python-resize-image==1.1.20", - "Babel==2.14.0", - "file-magic==0.4.1", - "libzim==3.4.0", - "beautifulsoup4==4.9.3", # upgrade to 4.10 and later to be done - "lxml==4.9.4", # upgrade to 4.10 and later to be done - "optimize-images==1.5.1", + "python-resize-image>=1.1.19,<1.2", + "Babel>=2.9,<3.0", + "file-magic>=0.4.0,<0.5", + "libzim>=3.4.0,<4.0", + "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done + "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done + "optimize-images>=1.3.6,<1.6", # youtube-dl should be updated as frequently as possible "yt-dlp" ] From 04964e13446dbca865087b5abbf6383f89a565c2 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 14:50:18 +0100 Subject: [PATCH 088/335] Use type instead of __class__ --- src/zimscraperlib/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 329400fe..2b8268f6 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -95,7 +95,7 @@ def get_options( if "outtmpl" not in options: outtmpl = cls.options.get("outtmpl", cls.defaults["outtmpl"]) if not isinstance(outtmpl, str): - raise ValueError(f"outtmpl must be a a str, {outtmpl.__class__} found") + raise ValueError(f"outtmpl must be a a str, {type(outtmpl)} found") if filepath: outtmpl = str(filepath) # send output to target_dir From 222bd80579b54ef152029c35616050b7c9aa2543 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 14:57:09 +0100 Subject: [PATCH 089/335] Update README contribution guidelines + user projects --- README.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ab3478a9..1d8870e7 100644 --- a/README.md +++ b/README.md @@ -47,27 +47,29 @@ sudo apt install libmagic1 wget ffmpeg \ # Contribution +This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing) + ```shell -pip -r requirements.txt -pip install tox pre-commit +pip install hatch +pip install ".[dev]" pre-commit install # For tests -tox +invoke coverage ``` # Users Non-exhaustive list of scrapers using it (check status when updating API): -* [openzim/youtube](https://github.com/openzim/youtube) +* [openzim/freecodecamp](https://github.com/openzim/freecodecamp) +* [openzim/gutenberg](https://github.com/openzim/gutenberg) +* [openzim/ifixit](https://github.com/openzim/ifixit) +* [openzim/kolibri](https://github.com/openzim/kolibri) * [openzim/nautilus](https://github.com/openzim/nautilus) - -# releasing - -* Update your dependencies: `pip install -U setuptools wheel twine` -* Make sure CHANGELOG.md is up-to-date -* Bump version on `src/zimscraperlib/VERSION` -* Build packages `python ./setup.py sdist bdist_wheel` -* Upload to PyPI `twine upload dist/zimscraperlib-2.0.0*`. -* Commit your Changelog + version bump changes -* Tag version on git `git tag -a v2.0.0` +* [openzim/nautilus](https://github.com/openzim/nautilus) +* [openzim/openedx](https://github.com/openzim/openedx) +* [openzim/sotoki](https://github.com/openzim/sotoki) +* [openzim/ted](https://github.com/openzim/ted) +* [openzim/warc2zim](https://github.com/openzim/warc2zim) +* [openzim/wikihow](https://github.com/openzim/wikihow) +* [openzim/youtube](https://github.com/openzim/youtube) From c46d33fcaa08b138f2283b77ee220e43d30fd2da Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 15:54:48 +0100 Subject: [PATCH 090/335] Use /usr/bin/env in every subprocess command --- src/zimscraperlib/image/optimization.py | 2 +- src/zimscraperlib/video/encoding.py | 1 + src/zimscraperlib/video/probing.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 80c9ac73..bed4c8a5 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -239,7 +239,7 @@ def optimize_gif( ensure_matches(src, "GIF") # use gifsicle - args = ["gifsicle"] + args = ["/usr/bin/env", "gifsicle"] if optimize_level: args += [f"-O{optimize_level}"] if max_colors: diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index e8b86d7b..20509e7a 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -33,6 +33,7 @@ def reencode( with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = pathlib.Path(tmp_dir).joinpath(f"video.tmp{dst_path.suffix}") args = [ + "/usr/bin/env", "ffmpeg", "-y", "-i", diff --git a/src/zimscraperlib/video/probing.py b/src/zimscraperlib/video/probing.py index 9c35fc0b..5f6b217f 100644 --- a/src/zimscraperlib/video/probing.py +++ b/src/zimscraperlib/video/probing.py @@ -13,6 +13,7 @@ def get_media_info(src_path): bitrate: file's main bitrate""" args = [ + "/usr/bin/env", "ffprobe", "-i", f"file:{src_path}", From ad4b2418046a654174a60bae93bfe49f9fb8aef9 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 15:55:06 +0100 Subject: [PATCH 091/335] Re-activate test This reverts commit b6db70833de60868543599a9da9e1c8a9ee30225. --- tests/download/test_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/download/test_download.py b/tests/download/test_download.py index bc37c34f..2c10093f 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -94,7 +94,6 @@ def test_first_block_download_custom_session(mocker, valid_http_url): requests.Session.assert_not_called() # pyright: ignore -@pytest.mark.skip(reason="name resolution issue for useragent.fr in Github Actions") @pytest.mark.slow def test_user_agent(): ua = "zimscraperlib-test" From f5b5460d5a5491ee97aa23905028a594b93f8ecf Mon Sep 17 00:00:00 2001 From: IMayBeABitShy Date: Tue, 6 Feb 2024 16:09:18 +0100 Subject: [PATCH 092/335] Fix handling of tag list in Creator.add_metadata (fix #125) This fixes issue #125. Previously, the type annotations of Creator.config_metadata() allowed "Tags" to be a list of strings, but didn't handle the conversion to a string, which resulted in python-libzim raising an error. This commit modifies Creator.add_metadata() to accept a list of strings and, if the key of the metadata is "Tags", join them into a single string. A regression test is included. --- src/zimscraperlib/zim/creator.py | 7 ++++- tests/zim/test_zim_creator.py | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 6ab905db..a703fdc4 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -183,10 +183,15 @@ def validate_metadata( def add_metadata( self, name: str, - content: Union[str, bytes, datetime.date, datetime.datetime], + content: Union[str, bytes, datetime.date, datetime.datetime, Iterable[str]], mimetype: str = "text/plain;charset=UTF-8", ): self.validate_metadata(name, content) + # handle necessary type conversions + if name == "Tags" and not isinstance(content, str): + # join list of tags into a single string + content = ";".join(content) + # NOTE: conversion of "Date" is handled in python-libzim super().add_metadata(name, content, mimetype) def config_metadata( diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 9c27685c..2041d312 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -573,6 +573,54 @@ def test_config_metadata(tmp_path, png_image): assert reader.get_text_metadata("TestMetadata") == "Test Metadata" +def test_creator_metadata_list_as_tags(tmp_path, png_image): + """ + Test creator metadata setup when "Tags" is a list of strings. + + The test case is based on test_config_metadata() + + This is also a regression test for #125. + """ + fpath = tmp_path / "test_config.zim" + with open(png_image, "rb") as fh: + png_data = fh.read() + creator = Creator(fpath, "").config_metadata( + Name="wikipedia_fr_football", + Title="English Wikipedia", + Creator="English speaking Wikipedia contributors", + Publisher="Wikipedia user Foobar", + Date="2009-11-21", + Description="All articles (without images) from the english Wikipedia", + LongDescription="This ZIM file contains all articles (without images)" + " from the english Wikipedia by 2009-11-10. The topics are...", + Language="eng", + License="CC-BY", + Tags=[ + "wikipedia", + "_category:wikipedia", + "_pictures:no", + "_videos:no", + "_details:yes", + "_ftindex:yes", + ], + Flavour="nopic", + Source="https://en.wikipedia.org/", + Scraper="mwoffliner 1.2.3", + Illustration_48x48_at_1=png_data, + TestMetadata="Test Metadata", + ) + with creator: + pass + + assert fpath.exists() + reader = Archive(fpath) + assert ( + reader.get_text_metadata("Tags") + == "wikipedia;_category:wikipedia;_pictures:no;_videos:no;" + "_details:yes;_ftindex:yes" + ) + + @pytest.mark.parametrize( "name,value,valid", [ From efb824d7dab70cf107ea599c41740be12482f9e9 Mon Sep 17 00:00:00 2001 From: IMayBeABitShy Date: Fri, 9 Feb 2024 14:31:21 +0100 Subject: [PATCH 093/335] Remove comments I didn't think I'd ever encounter an open source project opposed to comments, but here we are. Some comments will soon be outdated anyway. --- src/zimscraperlib/zim/creator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index a703fdc4..d209da60 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -187,11 +187,8 @@ def add_metadata( mimetype: str = "text/plain;charset=UTF-8", ): self.validate_metadata(name, content) - # handle necessary type conversions if name == "Tags" and not isinstance(content, str): - # join list of tags into a single string content = ";".join(content) - # NOTE: conversion of "Date" is handled in python-libzim super().add_metadata(name, content, mimetype) def config_metadata( From 38daa9c0e809d2de312f1ed50580e9840d551575 Mon Sep 17 00:00:00 2001 From: IMayBeABitShy Date: Fri, 9 Feb 2024 14:52:26 +0100 Subject: [PATCH 094/335] Convert date/datetime for Date metadata --- src/zimscraperlib/zim/creator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index d209da60..b3b4f1e0 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -187,6 +187,8 @@ def add_metadata( mimetype: str = "text/plain;charset=UTF-8", ): self.validate_metadata(name, content) + if name == "Date" and isinstance(content, (datetime.date, datetime.datetime)): + content = content.strftime("%Y-%m-%d").encode("UTF-8") if name == "Tags" and not isinstance(content, str): content = ";".join(content) super().add_metadata(name, content, mimetype) From fba609de41f97910523a35a795894a4166e98d20 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 17:48:14 +0100 Subject: [PATCH 095/335] Mutualize test code base + add pyright hint pyright ignore is mandatory because the type checker has no idea about the impact of validate_metadata/validate_tag. This might be fixed by a shared logic and a TypeGuard but is not available until Python 3.10 ; other solution would be to transfer metadata to a way more typed container after validation. --- src/zimscraperlib/zim/creator.py | 8 +++- tests/zim/test_zim_creator.py | 72 ++++++++++---------------------- 2 files changed, 28 insertions(+), 52 deletions(-) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index b3b4f1e0..4e25d1de 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -17,6 +17,7 @@ - content stored on object - can be used to store a filepath and content read from it (not stored) """ +import collections.abc import datetime import pathlib import re @@ -189,7 +190,12 @@ def add_metadata( self.validate_metadata(name, content) if name == "Date" and isinstance(content, (datetime.date, datetime.datetime)): content = content.strftime("%Y-%m-%d").encode("UTF-8") - if name == "Tags" and not isinstance(content, str): + if ( + name == "Tags" + and not isinstance(content, str) + and not isinstance(content, bytes) + and isinstance(content, collections.abc.Iterable) + ): content = ";".join(content) super().add_metadata(name, content, mimetype) diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 2041d312..9db1b1ca 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -514,7 +514,26 @@ def test_check_metadata(tmp_path): Creator(tmp_path, "").config_dev_metadata(LongDescription="T" * 5000).start() -def test_config_metadata(tmp_path, png_image): +@pytest.mark.parametrize( + "tags", + [ + ( + "wikipedia;_category:wikipedia;_pictures:no;_videos:no;_details:yes;" + "_ftindex:yes" + ), + ( + [ + "wikipedia", + "_category:wikipedia", + "_pictures:no", + "_videos:no", + "_details:yes", + "_ftindex:yes", + ] + ), + ], +) +def test_config_metadata(tmp_path, png_image, tags): fpath = tmp_path / "test_config.zim" with open(png_image, "rb") as fh: png_data = fh.read() @@ -529,8 +548,7 @@ def test_config_metadata(tmp_path, png_image): " from the english Wikipedia by 2009-11-10. The topics are...", Language="eng", License="CC-BY", - Tags="wikipedia;_category:wikipedia;_pictures:no;_videos:no;" - "_details:yes;_ftindex:yes", + Tags=tags, Flavour="nopic", Source="https://en.wikipedia.org/", Scraper="mwoffliner 1.2.3", @@ -573,54 +591,6 @@ def test_config_metadata(tmp_path, png_image): assert reader.get_text_metadata("TestMetadata") == "Test Metadata" -def test_creator_metadata_list_as_tags(tmp_path, png_image): - """ - Test creator metadata setup when "Tags" is a list of strings. - - The test case is based on test_config_metadata() - - This is also a regression test for #125. - """ - fpath = tmp_path / "test_config.zim" - with open(png_image, "rb") as fh: - png_data = fh.read() - creator = Creator(fpath, "").config_metadata( - Name="wikipedia_fr_football", - Title="English Wikipedia", - Creator="English speaking Wikipedia contributors", - Publisher="Wikipedia user Foobar", - Date="2009-11-21", - Description="All articles (without images) from the english Wikipedia", - LongDescription="This ZIM file contains all articles (without images)" - " from the english Wikipedia by 2009-11-10. The topics are...", - Language="eng", - License="CC-BY", - Tags=[ - "wikipedia", - "_category:wikipedia", - "_pictures:no", - "_videos:no", - "_details:yes", - "_ftindex:yes", - ], - Flavour="nopic", - Source="https://en.wikipedia.org/", - Scraper="mwoffliner 1.2.3", - Illustration_48x48_at_1=png_data, - TestMetadata="Test Metadata", - ) - with creator: - pass - - assert fpath.exists() - reader = Archive(fpath) - assert ( - reader.get_text_metadata("Tags") - == "wikipedia;_category:wikipedia;_pictures:no;_videos:no;" - "_details:yes;_ftindex:yes" - ) - - @pytest.mark.parametrize( "name,value,valid", [ From 10ca3fac77904492f5ac6f5bd3f2b5fc588a83c7 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 20:03:56 +0100 Subject: [PATCH 096/335] Disable pyright bytes type promotion --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 779791ea..1184b0b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,3 +240,4 @@ exclude = [".env/**", ".venv/**"] extraPaths = ["src"] pythonVersion = "3.8" typeCheckingMode="basic" +disableBytesTypePromotions = true From ab988e54bcfb5ab49377d3ad9f9e47f9114fdddd Mon Sep 17 00:00:00 2001 From: Kevin McMurtrie Date: Sun, 28 Jan 2024 12:32:24 -0800 Subject: [PATCH 097/335] Fix VP8 low bitrate Tested with nearly motionless security camera footage and a television news recording of a press release. --- src/zimscraperlib/video/presets.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index da55da2d..9a22f03d 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -32,9 +32,8 @@ class VideoWebmLow(Config): """Low Quality webm video 480:h format with height adjusted to keep aspect ratio - 300k video bitrate - 48k audio bitrate - highly degraded quality (30, 42)""" + 200k target, 300k max video bitrate + 48k audio bitrate""" VERSION = 1 @@ -44,11 +43,10 @@ class VideoWebmLow(Config): options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset - "-b:v": "300k", # target video bitrate + "-b:v": "200k", # target video bitrate "-maxrate": "300k", # max video bitrate - "-minrate": "300k", # min video bitrate - "-qmin": "30", # min quantizer scale - "-qmax": "42", # max quantizer scale + "-bufsize": "512k", # target bitrate window + "-qmax": "30", # Max quantizer scale. Cap loss to reduce VP8 shimmer bug. "-vf": "scale='480:trunc(ow/a/2)*2'", # frame size "-codec:a": "libvorbis", # audio codec "-ar": "44100", # audio sampling rate From d143c987b4748b0f1ad71d49f495b176dd895a82 Mon Sep 17 00:00:00 2001 From: Kevin McMurtrie Date: Sun, 28 Jan 2024 21:54:58 -0800 Subject: [PATCH 098/335] Delete pointless identity test --- tests/video/test_video.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/tests/video/test_video.py b/tests/video/test_video.py index 31035273..0c0cb761 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -143,34 +143,7 @@ def test_preset_video_webm_low(): config = VideoWebmLow() assert config.VERSION == 1 args = config.to_ffmpeg_args() - assert len(args) == 24 - options_map = [ - ("codec:v", "libvpx"), - ("codec:a", "libvorbis"), - ("maxrate", "300k"), - ("minrate", "300k"), - ("b:v", "300k"), - ("ar", "44100"), - ("b:a", "48k"), - ("quality", "best"), - ("qmin", "30"), - ("qmax", "42"), - ("vf", "scale='480:trunc(ow/a/2)*2'"), - ] - for option, val in options_map: - idx = args.index(f"-{option}") - assert idx != -1 - assert args[idx + 1] == val - - # test updating values - config = VideoWebmLow(**{"-ar": "50000"}) - config["-bufsize"] = "900k" - args = config.to_ffmpeg_args() - idx = args.index("-ar") - assert idx != -1 and args[idx + 1] == "50000" - idx = args.index("-bufsize") - assert idx != -1 and args[idx + 1] == "900k" - + assert len(args) > 0 def test_preset_video_webm_high(): config = VideoWebmHigh() From 7880364f927edb442829e1bbd9bdeca0ac141f91 Mon Sep 17 00:00:00 2001 From: Kevin McMurtrie Date: Fri, 2 Feb 2024 20:24:01 -0800 Subject: [PATCH 099/335] Update presets.py Better tuning for very low motion and very high motion. --- src/zimscraperlib/video/presets.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index 9a22f03d..d001fa6f 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -32,7 +32,7 @@ class VideoWebmLow(Config): """Low Quality webm video 480:h format with height adjusted to keep aspect ratio - 200k target, 300k max video bitrate + 128k target video bitrate but stay within quality boundaries. 48k audio bitrate""" VERSION = 1 @@ -43,10 +43,9 @@ class VideoWebmLow(Config): options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset - "-b:v": "200k", # target video bitrate - "-maxrate": "300k", # max video bitrate - "-bufsize": "512k", # target bitrate window - "-qmax": "30", # Max quantizer scale. Cap loss to reduce VP8 shimmer bug. + "-b:v": "128k", # Adjust quantizer within min/max to target this bitrate + "-qmin": "18", # Reduce the bitrate on very still videos once the quality is good enough. + "-qmax": "40", # Increase the bitrate on very busy videos once the quality degrades too much. Also reduce key shimmer bug. "-vf": "scale='480:trunc(ow/a/2)*2'", # frame size "-codec:a": "libvorbis", # audio codec "-ar": "44100", # audio sampling rate From 10cc0e24a623c75437e953382804f899b051870e Mon Sep 17 00:00:00 2001 From: Kevin McMurtrie Date: Sat, 3 Feb 2024 15:37:17 -0800 Subject: [PATCH 100/335] Make style check happy? --- src/zimscraperlib/video/presets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index d001fa6f..c806cf4d 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -43,9 +43,9 @@ class VideoWebmLow(Config): options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset - "-b:v": "128k", # Adjust quantizer within min/max to target this bitrate - "-qmin": "18", # Reduce the bitrate on very still videos once the quality is good enough. - "-qmax": "40", # Increase the bitrate on very busy videos once the quality degrades too much. Also reduce key shimmer bug. + "-b:v": "128k", # Adjust quantizer within min/max to target this bitrate + "-qmin": "18", # Reduce the bitrate on very still videos + "-qmax": "40", # Increase the bitrate on very busy videos "-vf": "scale='480:trunc(ow/a/2)*2'", # frame size "-codec:a": "libvorbis", # audio codec "-ar": "44100", # audio sampling rate From a814079e730cb2830c126def204b74f9811be7a6 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 5 Feb 2024 09:12:36 +0100 Subject: [PATCH 101/335] Make style check happy --- src/zimscraperlib/video/presets.py | 4 ++-- tests/video/test_video.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index c806cf4d..89c2b3fb 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -44,8 +44,8 @@ class VideoWebmLow(Config): "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset "-b:v": "128k", # Adjust quantizer within min/max to target this bitrate - "-qmin": "18", # Reduce the bitrate on very still videos - "-qmax": "40", # Increase the bitrate on very busy videos + "-qmin": "18", # Reduce the bitrate on very still videos + "-qmax": "40", # Increase the bitrate on very busy videos "-vf": "scale='480:trunc(ow/a/2)*2'", # frame size "-codec:a": "libvorbis", # audio codec "-ar": "44100", # audio sampling rate diff --git a/tests/video/test_video.py b/tests/video/test_video.py index 0c0cb761..37788c01 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -145,6 +145,7 @@ def test_preset_video_webm_low(): args = config.to_ffmpeg_args() assert len(args) > 0 + def test_preset_video_webm_high(): config = VideoWebmHigh() assert config.VERSION == 1 From a3cb97168bc798ea0968c5475ebb3afe1f32d199 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 09:55:14 +0100 Subject: [PATCH 102/335] Add test details back --- tests/video/test_video.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/video/test_video.py b/tests/video/test_video.py index 37788c01..a3296a9c 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -143,7 +143,31 @@ def test_preset_video_webm_low(): config = VideoWebmLow() assert config.VERSION == 1 args = config.to_ffmpeg_args() - assert len(args) > 0 + assert len(args) == 20 + options_map = [ + ("codec:v", "libvpx"), + ("codec:a", "libvorbis"), + ("b:v", "128k"), + ("ar", "44100"), + ("b:a", "48k"), + ("quality", "best"), + ("qmin", "18"), + ("qmax", "40"), + ("vf", "scale='480:trunc(ow/a/2)*2'"), + ] + for option, val in options_map: + idx = args.index(f"-{option}") + assert idx != -1 + assert args[idx + 1] == val + + # test updating values + config = VideoWebmLow(**{"-ar": "50000"}) + config["-bufsize"] = "900k" + args = config.to_ffmpeg_args() + idx = args.index("-ar") + assert idx != -1 and args[idx + 1] == "50000" + idx = args.index("-bufsize") + assert idx != -1 and args[idx + 1] == "900k" def test_preset_video_webm_high(): From fb15186d02323643ddfb81d17b47e9763fe8b4a0 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 09:57:23 +0100 Subject: [PATCH 103/335] Bump VideoWebmLow VERSION number --- CHANGELOG.md | 1 + src/zimscraperlib/video/presets.py | 2 +- tests/video/test_video.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f3a4db..7bb2017f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120 - Suuport for Python 3.12, drop Python 3.7 #118 - Replace "iso-369" iso639-lang by "iso639-lang" library +- Rework the VideoWebmLow preset for faster encoding and smaller file size (preset has been bumped to version 2) ## [3.2.0] - 2023-12-16 diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index 89c2b3fb..37b7bbba 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -35,7 +35,7 @@ class VideoWebmLow(Config): 128k target video bitrate but stay within quality boundaries. 48k audio bitrate""" - VERSION = 1 + VERSION = 2 ext = "webm" mimetype = f"{preset_type}/webm" diff --git a/tests/video/test_video.py b/tests/video/test_video.py index a3296a9c..5b151f2e 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -141,7 +141,7 @@ def test_preset_has_mime_and_ext(): def test_preset_video_webm_low(): config = VideoWebmLow() - assert config.VERSION == 1 + assert config.VERSION == 2 args = config.to_ffmpeg_args() assert len(args) == 20 options_map = [ From 2fc4061ba5ff7f46dc5fd70f04bb720d23694930 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 14:33:14 +0100 Subject: [PATCH 104/335] Add a small utility to encode a video as scraper would do --- contrib/README.md | 3 +++ contrib/encode_video.py | 42 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 ++-- 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 contrib/README.md create mode 100644 contrib/encode_video.py diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 00000000..6b30c80e --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,3 @@ + +This folder contains some tooling around zimscraperlib: +- `encode_video.py`: a small utility to encode a video with an existing video preset, just like a scraper would do diff --git a/contrib/encode_video.py b/contrib/encode_video.py new file mode 100644 index 00000000..8b256e68 --- /dev/null +++ b/contrib/encode_video.py @@ -0,0 +1,42 @@ +import sys +from pathlib import Path +from typing import List + +from zimscraperlib import logger +from zimscraperlib.video import presets, reencode + + +def encode_video(src_path: Path, dst_path: Path, preset: str): + if not src_path.exists(): + raise ValueError(f"{src_path} does not exists") + try: + preset_cls = getattr(presets, preset) + except AttributeError: + logger.error(f"{preset} preset not found") + raise + logger.info(f"Encoding video {src_path} with {preset} version {preset_cls.VERSION}") + success, process = reencode( + src_path=src_path, + dst_path=dst_path, + ffmpeg_args=preset_cls().to_ffmpeg_args(), + with_process=True, + ) # pyright: ignore[reportGeneralTypeIssues] (returned type is variable, depending on `with_process` value) + if not success: + logger.error(f"conversion failed:\n{process.stdout}") + + +def run(args: List[str] = sys.argv): + if len(args) < 4: # noqa: PLR2004 + print(f"Usage: {args[0]} ") # noqa: T201 + print( # noqa: T201 + "\t\tpath to the video to encode." + "\t\tpath to the store the reencoded video." + "\t\tname of preset to use." + ) + return 1 + encode_video(Path(args[1]), Path(args[2]), args[3]) + return 0 + + +if __name__ == "__main__": + sys.exit(run()) diff --git a/pyproject.toml b/pyproject.toml index 1184b0b7..24d64bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,7 +113,7 @@ target-version = ['py38'] [tool.ruff] target-version = "py38" line-length = 88 -src = ["src"] +src = ["src", "contrib"] [tool.ruff.lint] select = [ @@ -235,7 +235,7 @@ exclude_lines = [ ] [tool.pyright] -include = ["src", "tests", "tasks.py"] +include = ["contrib", "src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**"] extraPaths = ["src"] pythonVersion = "3.8" From a7307d1e959c9af2119e622af87fdc765911a0b2 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 14:37:01 +0100 Subject: [PATCH 105/335] Add control on ffmpeg CPU threads consumption, with a default of 1 --- CHANGELOG.md | 1 + src/zimscraperlib/video/encoding.py | 41 ++++++++++--- tests/video/test_encoding.py | 94 +++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 tests/video/test_encoding.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bb2017f..d2fb1a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Suuport for Python 3.12, drop Python 3.7 #118 - Replace "iso-369" iso639-lang by "iso639-lang" library - Rework the VideoWebmLow preset for faster encoding and smaller file size (preset has been bumped to version 2) +- When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value) ## [3.2.0] - 2023-12-16 diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index 20509e7a..f66618af 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -6,11 +6,35 @@ import shutil import subprocess import tempfile +from typing import List, Optional from zimscraperlib import logger from zimscraperlib.logging import nicer_args_join +def _build_ffmpeg_args( + src_path: pathlib.Path, + tmp_path: pathlib.Path, + ffmpeg_args: List[str], + threads: Optional[int], +) -> List[str]: + if threads: + if "-threads" in ffmpeg_args: + raise AttributeError("Cannot set the number of threads, already set") + else: + ffmpeg_args += ["-threads", str(threads)] + args = [ + "/usr/bin/env", + "ffmpeg", + "-y", + "-i", + f"file:{src_path}", + *ffmpeg_args, + f"file:{tmp_path}", + ] + return args + + def reencode( src_path, dst_path, @@ -18,6 +42,7 @@ def reencode( delete_src=False, # noqa: FBT002 with_process=False, # noqa: FBT002 failsafe=True, # noqa: FBT002 + threads: Optional[int] = 1, ): """Runs ffmpeg with given ffmpeg_args @@ -25,6 +50,7 @@ def reencode( src_path - Path to source file dst_path - Path to destination file ffmpeg_args - A list of ffmpeg arguments + threads - Number of encoding threads used by ffmpeg delete_src - Delete source file after convertion with_process - Optionally return the output from ffmpeg (stderr and stdout) failsafe - Run in failsafe mode @@ -32,15 +58,12 @@ def reencode( with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = pathlib.Path(tmp_dir).joinpath(f"video.tmp{dst_path.suffix}") - args = [ - "/usr/bin/env", - "ffmpeg", - "-y", - "-i", - f"file:{src_path}", - *ffmpeg_args, - f"file:{tmp_path}", - ] + args = _build_ffmpeg_args( + src_path=src_path, + tmp_path=tmp_path, + ffmpeg_args=ffmpeg_args, + threads=threads, + ) logger.debug( f"Encode {src_path} -> {dst_path} video format = {dst_path.suffix}" ) diff --git a/tests/video/test_encoding.py b/tests/video/test_encoding.py new file mode 100644 index 00000000..292f660a --- /dev/null +++ b/tests/video/test_encoding.py @@ -0,0 +1,94 @@ +import re +from pathlib import Path +from typing import List, Optional + +import pytest + +from zimscraperlib.video.encoding import _build_ffmpeg_args + + +@pytest.mark.parametrize( + "src_path,tmp_path,ffmpeg_args,threads,expected", + [ + ( + Path("path1/file1.mp4"), + Path("path1/fileout.mp4"), + [ + "-codec:v", + "libx265", + ], + None, + [ + "/usr/bin/env", + "ffmpeg", + "-y", + "-i", + "file:path1/file1.mp4", + "-codec:v", + "libx265", + "file:path1/fileout.mp4", + ], + ), + ( + Path("path2/file2.mp4"), + Path("path12/tmpfile.mp4"), + [ + "-b:v", + "300k", + ], + 1, + [ + "/usr/bin/env", + "ffmpeg", + "-y", + "-i", + "file:path2/file2.mp4", + "-b:v", + "300k", + "-threads", + "1", + "file:path12/tmpfile.mp4", + ], + ), + ( + Path("path2/file2.mp4"), + Path("path12/tmpfile.mp4"), + [ + "-b:v", + "300k", + "-threads", + "1", + ], + 1, + None, + ), + ], +) +def test_build_ffmpeg_args( + src_path: Path, + tmp_path: Path, + ffmpeg_args: List[str], + threads: Optional[int], + expected: Optional[List[str]], +): + if expected: + assert ( + _build_ffmpeg_args( + src_path=src_path, + tmp_path=tmp_path, + ffmpeg_args=ffmpeg_args, + threads=threads, + ) + == expected + ) + else: + with pytest.raises( + AttributeError, + match=re.escape("Cannot set the number of threads, already set"), + ): + _build_ffmpeg_args( + src_path=src_path, + tmp_path=tmp_path, + ffmpeg_args=ffmpeg_args, + threads=threads, + ) From 4214abbdc827fc2e39010766607afa1eb6adbcca Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 5 Feb 2024 09:11:28 +0100 Subject: [PATCH 106/335] Fix ffprobe test failing on Alpine --- tests/video/test_video.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/video/test_video.py b/tests/video/test_video.py index 5b151f2e..ef541112 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -130,7 +130,17 @@ def test_get_media_info(media_format, media, expected, test_files): with tempfile.TemporaryDirectory() as t: src = pathlib.Path(t).joinpath(media) shutil.copy2(test_files[media_format], src) - assert get_media_info(src) == expected + result = get_media_info(src) + assert result.keys() == expected.keys() + assert result["codecs"] == expected["codecs"] + assert result["duration"] == expected["duration"] + # for bitrate, we need to allow some variability, not all ffmpeg version are + # reporting the same values (e.g. Alpine Linux is reporting 3837275 instead of + # 3818365 for video.mp4) ; we allow 1% variability with following assertion + assert ( + abs(100.0 * (result["bitrate"] - expected["bitrate"]) / expected["bitrate"]) + < 1 + ) def test_preset_has_mime_and_ext(): From cb2bad14dfd55304cbfb44e6559691dc93103137 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 11:42:10 +0100 Subject: [PATCH 107/335] Move to python-magic for alpine compatibility python-magic is supporting Alpine Linux since 0.4.24 (hence the minimum version) while file-magic is still not supporting it. --- pyproject.toml | 2 +- src/zimscraperlib/filesystem.py | 6 +++++- tests/filesystem/test_filesystem.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 24d64bf0..2fec6ce0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "colorthief==0.2.1", "python-resize-image>=1.1.19,<1.2", "Babel>=2.9,<3.0", - "file-magic>=0.4.0,<0.5", + "python-magic>=0.4.24,<0.5", "libzim>=3.4.0,<4.0", "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done diff --git a/src/zimscraperlib/filesystem.py b/src/zimscraperlib/filesystem.py index 509675eb..7b22dfe3 100644 --- a/src/zimscraperlib/filesystem.py +++ b/src/zimscraperlib/filesystem.py @@ -31,7 +31,11 @@ def get_content_mimetype(content: bytes) -> str: """MIME Type of content retrieved from magic headers""" try: - detected_mime = magic.detect_from_content(content).mime_type + detected_mime = magic.from_buffer(content, mime=True) + if isinstance( + detected_mime, bytes + ): # pragma: no cover (old python-magic versions where returning bytes) + detected_mime = detected_mime.decode() except UnicodeDecodeError: return "application/octet-stream" return MIME_OVERRIDES.get(detected_mime, detected_mime) diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index 71daba63..08579c16 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -28,10 +28,10 @@ def test_content_mimetype_fallback(monkeypatch, undecodable_byte_stream): assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" # mock then so we keep coverage on systems where magic works - def raising_magic(*args): # noqa: ARG001 + def raising_magic(*args, **kwargs): # noqa: ARG001 raise UnicodeDecodeError("nocodec", b"", 0, 1, "noreason") - monkeypatch.setattr(magic, "detect_from_content", raising_magic) + monkeypatch.setattr(magic, "from_buffer", raising_magic) assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" From 28647a58faf351cb0ece361820bfb72554804b23 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 14:27:13 +0100 Subject: [PATCH 108/335] Instructions for Alpine Linux --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 1d8870e7..dcd71ae4 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,11 @@ sudo apt install libmagic1 wget ffmpeg \ libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle ``` +## Alpine +``` +apk add ffmpeg gifsicle libmagic wget libjpeg +``` + # Contribution This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing) From f3251af53d632f5327a9cb18de4ea8a7fe196736 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 08:45:58 +0100 Subject: [PATCH 109/335] Update CHANGELOG --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2fb1a8b..fa3bbf95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120 -- Suuport for Python 3.12, drop Python 3.7 #118 -- Replace "iso-369" iso639-lang by "iso639-lang" library +- Add support for Python 3.12, drop Python 3.7 support #118 +- Replace "iso-369" by "iso639-lang" library +- Replace "file-magic" by "python-magic" library for Alpine Linux support and better maintenance - Rework the VideoWebmLow preset for faster encoding and smaller file size (preset has been bumped to version 2) - When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value) From d229055fc3f5a13495153da1c073df8dfbfe9aa8 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 15:52:35 +0100 Subject: [PATCH 110/335] Minimal supported python-magic version is 0.4.3 indeed --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2fec6ce0..53ad3b1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "colorthief==0.2.1", "python-resize-image>=1.1.19,<1.2", "Babel>=2.9,<3.0", - "python-magic>=0.4.24,<0.5", + "python-magic>=0.4.3,<0.5", "libzim>=3.4.0,<4.0", "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done From 68d3e30806645a30272ce54c79356925f1fe4e5a Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 15:55:47 +0100 Subject: [PATCH 111/335] Explicitely mention i18n not working on Alpine --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dcd71ae4..e909352f 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ sudo apt install libmagic1 wget ffmpeg \ apk add ffmpeg gifsicle libmagic wget libjpeg ``` +**Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing. + # Contribution This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing) From 87c0f33b42bb247ed986a2bf3f418a6d2fa318e3 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 21:42:18 +0100 Subject: [PATCH 112/335] Add more admissible types for convert_image + corresponding tests --- src/zimscraperlib/image/convertion.py | 7 +++++-- src/zimscraperlib/image/transformation.py | 2 +- src/zimscraperlib/image/utils.py | 7 ++++--- tests/image/test_image.py | 24 +++++++++++++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/zimscraperlib/image/convertion.py b/src/zimscraperlib/image/convertion.py index 31674236..3d5e0486 100644 --- a/src/zimscraperlib/image/convertion.py +++ b/src/zimscraperlib/image/convertion.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +import io import pathlib -from typing import Optional +from typing import Union import PIL @@ -13,7 +14,9 @@ def convert_image( - src: pathlib.Path, dst: pathlib.Path, **params: Optional[dict] + src: Union[pathlib.Path, io.BytesIO], + dst: Union[pathlib.Path, io.BytesIO], + **params: str, ) -> None: """convert an image file from one format to another params: Image.save() parameters. Depends on dest format. diff --git a/src/zimscraperlib/image/transformation.py b/src/zimscraperlib/image/transformation.py index 4db56cc6..287b7efb 100644 --- a/src/zimscraperlib/image/transformation.py +++ b/src/zimscraperlib/image/transformation.py @@ -19,7 +19,7 @@ def resize_image( dst: Optional[Union[pathlib.Path, io.BytesIO]] = None, method: Optional[str] = "width", allow_upscaling: Optional[bool] = True, # noqa: FBT002 - **params: Optional[dict], + **params: str, ) -> None: """resize an image to requested dimensions diff --git a/src/zimscraperlib/image/utils.py b/src/zimscraperlib/image/utils.py index 2568492d..6427a298 100644 --- a/src/zimscraperlib/image/utils.py +++ b/src/zimscraperlib/image/utils.py @@ -1,17 +1,18 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 nu +import io import pathlib -from typing import Optional +from typing import Optional, Union from PIL import Image def save_image( src: Image, # pyright: ignore - dst: pathlib.Path, + dst: Union[pathlib.Path, io.BytesIO], fmt: Optional[str] = None, - **params: Optional[dict], + **params: str, ) -> None: """PIL.Image.save() wrapper setting default parameters""" args = {"JPEG": {"quality": 100}, "PNG": {}}.get(fmt, {}) # pyright: ignore diff --git a/tests/image/test_image.py b/tests/image/test_image.py index 28041737..0953a093 100644 --- a/tests/image/test_image.py +++ b/tests/image/test_image.py @@ -296,6 +296,30 @@ def test_change_image_format_defaults(png_image, tmp_path): assert dst_image.format == "WEBP" +def test_convert_io_src_dst(png_image: pathlib.Path): + src = io.BytesIO(png_image.read_bytes()) + dst = io.BytesIO() + convert_image(src, dst, fmt="PNG") + dst_image = Image.open(dst) + assert dst_image.format == "PNG" + + +def test_convert_io_src_path_dst(png_image: pathlib.Path, tmp_path: pathlib.Path): + src = io.BytesIO(png_image.read_bytes()) + dst = tmp_path / "test.png" + convert_image(src, dst, fmt="PNG") + dst_image = Image.open(dst) + assert dst_image.format == "PNG" + + +def test_convert_path_src_io_dst(png_image: pathlib.Path): + src = png_image + dst = io.BytesIO() + convert_image(src, dst, fmt="PNG") + dst_image = Image.open(dst) + assert dst_image.format == "PNG" + + @pytest.mark.parametrize( "fmt,exp_size", [("png", 128), ("jpg", 128)], From 44f318155c4f90306c2d9ed6e6e3f2c75fde9334 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 10:34:32 +0100 Subject: [PATCH 113/335] Fix StaticItem typing issue + remove reference to callback removed long ago --- src/zimscraperlib/zim/filesystem.py | 4 ++-- src/zimscraperlib/zim/items.py | 10 ++++------ tests/zim/test_zim_creator.py | 25 ++++++++----------------- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index e0f3d6a2..cb3fb9c3 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -46,8 +46,8 @@ def __init__( self, root: pathlib.Path, filepath: pathlib.Path, - ): # pyright: ignore - super().__init__(root=root, filepath=filepath) # pyright: ignore + ): + super().__init__(root=root, filepath=filepath) # first look inside the file's magic headers self.mimetype = get_file_mimetype(self.filepath) # most web-specific files are plain text. In this case, use extension diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index a3d20d74..45026dc9 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -9,7 +9,7 @@ import re import tempfile import urllib.parse -from typing import Dict, Union +from typing import Any import libzim.writer # pyright: ignore @@ -23,11 +23,9 @@ class Item(libzim.writer.Item): - """libzim.writer.Item returning props for path/title/mimetype plus a callback + """libzim.writer.Item returning props for path/title/mimetype""" - Calls your `callback` prop on deletion""" - - def __init__(self, **kwargs: Dict[str, Union[str, bool, bytes]]): + def __init__(self, **kwargs: Any): super().__init__() for k, v in kwargs.items(): setattr(self, k, v) @@ -106,7 +104,7 @@ def download_for_size(url, on_disk, tmp_dir=None): size, _ = stream_file(url.geturl(), fpath=fpath, byte_stream=stream) return fpath or stream, size - def __init__(self, url: str, **kwargs): + def __init__(self, url: str, **kwargs: Any): super().__init__(**kwargs) self.url = urllib.parse.urlparse(url) use_disk = getattr(self, "use_disk", False) diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 9db1b1ca..d9d8ef52 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -4,7 +4,6 @@ import base64 import datetime import io -import os import pathlib import random import shutil @@ -119,7 +118,7 @@ def test_noindexlanguage(tmp_path): creator = Creator(fpath, "welcome").config_dev_metadata(Language="bam") creator.config_indexing(False) with creator as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore + creator.add_item(StaticItem(path="welcome", content="hello")) creator.add_item_for("index", "Index", content="-", mimetype="text/html") reader = Archive(fpath) @@ -165,15 +164,11 @@ def test_add_item_for_delete_fail(tmp_path, png_image): # copy file to local path shutil.copyfile(png_image, local_path) - def remove_source(item): - os.remove(item.filepath) - with Creator(fpath, "welcome").config_dev_metadata() as creator: creator.add_item( StaticItem( - filepath=local_path, # pyright: ignore - path="index", # pyright: ignore - callback=remove_source, # pyright: ignore + filepath=local_path, + path="index", ), callback=(delete_callback, local_path), ) @@ -188,18 +183,18 @@ def test_compression(tmp_path): with Creator( tmp_path / "test.zim", "welcome", compression="zstd" ).config_dev_metadata() as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore + creator.add_item(StaticItem(path="welcome", content="hello")) with Creator( fpath, "welcome", compression=Compression.zstd # pyright: ignore ).config_dev_metadata() as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore + creator.add_item(StaticItem(path="welcome", content="hello")) def test_double_finish(tmp_path): fpath = tmp_path / "test.zim" with Creator(fpath, "welcome").config_dev_metadata() as creator: - creator.add_item(StaticItem(path="welcome", content="hello")) # pyright: ignore + creator.add_item(StaticItem(path="welcome", content="hello")) # ensure we can finish an already finished creator creator.finish() @@ -219,11 +214,7 @@ def test_sourcefile_removal(tmp_path, html_file): # copy html to folder src_path = pathlib.Path(tmpdir.name, "source.html") shutil.copyfile(html_file, src_path) - creator.add_item( - StaticItem( - filepath=src_path, path=src_path.name, ref=tmpdir # pyright: ignore - ) - ) + creator.add_item(StaticItem(filepath=src_path, path=src_path.name, ref=tmpdir)) del tmpdir assert not src_path.exists() @@ -241,7 +232,7 @@ def test_sourcefile_removal_std(tmp_path, html_file): StaticItem( filepath=paths[-1], path=paths[-1].name, - mimetype="text/html", # pyright: ignore + mimetype="text/html", ), callback=(delete_callback, paths[-1]), ) From 954c253d0f1933af144c9b9d83823cd83e7a736e Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 15:25:18 +0100 Subject: [PATCH 114/335] Add expected args to Item and subclasses init method --- src/zimscraperlib/zim/filesystem.py | 4 +- src/zimscraperlib/zim/items.py | 78 +++++++++++++++++++++-------- tests/zim/test_zim_creator.py | 2 + 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index cb3fb9c3..0a81a7ee 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -47,7 +47,9 @@ def __init__( root: pathlib.Path, filepath: pathlib.Path, ): - super().__init__(root=root, filepath=filepath) + super().__init__() + self.root = root + self.filepath = filepath # first look inside the file's magic headers self.mimetype = get_file_mimetype(self.filepath) # most web-specific files are plain text. In this case, use extension diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index 45026dc9..881f2d50 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -9,7 +9,7 @@ import re import tempfile import urllib.parse -from typing import Any +from typing import Any, Optional import libzim.writer # pyright: ignore @@ -25,8 +25,19 @@ class Item(libzim.writer.Item): """libzim.writer.Item returning props for path/title/mimetype""" - def __init__(self, **kwargs: Any): + def __init__( + self, + path: Optional[str] = None, + title: Optional[str] = None, + mimetype: Optional[str] = None, + hints: Optional[dict] = None, + **kwargs: Any, + ): super().__init__() + self.path = path + self.title = title + self.mimetype = mimetype + self.hints = hints for k, v in kwargs.items(): setattr(self, k, v) @@ -35,16 +46,16 @@ def should_index(self): return self.get_mimetype().startswith("text/html") def get_path(self) -> str: - return getattr(self, "path", "") + return self.path or "" def get_title(self) -> str: - return getattr(self, "title", "") + return self.title or "" def get_mimetype(self) -> str: - return getattr(self, "mimetype", "") + return self.mimetype or "" def get_hints(self) -> dict: - return getattr(self, "hints", {}) + return self.hints or {} class StaticItem(Item): @@ -55,19 +66,37 @@ class StaticItem(Item): more efficiently: now when the libzim destroys the CP, python will destroy the Item and we can be notified that we're effectively through with our content""" + def __init__( + self, + content: Optional[str] = None, + fileobj: Optional[io.IOBase] = None, + filepath: Optional[pathlib.Path] = None, + path: Optional[str] = None, + title: Optional[str] = None, + mimetype: Optional[str] = None, + hints: Optional[dict] = None, + **kwargs: Any, + ): + super().__init__( + path=path, title=title, mimetype=mimetype, hints=hints, **kwargs + ) + self.content = content + self.fileobj = fileobj + self.filepath = filepath + def get_contentprovider(self) -> libzim.writer.ContentProvider: # content was set manually - if getattr(self, "content", None) is not None: + if self.content is not None: return StringProvider(content=self.content, ref=self) # using a file-like object - if getattr(self, "fileobj", None): + if self.fileobj: return FileLikeProvider( fileobj=self.fileobj, ref=self, size=getattr(self, "size", None) ) # we had to download locally to get size - if getattr(self, "filepath", None): + if self.filepath: return FileProvider( filepath=self.filepath, ref=self, size=getattr(self, "size", None) ) @@ -104,10 +133,22 @@ def download_for_size(url, on_disk, tmp_dir=None): size, _ = stream_file(url.geturl(), fpath=fpath, byte_stream=stream) return fpath or stream, size - def __init__(self, url: str, **kwargs: Any): - super().__init__(**kwargs) + def __init__( + self, + url: str, + path: Optional[str] = None, + title: Optional[str] = None, + mimetype: Optional[str] = None, + hints: Optional[dict] = None, + *, + use_disk: bool = False, + **kwargs: Any, + ): + super().__init__( + path=path, title=title, mimetype=mimetype, hints=hints, **kwargs + ) self.url = urllib.parse.urlparse(url) - use_disk = getattr(self, "use_disk", False) + self.use_disk = use_disk # fetch headers to retrieve size and type try: @@ -136,7 +177,7 @@ def __init__(self, url: str, **kwargs: Any): except Exception: # we couldn't retrieve size so we have to download resource to target, self.size = self.download_for_size( - self.url, on_disk=use_disk, tmp_dir=getattr(self, "tmp_dir", None) + self.url, on_disk=self.use_disk, tmp_dir=getattr(self, "tmp_dir", None) ) # downloaded to disk and using a file path from now on if use_disk: @@ -146,16 +187,11 @@ def __init__(self, url: str, **kwargs: Any): self.fileobj = target def get_path(self) -> str: - return getattr(self, "path", re.sub(r"^/", "", self.url.path)) - - def get_title(self) -> str: - return getattr(self, "title", "") + return self.path or re.sub(r"^/", "", self.url.path) def get_mimetype(self) -> str: - return getattr( - self, - "mimetype", - self.headers.get("Content-Type", "application/octet-stream"), + return self.mimetype or self.headers.get( + "Content-Type", "application/octet-stream" ) def get_contentprovider(self): diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index d9d8ef52..a8770c4b 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -40,6 +40,8 @@ def get_contentprovider(self): class FileLikeProviderItem(StaticItem): def get_contentprovider(self): + if not self.fileobj: + raise AttributeError("fileobj cannot be None") return FileLikeProvider(self.fileobj) From 824504f33d3b13d1254e445d81b590b5b73c8e7e Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 16:58:34 +0100 Subject: [PATCH 115/335] Do not store useless properties on Items We might have many Items, some of them long-lived, so it is maybe better to not store many properties with a None value because they are unused and hence keep a lean memory footprint --- src/zimscraperlib/zim/items.py | 64 +++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index 881f2d50..5f4c6ed0 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -34,10 +34,14 @@ def __init__( **kwargs: Any, ): super().__init__() - self.path = path - self.title = title - self.mimetype = mimetype - self.hints = hints + if path: + kwargs["path"] = path + if title: + kwargs["title"] = title + if mimetype: + kwargs["mimetype"] = mimetype + if hints: + kwargs["hints"] = hints for k, v in kwargs.items(): setattr(self, k, v) @@ -46,16 +50,16 @@ def should_index(self): return self.get_mimetype().startswith("text/html") def get_path(self) -> str: - return self.path or "" + return getattr(self, "path", "") def get_title(self) -> str: - return self.title or "" + return getattr(self, "title", "") def get_mimetype(self) -> str: - return self.mimetype or "" + return getattr(self, "mimetype", "") def get_hints(self) -> dict: - return self.hints or {} + return getattr(self, "hints", {}) class StaticItem(Item): @@ -77,28 +81,34 @@ def __init__( hints: Optional[dict] = None, **kwargs: Any, ): + if content: + kwargs["content"] = content + if fileobj: + kwargs["fileobj"] = fileobj + if filepath: + kwargs["filepath"] = filepath super().__init__( path=path, title=title, mimetype=mimetype, hints=hints, **kwargs ) - self.content = content - self.fileobj = fileobj - self.filepath = filepath def get_contentprovider(self) -> libzim.writer.ContentProvider: # content was set manually - if self.content is not None: - return StringProvider(content=self.content, ref=self) + content = getattr(self, "content", None) + if content is not None: + return StringProvider(content=content, ref=self) # using a file-like object - if self.fileobj: + fileobj = getattr(self, "fileobj", None) + if fileobj: return FileLikeProvider( - fileobj=self.fileobj, ref=self, size=getattr(self, "size", None) + fileobj=fileobj, ref=self, size=getattr(self, "size", None) ) # we had to download locally to get size - if self.filepath: + filepath = getattr(self, "filepath", None) + if filepath: return FileProvider( - filepath=self.filepath, ref=self, size=getattr(self, "size", None) + filepath=filepath, ref=self, size=getattr(self, "size", None) ) raise NotImplementedError("No data to provide`") @@ -140,15 +150,16 @@ def __init__( title: Optional[str] = None, mimetype: Optional[str] = None, hints: Optional[dict] = None, - *, - use_disk: bool = False, + use_disk: Optional[bool] = None, **kwargs: Any, ): + if use_disk: + kwargs["use_disk"] = use_disk super().__init__( path=path, title=title, mimetype=mimetype, hints=hints, **kwargs ) self.url = urllib.parse.urlparse(url) - self.use_disk = use_disk + use_disk = getattr(self, "use_disk", False) # fetch headers to retrieve size and type try: @@ -177,7 +188,7 @@ def __init__( except Exception: # we couldn't retrieve size so we have to download resource to target, self.size = self.download_for_size( - self.url, on_disk=self.use_disk, tmp_dir=getattr(self, "tmp_dir", None) + self.url, on_disk=use_disk, tmp_dir=getattr(self, "tmp_dir", None) ) # downloaded to disk and using a file path from now on if use_disk: @@ -187,11 +198,16 @@ def __init__( self.fileobj = target def get_path(self) -> str: - return self.path or re.sub(r"^/", "", self.url.path) + return getattr(self, "path", re.sub(r"^/", "", self.url.path)) + + def get_title(self) -> str: + return getattr(self, "title", "") def get_mimetype(self) -> str: - return self.mimetype or self.headers.get( - "Content-Type", "application/octet-stream" + return getattr( + self, + "mimetype", + self.headers.get("Content-Type", "application/octet-stream"), ) def get_contentprovider(self): From 12841a5ce209adaad515e2fff04e8738964e96e9 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 17:17:36 +0100 Subject: [PATCH 116/335] Update CHANGELOG for last PR changes --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa3bbf95..719c3a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Rework the VideoWebmLow preset for faster encoding and smaller file size (preset has been bumped to version 2) - When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value) +## Fixed + +- Fixed type hints of `zimscraperlib.zim.Item` and subclasses, and `zimscraperlib.image.optimization:convert_image` + ## [3.2.0] - 2023-12-16 ### Added From 1a9b8f7f3d3490cebd3ec09a2c4fd40f6232253f Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 17:24:30 +0100 Subject: [PATCH 117/335] New `auto_metadata_check` parameter in `zimscraperlib.zim.creator.Creator` initializer --- CHANGELOG.md | 4 ++++ src/zimscraperlib/zim/creator.py | 19 ++++++++++++++----- tests/zim/test_zim_creator.py | 6 ++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 719c3a77..d4f8e00f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- New `auto_metadata_check` parameter in `zimscraperlib.zim.creator.Creator` initializer, allowing to disable metadata check at startup (assuming the user will validate them on its own) #119 + ### Changed - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120 diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 4e25d1de..a8880752 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -97,7 +97,12 @@ class Creator(libzim.writer.Creator): a segmentation fault at garbage collection (on exit mostly). Meaning you should exit right after an exception in your code (during zim creation) - Use workaround_nocancel=False to disable the workaround.""" + Use workaround_nocancel=False to disable the workaround. + + By default, all metadata are validated for compliance with openZIM guidelines and + conventions. Set auto_metadata_check=False to disable this validation (you can still + do checks manually with the validation methods or your own logic). + """ def __init__( self, @@ -106,6 +111,7 @@ def __init__( compression: Optional[str] = None, workaround_nocancel: Optional[bool] = True, # noqa: FBT002 ignore_duplicates: Optional[bool] = False, # noqa: FBT002 + auto_metadata_check: bool = True, # noqa: FBT001, FBT002 ): super().__init__(filename=filename) self._metadata = {} @@ -123,6 +129,7 @@ def __init__( self.workaround_nocancel = workaround_nocancel self.ignore_duplicates = ignore_duplicates + self.auto_metadata_check = auto_metadata_check def config_indexing( self, indexing: bool, language: Optional[str] = None # noqa: FBT001 @@ -141,9 +148,10 @@ def start(self): if not all(self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS): raise ValueError("Mandatory metadata are not all set.") - for name, value in self._metadata.items(): - if value: - self.validate_metadata(name, value) + if self.auto_metadata_check: + for name, value in self._metadata.items(): + if value: + self.validate_metadata(name, value) language = self._metadata.get("Language", "").split(",") if language[0] and not self.__indexing_configured: @@ -187,7 +195,8 @@ def add_metadata( content: Union[str, bytes, datetime.date, datetime.datetime, Iterable[str]], mimetype: str = "text/plain;charset=UTF-8", ): - self.validate_metadata(name, content) + if self.auto_metadata_check: + self.validate_metadata(name, content) if name == "Date" and isinstance(content, (datetime.date, datetime.datetime)): content = content.strftime("%Y-%m-%d").encode("UTF-8") if ( diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index a8770c4b..0dd79871 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -507,6 +507,12 @@ def test_check_metadata(tmp_path): Creator(tmp_path, "").config_dev_metadata(LongDescription="T" * 5000).start() +def test_relax_metadata(tmp_path): + Creator(tmp_path, "", auto_metadata_check=False).config_dev_metadata( + Description="T" * 90 + ).start() + + @pytest.mark.parametrize( "tags", [ From 8d6795e1d36bec0687cb214393f0f1a9e7f90b50 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 18:35:50 +0100 Subject: [PATCH 118/335] Rename auto_metadata_checks to disable_metadata_checks --- CHANGELOG.md | 2 +- src/zimscraperlib/zim/creator.py | 12 ++++++------ tests/zim/test_zim_creator.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f8e00f..ba6b749e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- New `auto_metadata_check` parameter in `zimscraperlib.zim.creator.Creator` initializer, allowing to disable metadata check at startup (assuming the user will validate them on its own) #119 +- New `disable_metadata_checks` parameter in `zimscraperlib.zim.creator.Creator` initializer, allowing to disable metadata check at startup (assuming the user will validate them on its own) #119 ### Changed diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index a8880752..37dc1714 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -100,8 +100,8 @@ class Creator(libzim.writer.Creator): Use workaround_nocancel=False to disable the workaround. By default, all metadata are validated for compliance with openZIM guidelines and - conventions. Set auto_metadata_check=False to disable this validation (you can still - do checks manually with the validation methods or your own logic). + conventions. Set disable_metadata_checks=False to disable this validation (you can + still do checks manually with the validation methods or your own logic). """ def __init__( @@ -111,7 +111,7 @@ def __init__( compression: Optional[str] = None, workaround_nocancel: Optional[bool] = True, # noqa: FBT002 ignore_duplicates: Optional[bool] = False, # noqa: FBT002 - auto_metadata_check: bool = True, # noqa: FBT001, FBT002 + disable_metadata_checks: bool = True, # noqa: FBT001, FBT002 ): super().__init__(filename=filename) self._metadata = {} @@ -129,7 +129,7 @@ def __init__( self.workaround_nocancel = workaround_nocancel self.ignore_duplicates = ignore_duplicates - self.auto_metadata_check = auto_metadata_check + self.disable_metadata_checks = disable_metadata_checks def config_indexing( self, indexing: bool, language: Optional[str] = None # noqa: FBT001 @@ -148,7 +148,7 @@ def start(self): if not all(self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS): raise ValueError("Mandatory metadata are not all set.") - if self.auto_metadata_check: + if self.disable_metadata_checks: for name, value in self._metadata.items(): if value: self.validate_metadata(name, value) @@ -195,7 +195,7 @@ def add_metadata( content: Union[str, bytes, datetime.date, datetime.datetime, Iterable[str]], mimetype: str = "text/plain;charset=UTF-8", ): - if self.auto_metadata_check: + if self.disable_metadata_checks: self.validate_metadata(name, content) if name == "Date" and isinstance(content, (datetime.date, datetime.datetime)): content = content.strftime("%Y-%m-%d").encode("UTF-8") diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 0dd79871..dc4d4741 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -508,7 +508,7 @@ def test_check_metadata(tmp_path): def test_relax_metadata(tmp_path): - Creator(tmp_path, "", auto_metadata_check=False).config_dev_metadata( + Creator(tmp_path, "", disable_metadata_checks=False).config_dev_metadata( Description="T" * 90 ).start() From 7e19b92cf3bfb1cd3c66ee92cbb9485c9471bebe Mon Sep 17 00:00:00 2001 From: benoit74 Date: Wed, 14 Feb 2024 08:08:38 +0100 Subject: [PATCH 119/335] Fix disable_metadata_checks behavior --- src/zimscraperlib/zim/creator.py | 8 ++++---- tests/zim/test_zim_creator.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 37dc1714..6d03ee3b 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -100,7 +100,7 @@ class Creator(libzim.writer.Creator): Use workaround_nocancel=False to disable the workaround. By default, all metadata are validated for compliance with openZIM guidelines and - conventions. Set disable_metadata_checks=False to disable this validation (you can + conventions. Set disable_metadata_checks=True to disable this validation (you can still do checks manually with the validation methods or your own logic). """ @@ -111,7 +111,7 @@ def __init__( compression: Optional[str] = None, workaround_nocancel: Optional[bool] = True, # noqa: FBT002 ignore_duplicates: Optional[bool] = False, # noqa: FBT002 - disable_metadata_checks: bool = True, # noqa: FBT001, FBT002 + disable_metadata_checks: bool = False, # noqa: FBT001, FBT002 ): super().__init__(filename=filename) self._metadata = {} @@ -148,7 +148,7 @@ def start(self): if not all(self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS): raise ValueError("Mandatory metadata are not all set.") - if self.disable_metadata_checks: + if not self.disable_metadata_checks: for name, value in self._metadata.items(): if value: self.validate_metadata(name, value) @@ -195,7 +195,7 @@ def add_metadata( content: Union[str, bytes, datetime.date, datetime.datetime, Iterable[str]], mimetype: str = "text/plain;charset=UTF-8", ): - if self.disable_metadata_checks: + if not self.disable_metadata_checks: self.validate_metadata(name, content) if name == "Date" and isinstance(content, (datetime.date, datetime.datetime)): content = content.strftime("%Y-%m-%d").encode("UTF-8") diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index dc4d4741..63e644dc 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -508,7 +508,7 @@ def test_check_metadata(tmp_path): def test_relax_metadata(tmp_path): - Creator(tmp_path, "", disable_metadata_checks=False).config_dev_metadata( + Creator(tmp_path, "", disable_metadata_checks=True).config_dev_metadata( Description="T" * 90 ).start() From d2895ee25e2bad0800c96e9071ca861b12a6392b Mon Sep 17 00:00:00 2001 From: benoit74 Date: Wed, 14 Feb 2024 08:37:33 +0100 Subject: [PATCH 120/335] libzim StringProvider support both bytes and str for content --- src/zimscraperlib/zim/items.py | 6 ++++-- src/zimscraperlib/zim/providers.py | 2 +- tests/zim/conftest.py | 20 ++++++++++++++++++++ tests/zim/test_zim_creator.py | 24 +++++++++++++++++++++++- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index 5f4c6ed0..0ecd654c 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -9,7 +9,7 @@ import re import tempfile import urllib.parse -from typing import Any, Optional +from typing import Any, Optional, Union import libzim.writer # pyright: ignore @@ -72,7 +72,7 @@ class StaticItem(Item): def __init__( self, - content: Optional[str] = None, + content: Optional[Union[str, bytes]] = None, fileobj: Optional[io.IOBase] = None, filepath: Optional[pathlib.Path] = None, path: Optional[str] = None, @@ -95,6 +95,8 @@ def get_contentprovider(self) -> libzim.writer.ContentProvider: # content was set manually content = getattr(self, "content", None) if content is not None: + if not isinstance(content, (str, bytes)): + raise AttributeError(f"Unexpected type for content: {type(content)}") return StringProvider(content=content, ref=self) # using a file-like object diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index a698b4f5..3180b247 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -31,7 +31,7 @@ def __init__( class StringProvider(libzim.writer.StringProvider): - def __init__(self, content: str, ref: Optional[object] = None): + def __init__(self, content: Union[str, bytes], ref: Optional[object] = None): super().__init__(content) self.ref = ref diff --git a/tests/zim/conftest.py b/tests/zim/conftest.py index f5a7f59b..1efb5129 100644 --- a/tests/zim/conftest.py +++ b/tests/zim/conftest.py @@ -24,6 +24,26 @@ def html_str(): """ +@pytest.fixture(scope="function") +def html_str_cn(): + """sample HTML content with chinese characters""" + return """ + +