From 9fab60db68fe9bde9d2c7e05c541a410e0f44448 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Fri, 17 Mar 2023 01:24:03 +0100 Subject: [PATCH 1/3] Count translators with Python and polib, remove aliases and look for new ones --- .github/workflows/update-lint-and-build.yml | 2 +- README.md | 2 +- manage_translation.py | 44 ++++++++++++++++----- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/update-lint-and-build.yml b/.github/workflows/update-lint-and-build.yml index e5915c7afa..03b066e3b9 100644 --- a/.github/workflows/update-lint-and-build.yml +++ b/.github/workflows/update-lint-and-build.yml @@ -19,7 +19,7 @@ jobs: with: python-version: 3 - run: sudo apt-get install -y gettext - - run: pip install requests cogapp + - run: pip install requests cogapp polib - uses: actions/checkout@master with: ref: ${{ matrix.version }} diff --git a/README.md b/README.md index 89f91807d0..bf503fc69e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ f'''![build](https://github.com/python/python-docs-pl/workflows/.github/workflow ![build](https://github.com/python/python-docs-pl/workflows/.github/workflows/update-and-build.yml/badge.svg) ![48.54% przełącznika języków](https://img.shields.io/badge/przełącznik_języków-48.54%25-0.svg) ![postęp tłumaczenia całości dokumentacji](https://img.shields.io/badge/całość-3.51%25-0.svg) -![19 tłumaczy](https://img.shields.io/badge/tłumaczy-19-0.svg) +![18 tłumaczy](https://img.shields.io/badge/tłumaczy-18-0.svg) Jeśli znalazłeś(-aś) błąd lub masz sugestię, diff --git a/manage_translation.py b/manage_translation.py index 11ad8a2a27..7c5021e30b 100755 --- a/manage_translation.py +++ b/manage_translation.py @@ -12,15 +12,19 @@ # * regenerate_tx_config: recreate configuration for all resources. from argparse import ArgumentParser -from collections import Counter import os from dataclasses import dataclass +from difflib import SequenceMatcher +from itertools import combinations +from pathlib import Path from re import match -from subprocess import call, run +from subprocess import call import sys from typing import Self, Callable from urllib.parse import urlparse, parse_qs +from polib import pofile + LANGUAGE = 'pl' @@ -168,14 +172,34 @@ def progress_from_resources(resources: list[ResourceLanguageStatistics], filter_ def get_number_of_translators(): - process = run( - ['grep', '-ohP', r'(?<=^# )(.+)(?=, \d+$)', '-r', '.'], - capture_output=True, - text=True, - ) - translators = [match('(.*)( <.*>)?', t).group(1) for t in process.stdout.splitlines()] - unique_translators = Counter(translators).keys() - return len(unique_translators) + translators = _fetch_translators() + _remove_aliases(translators) + _check_for_new_aliases(translators) + return len(translators) + + +def _fetch_translators() -> set[str]: + translators = set() + for file in Path().rglob('*.po'): + header = pofile(os.fsdecode(file)).header.splitlines() + for translator_record in header[header.index('Translators:') + 1:]: + translator, _year = translator_record.split(', ') + translators.add(translator) + return translators + + +def _remove_aliases(translators: set[str]) -> None: + for alias, main in (("m_aciek ", "Maciej Olko "),): + translators.remove(alias) + assert main in translators + + +def _check_for_new_aliases(translators) -> None: + for pair in combinations(translators, 2): + if (ratio := SequenceMatcher(lambda x: x in '<>@', *pair).ratio()) > 0.64: + raise ValueError( + f"{pair} are similar ({ratio:.3f}). Please add them to aliases list or bump the limit." + ) def language_switcher(entry: ResourceLanguageStatistics) -> bool: From d81c01896ac39fa975f6260cd1d877b9464a6ba2 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 25 Mar 2023 16:54:51 +0100 Subject: [PATCH 2/3] Convert ValueError to UserWarning --- .github/workflows/update-lint-and-build.yml | 2 +- manage_translation.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-lint-and-build.yml b/.github/workflows/update-lint-and-build.yml index 03b066e3b9..6169b8847c 100644 --- a/.github/workflows/update-lint-and-build.yml +++ b/.github/workflows/update-lint-and-build.yml @@ -31,7 +31,7 @@ jobs: - run: ./manage_translation.py fetch env: TX_TOKEN: ${{ secrets.TX_TOKEN }} - - run: cog -rP README.md + - run: python -Werror -m cogapp -rP README.md env: TX_TOKEN: ${{ secrets.TX_TOKEN }} - run: git config --local user.email github-actions@github.com diff --git a/manage_translation.py b/manage_translation.py index 7c5021e30b..1edb2e26ad 100755 --- a/manage_translation.py +++ b/manage_translation.py @@ -22,6 +22,7 @@ import sys from typing import Self, Callable from urllib.parse import urlparse, parse_qs +from warnings import warn from polib import pofile @@ -197,7 +198,7 @@ def _remove_aliases(translators: set[str]) -> None: def _check_for_new_aliases(translators) -> None: for pair in combinations(translators, 2): if (ratio := SequenceMatcher(lambda x: x in '<>@', *pair).ratio()) > 0.64: - raise ValueError( + warn( f"{pair} are similar ({ratio:.3f}). Please add them to aliases list or bump the limit." ) From 44298318e04c37c1d7e81daa9467086659d38ee4 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Apr 2023 00:26:33 +0200 Subject: [PATCH 3/3] Remove superfluous fsdecode() call --- manage_translation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage_translation.py b/manage_translation.py index 1edb2e26ad..b512606e27 100755 --- a/manage_translation.py +++ b/manage_translation.py @@ -182,7 +182,7 @@ def get_number_of_translators(): def _fetch_translators() -> set[str]: translators = set() for file in Path().rglob('*.po'): - header = pofile(os.fsdecode(file)).header.splitlines() + header = pofile(file).header.splitlines() for translator_record in header[header.index('Translators:') + 1:]: translator, _year = translator_record.split(', ') translators.add(translator)