diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 84a0ea46..459db219 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -1,6 +1,6 @@ on: schedule: - - cron: '*/30 * * * *' + - cron: '17 * * * *' push: branches: - 'main' @@ -53,9 +53,31 @@ jobs: - name: Debug index.html if pull request if: github.event_name == 'pull_request' run: | - curl -Lo index.html-public https://github.com/m-aciek/pydocs-translation-dashboard/raw/refs/heads/gh-pages/index.html + curl -Lo index.html-public https://github.com/python-docs-translations/dashboard/raw/refs/heads/gh-pages/index.html diff --color=always -u index.html-public index.html || : cat index.html + - if: github.event_name == 'pull_request' + run: uv run generate_metadata.py https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/index.json # generates "metadata.html" + - if: github.event_name != 'pull_request' + run: uv run generate_metadata.py https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/${{ github.ref_name }}/index.json # generates "metadata.html" + - run: cp metadata.html warnings* build + - name: Deploy metadata view 🚀 + if: github.event_name != 'pull_request' + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: build + clean: false + git-config-name: github-actions[bot] + git-config-email: 41898282+github-actions[bot]@users.noreply.github.com + - name: Deploy metadata view to subdirectory if pull request 🚀 + if: github.event_name == 'pull_request' + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: build + target-folder: ${{ github.ref_name }} + clean: false + git-config-name: github-actions[bot] + git-config-email: 41898282+github-actions[bot]@users.noreply.github.com - uses: actions/upload-artifact@v4 with: name: build diff --git a/.gitignore b/.gitignore index dcaf7169..834e2798 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ index.html +metadata.html +warnings-*.txt diff --git a/build_warnings.py b/build_warnings.py new file mode 100644 index 00000000..95faa092 --- /dev/null +++ b/build_warnings.py @@ -0,0 +1,37 @@ +from pathlib import Path +from re import findall +from shutil import copyfile + +import sphinx.cmd.build + + +def number(clones_dir: str, repo: str, language_code: str) -> int: + language_part, *locale = language_code.split('-') + if locale: + lang_with_locale = f'{language_part}_{locale[0].upper()}' + else: + lang_with_locale = language_part + locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES') + locale_dir.mkdir(parents=True) + for po_file in Path(clones_dir, repo).rglob('*.po'): + relative_path = po_file.relative_to(Path(clones_dir, repo)) + target_file = locale_dir / relative_path + target_file.parent.mkdir(parents=True, exist_ok=True) + copyfile(po_file, target_file) + sphinx.cmd.build.main( + ( + '--builder', + 'html', + '--jobs', + 'auto', + '--define', + f'language={language_code}', + '--verbose', + '--warning-file', + warning_file := f'{clones_dir}/warnings-{language_code}.txt', + f'{clones_dir}/cpython/Doc', # sourcedir + f'./sphinxbuild/{language_code}', # outputdir + ) + ) + copyfile(warning_file, f'warnings-{language_code}.txt') + return len(findall('ERROR|WARNING', Path(warning_file).read_text())) diff --git a/completion.py b/completion.py index ec28cc72..c98505a0 100644 --- a/completion.py +++ b/completion.py @@ -22,9 +22,12 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]: def get_completion( clones_dir: str, repo: str -) -> tuple[float, 'TranslatorsData', str, float]: +) -> tuple[float, 'TranslatorsData', str | None, float]: clone_path = Path(clones_dir, repo) - for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']: + for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + [ + 'master', + 'main', + ]: try: clone_repo = git.Repo.clone_from( f'https://github.com/{repo}.git', clone_path, branch=branch @@ -32,6 +35,7 @@ def get_completion( except git.GitCommandError: print(f'failed to clone {repo} {branch}') translators_data = TranslatorsData(0, False) + branch = '' continue else: translators_number = translators.get_number(clone_path) @@ -66,7 +70,7 @@ def get_completion( change = completion - month_ago_completion - return completion, translators_data, branch, change + return completion, translators_data, branch or None, change @dataclass(frozen=True) diff --git a/generate.py b/generate.py index 48b98919..c77741e6 100644 --- a/generate.py +++ b/generate.py @@ -23,11 +23,11 @@ from jinja2 import Template from urllib3 import PoolManager -import contribute import build_status +import contribute from visitors import get_number_of_visitors from completion import branches_from_devguide, get_completion, TranslatorsData -from repositories import get_languages_and_repos, Language +from repositories import Language, get_languages_and_repos generation_time = datetime.now(timezone.utc) diff --git a/generate_metadata.py b/generate_metadata.py new file mode 100644 index 00000000..a7d90b70 --- /dev/null +++ b/generate_metadata.py @@ -0,0 +1,112 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "gitpython", +# "potodo", +# "jinja2", +# "sphinx", +# "python-docs-theme", +# "dacite", +# "sphinx-lint", +# ] +# /// +import concurrent.futures +import itertools +import logging +import subprocess +from collections.abc import Iterator, Sequence +from datetime import datetime, timezone +from json import loads +from pathlib import Path +from sys import argv +from tempfile import TemporaryDirectory + +import dacite +import git +from git import Repo +from jinja2 import Template +from urllib3 import request + +import build_warnings +import sphinx_lint +from completion import branches_from_devguide +from generate import LanguageProjectData +from repositories import Language + +generation_time = datetime.now(timezone.utc) + + +def get_projects_metadata( + completion_progress: Sequence[LanguageProjectData], +) -> Iterator[tuple[int, int]]: + with TemporaryDirectory() as clones_dir: + Repo.clone_from( + 'https://github.com/python/devguide.git', + devguide_dir := Path(clones_dir, 'devguide'), + depth=1, + ) + latest_branch = branches_from_devguide(devguide_dir)[0] + Repo.clone_from( + 'https://github.com/python/cpython.git', + cpython_dir := Path(clones_dir, 'cpython'), + depth=1, + branch=latest_branch, + ) + subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True) + subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True) + with concurrent.futures.ProcessPoolExecutor() as executor: + return executor.map( + get_metadata, + *zip( + *map(get_language_repo_branch_and_completion, completion_progress) + ), + itertools.repeat(clones_dir), + ) + + +def get_metadata( + language: Language, + repo: str | None, + branch: str | None, + completion: float, + clones_dir: str, +) -> tuple[int, int]: + if repo: + clone_path = Path(clones_dir, repo) + git.Repo.clone_from(f'https://github.com/{repo}.git', clone_path, branch=branch) + return ( + repo + and completion + and ( + build_warnings.number(clones_dir, repo, language.code), + sphinx_lint.store_and_count_failures(clones_dir, repo, language.code), + ) + ) or (0, 0) + + +def get_language_repo_branch_and_completion( + project: LanguageProjectData, +) -> tuple[Language, str | None, str | None, float]: + return project.language, project.repository, project.branch, project.completion + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + logging.info(f'starting at {generation_time}') + template = Template(Path('metadata.html.jinja').read_text()) + if (index_path := Path('index.json')).exists(): + index_json = loads(Path('index.json').read_text()) + else: + index_json = request('GET', argv[1]).json() + + completion_progress = [ + dacite.from_dict(LanguageProjectData, project) for project in index_json + ] + + output = template.render( + metadata=zip(completion_progress, get_projects_metadata(completion_progress)), + generation_time=generation_time, + duration=(datetime.now(timezone.utc) - generation_time).seconds, + ) + + Path('metadata.html').write_text(output) diff --git a/metadata.html.jinja b/metadata.html.jinja new file mode 100644 index 00000000..f77b83fd --- /dev/null +++ b/metadata.html.jinja @@ -0,0 +1,45 @@ + + + Python Docs Translation Dashboard + + + + + +

Python Docs Translation Dashboard

+ + + + + + + + + + + + +{% for project, metadata in metadata | sort(attribute='0.completion,0.translators.number') | reverse %} + + + + + + + +{% endfor %} + +
languagebranchbuild warnings*lint failures30 days change
{{ project.language.name }} ({{ project.language.code }}){{ project.branch }} + {% if project.completion %}{{ metadata[0] }}{% else %}{{ metadata[0] }}{% endif %} + + {% if project.completion %}{{ metadata[1] }}{% else %}{{ metadata[1] }}{% endif %} + + +{{ "{:.2f}".format(project.change) }}% +
+

* number of Sphinx build process warnings

+

For more information about translations, see the Python Developer’s Guide.

+

Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).

+ + diff --git a/sphinx_lint.py b/sphinx_lint.py new file mode 100644 index 00000000..ffe45626 --- /dev/null +++ b/sphinx_lint.py @@ -0,0 +1,18 @@ +from collections.abc import Iterator +from itertools import chain +from pathlib import Path + +from sphinxlint import check_file, checkers + + +def store_and_count_failures(clones_dir: str, repo: str, language_code: str) -> int: + failed_checks = list(chain.from_iterable(yield_failures(clones_dir, repo))) + filepath = Path(f'warnings-lint-{language_code}.txt') + filepath.write_text('\n'.join([str(c) for c in failed_checks])) + return len(failed_checks) + + +def yield_failures(clones_dir: str, repo: str) -> Iterator[str]: + enabled_checkers = [c for c in checkers.all_checkers.values() if c.enabled] + for path in Path(clones_dir, repo).rglob('*.po'): + yield check_file(path.as_posix(), enabled_checkers) diff --git a/style.css b/style.css index 3a67a64a..10284701 100644 --- a/style.css +++ b/style.css @@ -3,7 +3,6 @@ body { } table { border-collapse: collapse; - width: 100%; } th, td { border: 1px solid #ddd; @@ -39,13 +38,25 @@ th { .progress-bar.low + .progress-bar-outer-label { display: inline-block; } -td[data-label="visitors"], td[data-label="translators"] { +td[data-label="visitors"], td[data-label="translators"], td[data-label="warnings"], td[data-label="branch"], td[data-label="lint"] { text-align: right; } td[data-label="completion"] { width: 100%; line-height: 0; } +.switchpages{ + position:absolute; + top:10px; + right: 10px; + } + +@media screen and (max-width: 675px) { + .switchpages{ + all: unset; + } +} + @media screen and (max-width: 600px) { table, thead, tbody, th, td, tr { display: block; diff --git a/template.html.jinja b/template.html.jinja index 60129a32..c92dc472 100644 --- a/template.html.jinja +++ b/template.html.jinja @@ -4,9 +4,13 @@ +

Python Docs Translation Dashboard

+ @@ -32,7 +36,7 @@ @@ -52,8 +56,8 @@ {% endfor %}
{% if project.built %} - + {{ '{:,}'.format(project.visitors) }} {% else %} @@ -40,7 +44,7 @@ {% endif %} - {% if project.translators.link %}{% endif %} + {% if project.translators.link %}{% endif %} {{ project.translators.number }} {% if project.translators.link %}{% endif %}
-

* sum of daily unique visitors since 8 June 2024

-

For more information about translations, see the Python Developer’s Guide.

+

* sum of daily unique visitors since 8 June 2024

+

For more information about translations, see the Python Developer’s Guide.

Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).