Metadata

.github/workflows/update.yml

@@ -1,6 +1,6 @@
 on:
   schedule:
-    - cron: '*/30 * * * *'
+    - cron: '17 * * * *'
   push:
     branches:
       - 'main'
@@ -53,9 +53,31 @@ jobs:
       - name: Debug index.html if pull request
         if: github.event_name == 'pull_request'
         run: |
-          curl -Lo index.html-public https://github.com/m-aciek/pydocs-translation-dashboard/raw/refs/heads/gh-pages/index.html
+          curl -Lo index.html-public https://github.com/python-docs-translations/dashboard/raw/refs/heads/gh-pages/index.html
           diff --color=always -u index.html-public index.html || :
           cat index.html
+      - if: github.event_name == 'pull_request'
+        run: uv run generate_metadata.py https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/index.json  # generates "metadata.html"
+      - if: github.event_name != 'pull_request'
+        run: uv run generate_metadata.py https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/${{ github.ref_name }}/index.json  # generates "metadata.html"
+      - run: cp metadata.html warnings* build
+      - name: Deploy metadata view 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build
+          clean: false
+          git-config-name: github-actions[bot]
+          git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
+      - name: Deploy metadata view to subdirectory if pull request 🚀
+        if: github.event_name == 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build
+          target-folder: ${{ github.ref_name }}
+          clean: false
+          git-config-name: github-actions[bot]
+          git-config-email: 41898282+github-actions[bot]@users.noreply.github.com
       - uses: actions/upload-artifact@v4
         with:
           name: build

.gitignore

@@ -1 +1,3 @@
 index.html
+metadata.html
+warnings-*.txt

build_warnings.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from re import findall
+from shutil import copyfile
+
+import sphinx.cmd.build
+
+
+def number(clones_dir: str, repo: str, language_code: str) -> int:
+    language_part, *locale = language_code.split('-')
+    if locale:
+        lang_with_locale = f'{language_part}_{locale[0].upper()}'
+    else:
+        lang_with_locale = language_part
+    locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES')
+    locale_dir.mkdir(parents=True)
+    for po_file in Path(clones_dir, repo).rglob('*.po'):
+        relative_path = po_file.relative_to(Path(clones_dir, repo))
+        target_file = locale_dir / relative_path
+        target_file.parent.mkdir(parents=True, exist_ok=True)
+        copyfile(po_file, target_file)
+    sphinx.cmd.build.main(
+        (
+            '--builder',
+            'html',
+            '--jobs',
+            'auto',
+            '--define',
+            f'language={language_code}',
+            '--verbose',
+            '--warning-file',
+            warning_file := f'{clones_dir}/warnings-{language_code}.txt',
+            f'{clones_dir}/cpython/Doc',  # sourcedir
+            f'./sphinxbuild/{language_code}',  # outputdir
+        )
+    )
+    copyfile(warning_file, f'warnings-{language_code}.txt')
+    return len(findall('ERROR|WARNING', Path(warning_file).read_text()))

completion.py

@@ -22,16 +22,20 @@ def branches_from_devguide(devguide_dir: Path) -> list[str]:
 
 def get_completion(
     clones_dir: str, repo: str
-) -> tuple[float, 'TranslatorsData', str, float]:
+) -> tuple[float, 'TranslatorsData', str | None, float]:
     clone_path = Path(clones_dir, repo)
-    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + ['master']:
+    for branch in branches_from_devguide(Path(clones_dir, 'devguide')) + [
+        'master',
+        'main',
+    ]:
         try:
             clone_repo = git.Repo.clone_from(
                 f'https://github.com/{repo}.git', clone_path, branch=branch
             )
         except git.GitCommandError:
             print(f'failed to clone {repo} {branch}')
             translators_data = TranslatorsData(0, False)
+            branch = ''
             continue
         else:
             translators_number = translators.get_number(clone_path)
@@ -66,7 +70,7 @@ def get_completion(
 
     change = completion - month_ago_completion
 
-    return completion, translators_data, branch, change
+    return completion, translators_data, branch or None, change
 
 
 @dataclass(frozen=True)

generate.py

@@ -23,11 +23,11 @@
 from jinja2 import Template
 from urllib3 import PoolManager
 
-import contribute
 import build_status
+import contribute
 from visitors import get_number_of_visitors
 from completion import branches_from_devguide, get_completion, TranslatorsData
-from repositories import get_languages_and_repos, Language
+from repositories import Language, get_languages_and_repos
 
 generation_time = datetime.now(timezone.utc)
 

generate_metadata.py

@@ -0,0 +1,112 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+#     "gitpython",
+#     "potodo",
+#     "jinja2",
+#     "sphinx",
+#     "python-docs-theme",
+#     "dacite",
+#     "sphinx-lint",
+# ]
+# ///
+import concurrent.futures
+import itertools
+import logging
+import subprocess
+from collections.abc import Iterator, Sequence
+from datetime import datetime, timezone
+from json import loads
+from pathlib import Path
+from sys import argv
+from tempfile import TemporaryDirectory
+
+import dacite
+import git
+from git import Repo
+from jinja2 import Template
+from urllib3 import request
+
+import build_warnings
+import sphinx_lint
+from completion import branches_from_devguide
+from generate import LanguageProjectData
+from repositories import Language
+
+generation_time = datetime.now(timezone.utc)
+
+
+def get_projects_metadata(
+    completion_progress: Sequence[LanguageProjectData],
+) -> Iterator[tuple[int, int]]:
+    with TemporaryDirectory() as clones_dir:
+        Repo.clone_from(
+            'https://github.com/python/devguide.git',
+            devguide_dir := Path(clones_dir, 'devguide'),
+            depth=1,
+        )
+        latest_branch = branches_from_devguide(devguide_dir)[0]
+        Repo.clone_from(
+            'https://github.com/python/cpython.git',
+            cpython_dir := Path(clones_dir, 'cpython'),
+            depth=1,
+            branch=latest_branch,
+        )
+        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
+        subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
+        with concurrent.futures.ProcessPoolExecutor() as executor:
+            return executor.map(
+                get_metadata,
+                *zip(
+                    *map(get_language_repo_branch_and_completion, completion_progress)
+                ),
+                itertools.repeat(clones_dir),
+            )
+
+
+def get_metadata(
+    language: Language,
+    repo: str | None,
+    branch: str | None,
+    completion: float,
+    clones_dir: str,
+) -> tuple[int, int]:
+    if repo:
+        clone_path = Path(clones_dir, repo)
+        git.Repo.clone_from(f'https://github.com/{repo}.git', clone_path, branch=branch)
+    return (
+        repo
+        and completion
+        and (
+            build_warnings.number(clones_dir, repo, language.code),
+            sphinx_lint.store_and_count_failures(clones_dir, repo, language.code),
+        )
+    ) or (0, 0)
+
+
+def get_language_repo_branch_and_completion(
+    project: LanguageProjectData,
+) -> tuple[Language, str | None, str | None, float]:
+    return project.language, project.repository, project.branch, project.completion
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    logging.info(f'starting at {generation_time}')
+    template = Template(Path('metadata.html.jinja').read_text())
+    if (index_path := Path('index.json')).exists():
+        index_json = loads(Path('index.json').read_text())
+    else:
+        index_json = request('GET', argv[1]).json()
+
+    completion_progress = [
+        dacite.from_dict(LanguageProjectData, project) for project in index_json
+    ]
+
+    output = template.render(
+        metadata=zip(completion_progress, get_projects_metadata(completion_progress)),
+        generation_time=generation_time,
+        duration=(datetime.now(timezone.utc) - generation_time).seconds,
+    )
+
+    Path('metadata.html').write_text(output)

metadata.html.jinja

@@ -0,0 +1,45 @@
+<html lang="en">
+<head>
+  <title>Python Docs Translation Dashboard</title>
+  <link rel="stylesheet" href="style.css">
+  <meta charset="UTF-8">
+  <base target="_blank">
+</head>
+<body>
+<h1>Python Docs Translation Dashboard</h1>
+<nav class="switchpages">
+<a href="index.html" target="_self">main</a> | meta
+</nav>
+<table>
+<thead>
+<tr>
+  <th>language</th>
+  <th>branch</th>
+  <th>build warnings*</th>
+  <th>lint failures</th>
+  <th>30 days change</th>
+</tr>
+</thead>
+<tbody>
+{% for project, metadata in metadata | sort(attribute='0.completion,0.translators.number') | reverse %}
+<tr>
+  <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
+  <td data-label="branch">{{ project.branch }}</td>
+  <td data-label="warnings">
+    {% if project.completion %}<a href="warnings-{{ project.language.code }}.txt">{{ metadata[0] }}</a>{% else %}{{ metadata[0] }}{% endif %}
+  </td>
+  <td data-label="lint">
+    {% if project.completion %}<a href="warnings-lint-{{ project.language.code }}.txt">{{ metadata[1] }}</a>{% else %}{{ metadata[1] }}{% endif %}
+  </td>
+  <td data-label="change">
+    +{{ "{:.2f}".format(project.change) }}%
+  </td>
+</tr>
+{% endfor %}
+</tbody>
+</table>
+<p>* number of Sphinx build process warnings</p>
+<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/">Python Developer’s Guide</a>.</p>
+<p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
+</body>
+</html>

sphinx_lint.py

@@ -0,0 +1,18 @@
+from collections.abc import Iterator
+from itertools import chain
+from pathlib import Path
+
+from sphinxlint import check_file, checkers
+
+
+def store_and_count_failures(clones_dir: str, repo: str, language_code: str) -> int:
+    failed_checks = list(chain.from_iterable(yield_failures(clones_dir, repo)))
+    filepath = Path(f'warnings-lint-{language_code}.txt')
+    filepath.write_text('\n'.join([str(c) for c in failed_checks]))
+    return len(failed_checks)
+
+
+def yield_failures(clones_dir: str, repo: str) -> Iterator[str]:
+    enabled_checkers = [c for c in checkers.all_checkers.values() if c.enabled]
+    for path in Path(clones_dir, repo).rglob('*.po'):
+        yield check_file(path.as_posix(), enabled_checkers)

style.css

@@ -3,7 +3,6 @@ body {
 }
 table {
   border-collapse: collapse;
-  width: 100%;
 }
 th, td {
   border: 1px solid #ddd;
@@ -39,13 +38,25 @@ th {
 .progress-bar.low + .progress-bar-outer-label {
   display: inline-block;
 }
-td[data-label="visitors"], td[data-label="translators"] {
+td[data-label="visitors"], td[data-label="translators"], td[data-label="warnings"], td[data-label="branch"], td[data-label="lint"] {
   text-align: right;
 }
 td[data-label="completion"] {
   width: 100%;
   line-height: 0;
 }
+.switchpages{
+    position:absolute;
+    top:10px;
+    right: 10px;
+ }
+
+@media screen and (max-width: 675px) {
+  .switchpages{
+    all: unset;
+ }
+}
+
 @media screen and (max-width: 600px) {
   table, thead, tbody, th, td, tr {
     display: block;

template.html.jinja

@@ -4,9 +4,13 @@
   <link rel="stylesheet" href="style.css">
   <meta charset="UTF-8">
   <meta name="description" content="Python Docs Translation Dashboard">
+  <base target="_blank">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
+<nav class="switchpages">
+main | <a href="metadata.html" target="_self">meta</a>
+</nav>
 <table>
 <thead>
 <tr>
@@ -32,15 +36,15 @@
   </td>
   <td data-label="visitors">
     {% if project.built %}
-      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))" target="_blank">
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))">
         {{ '{:,}'.format(project.visitors) }}
       </a>
     {% else %}
       {{ '{:,}'.format(project.visitors) }}
     {% endif %}
   </td>
   <td data-label="translators">
-    {% if project.translators.link %}<a href="{{ project.translators.link }}" target="_blank">{% endif %}
+    {% if project.translators.link %}<a href="{{ project.translators.link }}">{% endif %}
     {{ project.translators.number }}
     {% if project.translators.link %}</a>{% endif %}
   </td>
@@ -52,8 +56,8 @@
 {% endfor %}
 </tbody>
 </table>
-<p>* sum of <a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">daily unique visitors</a> since 8 June 2024</p>
-<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/" target="_blank">Python Developer’s Guide</a>.</p>
+<p>* sum of <a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies">daily unique visitors</a> since 8 June 2024</p>
+<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/">Python Developer’s Guide</a>.</p>
 <p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
 </body>
 <script>

visitors.py

@@ -14,7 +14,7 @@ def get_number_of_visitors(language: str, http: PoolManager) -> int:
     response = http.request(
         'GET',
         f'https://plausible.io/docs.python.org/export?{params}',
-        retries=Retry(status_forcelist=(500,502)),
+        retries=Retry(status_forcelist=(500, 502)),
     )
     info(f'visitors {response.status=} ({language=})')
     with (