Use fewer queries for translation progress stats on dashboards (mozil…

…la#3546)

pontoon/administration/templates/admin.html

@@ -27,7 +27,7 @@
       {% set main_link = url('pontoon.admin.project', project.slug) %}
       {% set chart_link = url('pontoon.admin.project', project.slug) %}
       {% set latest_activity = project.get_latest_activity() %}
-      {% set chart = project.get_chart() %}
+      {% set chart = project_stats.get(project.id, {'total': 0}) %}
 
       {{ ProjectList.item(project, main_link, chart_link, latest_activity, chart) }}
     {% endfor %}

pontoon/administration/views.py

@@ -49,7 +49,15 @@ def admin(request):
         .order_by("name")
     )
 
-    return render(request, "admin.html", {"admin": True, "projects": projects})
+    return render(
+        request,
+        "admin.html",
+        {
+            "admin": True,
+            "projects": projects,
+            "project_stats": projects.stats_data(),
+        },
+    )
 
 
 @login_required(redirect_field_name="", login_url="/403")

pontoon/base/aggregated_stats.py

@@ -1,5 +1,3 @@
-import math
-
 from functools import cached_property
 
 
@@ -42,7 +40,7 @@ def unreviewed_strings(self) -> int:
         return self._stats["unreviewed"]
 
     @property
-    def missing_strings(self):
+    def missing_strings(self) -> int:
         return (
             self.total_strings
             - self.approved_strings
@@ -51,42 +49,42 @@ def missing_strings(self):
             - self.strings_with_warnings
         )
 
+    @property
+    def complete(self) -> bool:
+        return (
+            self.total_strings
+            == self.approved_strings
+            + self.pretranslated_strings
+            + self.strings_with_warnings
+        )
+
 
-def get_completed_percent(obj):
-    if not obj.total_strings:
-        return 0
-    completed_strings = (
-        obj.approved_strings + obj.pretranslated_strings + obj.strings_with_warnings
-    )
-    return completed_strings / obj.total_strings * 100
-
-
-def get_chart_dict(obj: "AggregatedStats"):
-    """Get chart data dictionary"""
-    if ts := obj.total_strings:
-        return {
-            "total": ts,
-            "approved": obj.approved_strings,
-            "pretranslated": obj.pretranslated_strings,
-            "errors": obj.strings_with_errors,
-            "warnings": obj.strings_with_warnings,
-            "unreviewed": obj.unreviewed_strings,
-            "approved_share": round(obj.approved_strings / ts * 100),
-            "pretranslated_share": round(obj.pretranslated_strings / ts * 100),
-            "errors_share": round(obj.strings_with_errors / ts * 100),
-            "warnings_share": round(obj.strings_with_warnings / ts * 100),
-            "unreviewed_share": round(obj.unreviewed_strings / ts * 100),
-            "completion_percent": int(math.floor(get_completed_percent(obj))),
-        }
-
-
-def get_top_instances(qs):
+def get_top_instances(qs, stats: dict[int, dict[str, int]]) -> dict[str, object] | None:
     """
     Get top instances in the queryset.
     """
+
+    if not stats:
+        return None
+
+    def _missing(x: tuple[int, dict[str, int]]) -> int:
+        _, d = x
+        return (
+            d["total"]
+            - d["approved"]
+            - d["pretranslated"]
+            - d["errors"]
+            - d["warnings"]
+        )
+
+    max_total_id = max(stats.items(), key=lambda x: x[1]["total"])[0]
+    max_approved_id = max(stats.items(), key=lambda x: x[1]["approved"])[0]
+    max_suggestions_id = max(stats.items(), key=lambda x: x[1]["unreviewed"])[0]
+    max_missing_id = max(stats.items(), key=_missing)[0]
+
     return {
-        "most_strings": sorted(qs, key=lambda x: x.total_strings)[-1],
-        "most_translations": sorted(qs, key=lambda x: x.approved_strings)[-1],
-        "most_suggestions": sorted(qs, key=lambda x: x.unreviewed_strings)[-1],
-        "most_missing": sorted(qs, key=lambda x: x.missing_strings)[-1],
+        "most_strings": next(row for row in qs if row.id == max_total_id),
+        "most_translations": next(row for row in qs if row.id == max_approved_id),
+        "most_suggestions": next(row for row in qs if row.id == max_suggestions_id),
+        "most_missing": next(row for row in qs if row.id == max_missing_id),
     }

pontoon/base/models/locale.py

@@ -7,7 +7,7 @@
 from django.contrib.auth.models import Group
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.db.models import Prefetch
+from django.db.models import Prefetch, Sum
 
 from pontoon.base.aggregated_stats import AggregatedStats
 
@@ -79,6 +79,34 @@ def prefetch_project_locale(self, project):
             )
         )
 
+    def stats_data(self, project=None) -> dict[int, dict[str, int]]:
+        """Mapping of locale `id` to dict with counts."""
+        if project is not None:
+            query = self.filter(translatedresources__resource__project=project)
+        else:
+            query = self.filter(
+                translatedresources__resource__project__disabled=False,
+                translatedresources__resource__project__system_project=False,
+                translatedresources__resource__project__visibility="public",
+            )
+        data = query.annotate(
+            total=Sum("translatedresources__total_strings", default=0),
+            approved=Sum("translatedresources__approved_strings", default=0),
+            pretranslated=Sum("translatedresources__pretranslated_strings", default=0),
+            errors=Sum("translatedresources__strings_with_errors", default=0),
+            warnings=Sum("translatedresources__strings_with_warnings", default=0),
+            unreviewed=Sum("translatedresources__unreviewed_strings", default=0),
+        ).values(
+            "id",
+            "total",
+            "approved",
+            "pretranslated",
+            "errors",
+            "warnings",
+            "unreviewed",
+        )
+        return {row["id"]: row for row in data if row["total"]}
+
 
 class Locale(models.Model, AggregatedStats):
     @property
@@ -366,11 +394,6 @@ def get_latest_activity(self, project=None):
 
         return ProjectLocale.get_latest_activity(self, project)
 
-    def get_chart(self, project=None):
-        from pontoon.base.models.project_locale import ProjectLocale
-
-        return ProjectLocale.get_chart(self, project)
-
     def save(self, *args, **kwargs):
         old = Locale.objects.get(pk=self.pk) if self.pk else None
         super().save(*args, **kwargs)

pontoon/base/models/project.py

@@ -4,7 +4,7 @@
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db import models
-from django.db.models import Prefetch
+from django.db.models import Prefetch, Sum
 from django.db.models.manager import BaseManager
 from django.utils import timezone
 
@@ -86,6 +86,32 @@ def prefetch_project_locale(self, locale):
             )
         )
 
+    def stats_data(self, locale=None) -> dict[int, dict[str, int]]:
+        """Mapping of project `id` to dict with counts."""
+        query = (
+            self
+            if locale is None
+            else self.filter(resources__translatedresources__locale=locale)
+        )
+        tr = "resources__translatedresources"
+        data = query.annotate(
+            total=Sum(f"{tr}__total_strings", default=0),
+            approved=Sum(f"{tr}__approved_strings", default=0),
+            pretranslated=Sum(f"{tr}__pretranslated_strings", default=0),
+            errors=Sum(f"{tr}__strings_with_errors", default=0),
+            warnings=Sum(f"{tr}__strings_with_warnings", default=0),
+            unreviewed=Sum(f"{tr}__unreviewed_strings", default=0),
+        ).values(
+            "id",
+            "total",
+            "approved",
+            "pretranslated",
+            "errors",
+            "warnings",
+            "unreviewed",
+        )
+        return {row["id"]: row for row in data if row["total"]}
+
 
 class Project(models.Model, AggregatedStats):
     @property
@@ -252,11 +278,6 @@ def get_latest_activity(self, locale=None):
 
         return ProjectLocale.get_latest_activity(self, locale)
 
-    def get_chart(self, locale=None):
-        from pontoon.base.models.project_locale import ProjectLocale
-
-        return ProjectLocale.get_chart(self, locale)
-
     @property
     def avg_string_count(self):
         return int(self.total_strings / self.enabled_locales)

pontoon/base/models/project_locale.py

@@ -2,7 +2,7 @@
 from django.db import models
 
 from pontoon.base import utils
-from pontoon.base.aggregated_stats import AggregatedStats, get_chart_dict
+from pontoon.base.aggregated_stats import AggregatedStats
 from pontoon.base.models.locale import Locale
 from pontoon.base.models.project import Project
 
@@ -111,34 +111,3 @@ def get_latest_activity(cls, self, extra=None):
                 latest_translation = project_locale.latest_translation
 
         return latest_translation.latest_activity if latest_translation else None
-
-    @classmethod
-    def get_chart(cls, self, extra=None):
-        """
-        Get chart for project, locale or combination of both.
-
-        :param self: object to get data for,
-            instance of Project or Locale
-        :param extra: extra filter to be used,
-            instance of Project or Locale
-        """
-        chart = None
-
-        if getattr(self, "fetched_project_locale", None):
-            if self.fetched_project_locale:
-                chart = get_chart_dict(self.fetched_project_locale[0])
-
-        elif extra is None:
-            chart = get_chart_dict(self)
-
-        else:
-            project = self if isinstance(self, Project) else extra
-            locale = self if isinstance(self, Locale) else extra
-            project_locale = utils.get_object_or_none(
-                ProjectLocale, project=project, locale=locale
-            )
-
-            if project_locale is not None:
-                chart = get_chart_dict(project_locale)
-
-        return chart

pontoon/base/models/translated_resource.py

@@ -113,6 +113,16 @@ def count_total_strings(self):
             total += (self.locale.nplurals - 1) * plural_count
         return total
 
+    def stats_data(self) -> dict[str, int]:
+        return {
+            "total": self.total_strings,
+            "approved": self.approved_strings,
+            "pretranslated": self.pretranslated_strings,
+            "errors": self.strings_with_errors,
+            "warnings": self.strings_with_warnings,
+            "unreviewed": self.unreviewed_strings,
+        }
+
     def adjust_stats(
         self, before: dict[str, int], after: dict[str, int], tr_created: bool
     ):

pontoon/base/templates/widgets/progress_chart.html

@@ -1,13 +1,13 @@
 {% macro span(chart, link, link_parameter=False, has_params=False) %}
 {% if chart != None %}
   <div class="chart-wrapper">
-    <span class="percent">{{ chart.completion_percent }}%</span>
+    <span class="percent">{{ (100 * (chart.approved + chart.pretranslated + chart.warnings) / chart.total) | round(0, 'floor') | int }}%</span>
     <span class="chart">
-      <span class="translated" style="width:{{ chart.approved_share }}%;"></span>
-      <span class="pretranslated" style="width:{{ chart.pretranslated_share }}%;"></span>
-      <span class="warnings" style="width:{{ chart.warnings_share }}%;"></span>
-      <span class="errors" style="width:{{ chart.errors_share }}%;"></span>
-      <span class="missing" style="width:{{ 100 - chart.approved_share - chart.pretranslated_share - chart.warnings_share - chart.errors_share }}%;"></span>
+      <span class="translated" style="width:{{ (100 * chart.approved / chart.total) | round(1) }}%"></span>
+      <span class="pretranslated" style="width:{{ (100 * chart.pretranslated / chart.total) | round(1) }}%"></span>
+      <span class="warnings" style="width:{{ (100 * chart.warnings / chart.total) | round(1) }}%"></span>
+      <span class="errors" style="width:{{ (100 * chart.errors / chart.total) | round(1) }}%"></span>
+      <span class="missing" style="width:{{ (100 * (chart.total - chart.approved - chart.pretranslated - chart.warnings - chart.errors) / chart.total) | round(1) }}%"></span>
     </span>
     <span class="unreviewed-status fas fa-lightbulb{% if chart.unreviewed > 0 %} pending{% endif %}"></span>
   </div>

pontoon/insights/tasks.py

@@ -12,7 +12,6 @@
 from django.utils import timezone
 
 from pontoon.actionlog.models import ActionLog
-from pontoon.base.aggregated_stats import get_completed_percent
 from pontoon.base.models import (
     Entity,
     Locale,
@@ -216,7 +215,14 @@ def get_locale_insights_snapshot(
     entities_count,
 ):
     """Create LocaleInsightsSnapshot instance for the given locale and day using given data."""
-    locale_stats = TranslatedResource.objects.filter(locale=locale).string_stats()
+    lc_stats = TranslatedResource.objects.filter(locale=locale).string_stats()
+    lc_completion = (
+        100
+        * (lc_stats["approved"] + lc_stats["pretranslated"] + lc_stats["warnings"])
+        / lc_stats["total"]
+        if lc_stats["total"] > 0
+        else 0
+    )
 
     all_managers, all_reviewers = get_privileged_users_data(privileged_users)
     all_contributors = {c["user"] for c in contributors}
@@ -286,12 +292,12 @@ def get_locale_insights_snapshot(
         locale=locale,
         created_at=start_of_today,
         # Aggregated stats
-        total_strings=locale_stats["total"],
-        approved_strings=locale_stats["approved"],
-        pretranslated_strings=locale_stats["pretranslated"],
-        strings_with_errors=locale_stats["errors"],
-        strings_with_warnings=locale_stats["warnings"],
-        unreviewed_strings=locale_stats["unreviewed"],
+        total_strings=lc_stats["total"],
+        approved_strings=lc_stats["approved"],
+        pretranslated_strings=lc_stats["pretranslated"],
+        strings_with_errors=lc_stats["errors"],
+        strings_with_warnings=lc_stats["warnings"],
+        unreviewed_strings=lc_stats["unreviewed"],
         # Active users
         total_managers=total_managers,
         total_reviewers=total_reviewers,
@@ -307,7 +313,7 @@ def get_locale_insights_snapshot(
         # Time to review pretranslations
         time_to_review_pretranslations=time_to_review_pretranslations,
         # Translation activity
-        completion=round(get_completed_percent(locale), 2),
+        completion=round(lc_completion, 2),
         human_translations=human_translations,
         machinery_translations=machinery_translations,
         new_source_strings=entities_count,
@@ -334,6 +340,13 @@ def get_project_locale_insights_snapshot(
     pl_stats = TranslatedResource.objects.filter(
         locale=project_locale.locale, resource__project=project_locale.project
     ).string_stats()
+    pl_completion = (
+        100
+        * (pl_stats["approved"] + pl_stats["pretranslated"] + pl_stats["warnings"])
+        / pl_stats["total"]
+        if pl_stats["total"] > 0
+        else 0
+    )
 
     (
         human_translations,
@@ -361,7 +374,7 @@ def get_project_locale_insights_snapshot(
         strings_with_warnings=pl_stats["warnings"],
         unreviewed_strings=pl_stats["unreviewed"],
         # Translation activity
-        completion=round(get_completed_percent(project_locale), 2),
+        completion=round(pl_completion, 2),
         human_translations=human_translations,
         machinery_translations=machinery_translations,
         new_source_strings=entities_count,

pontoon/localizations/templates/localizations/widgets/resource_list.html

@@ -35,7 +35,7 @@ <h4>
 
     {% if deadline %}
     <td class="deadline">
-      {{ Deadline.deadline(resource.deadline, chart.completion_percent == 100) }}
+      {{ Deadline.deadline(resource.deadline, chart.total > 0 and chart.total == chart.approved + chart.pretranslated + chart.warnings) }}
     </td>
     {% endif %}