Merge pull request #327 from open-craft/navin/auto-date

feat: fill date opened field and owner in ospr project board

Author: feanil

.gitignore

@@ -15,3 +15,4 @@ htmlcov
 openedx_webhooks.egg-info/
 requirements/private.in
 requirements/private.txt
+.mise.toml

openedx_webhooks/bot_comments.py

@@ -5,19 +5,18 @@
 import binascii
 import json
 import re
-from collections import namedtuple
 
 from enum import Enum, auto
-from typing import Dict, Literal
+from typing import Dict
 
 import arrow
 from flask import render_template
 
 from openedx_webhooks.info import (
     get_jira_server_info,
+    get_repo_spec,
     is_draft_pull_request,
     pull_request_has_cla,
-    get_catalog_info,
 )
 from openedx_webhooks.types import JiraId, PrDict
 
@@ -94,26 +93,6 @@ def is_comment_kind(kind: BotComment, text: str) -> bool:
     return any(snip in text for snip in BOT_COMMENT_INDICATORS[kind])
 
 
-Lifecycle = Literal["experimental", "production", "deprecated"]
-RepoSpec: (str | None, Lifecycle | None) = namedtuple('RepoSpec', ['owner', 'lifecycle'])
-
-
-def _get_repo_spec(repo_full_name: str) -> RepoSpec:
-    """
-    Get the owner of the repo from its catalog-info.yaml file.
-    """
-    catalog_info = get_catalog_info(repo_full_name)
-    if not catalog_info:
-        return RepoSpec(None, None)
-    owner = catalog_info["spec"].get("owner", "")
-    owner_type = None
-    if ":" in owner:
-        owner_type, owner = owner.split(":")
-    if owner_type == "group":
-        owner = f"openedx/{owner}"
-    return RepoSpec(owner, catalog_info["spec"]["lifecycle"])
-
-
 def github_community_pr_comment(pull_request: PrDict) -> str:
     """
     For a newly-created pull request from an open source contributor,
@@ -124,7 +103,7 @@ def github_community_pr_comment(pull_request: PrDict) -> str:
     * contain a link to our process documentation
     """
     is_first_time = pull_request.get("author_association", None) in GITHUB_NEW_AUTHOR_ASSOCIATIONS
-    spec = _get_repo_spec(pull_request["base"]["repo"]["full_name"])
+    spec = get_repo_spec(pull_request["base"]["repo"]["full_name"])
 
     return render_template(
         "github_community_pr_comment.md.j2",

openedx_webhooks/gh_projects.py

@@ -7,8 +7,8 @@
 from glom import glom
 
 from openedx_webhooks.tasks import logger
-from openedx_webhooks.types import GhProject, PrDict, PrId
-from openedx_webhooks.utils import graphql_query
+from openedx_webhooks.types import GhPrMetaDict, GhProject, PrDict, PrId
+from openedx_webhooks.utils import graphql_query, memoize_timed, value_graphql_type
 
 # The name of the query is used by FakeGitHub while testing.
 
@@ -22,6 +22,7 @@
     pullRequest (number: $number) {
       projectItems (first: 100) {
         nodes {
+          id
           project {
             number
             owner {
@@ -37,6 +38,7 @@
 }
 """
 
+
 def pull_request_projects(pr: PrDict) -> Set[GhProject]:
     """Return the projects this PR is in.
 
@@ -90,8 +92,9 @@ def pull_request_projects(pr: PrDict) -> Set[GhProject]:
 }
 """
 
-def add_pull_request_to_project(prid: PrId, pr_node_id: str, project: GhProject) -> None:
-    """Add a pull request to a project.
+
+def add_pull_request_to_project(prid: PrId, pr_node_id: str, project: GhProject) -> str:
+    """Add a pull request to a project and returns its project item id.
 
     The project is a tuple: (orgname, number)
     """
@@ -104,3 +107,95 @@ def add_pull_request_to_project(prid: PrId, pr_node_id: str, project: GhProject)
     # Add the pull request.
     variables = {"projectId": proj_id, "prNodeId": pr_node_id}
     data = graphql_query(query=ADD_PROJECT_ITEM, variables=variables)
+    # data = {'addProjectV2ItemById': {'item': {'id': '<item_id>'}}}
+    return glom(data, "addProjectV2ItemById.item.id")
+
+
+ORG_PROJECT_METADATA = """\
+query OrgProjectMetadata ($orgname: String!, $number: Int!) {
+  organization(login: $orgname) {
+    projectV2(number: $number) {
+      id
+      fields(first: 100) {
+        nodes {
+          ... on ProjectV2FieldCommon {
+            id
+            name
+            dataType
+          }
+          ... on ProjectV2SingleSelectField {
+            options {
+              id
+              name
+            }
+          }
+        }
+      }
+    }
+  }
+}
+"""
+
+
+@memoize_timed(minutes=30)
+def get_project_metadata(project: GhProject) -> GhPrMetaDict:
+    # Find the project metadata.
+    variables = {"orgname": project[0], "number": project[1]}
+    data: GhPrMetaDict = graphql_query(query=ORG_PROJECT_METADATA, variables=variables)
+    return glom(data, {"id": "organization.projectV2.id", "fields": "organization.projectV2.fields.nodes"})
+
+
+UPDATE_PROJECT_ITEM = """\
+mutation UpdateProjectItem (
+  $projectId: ID!
+  $itemId: ID!
+  $fieldId: ID!
+  $value: {fieldType}
+) {{
+  updateProjectV2ItemFieldValue (
+    input: {{
+      projectId: $projectId,
+      itemId: $itemId,
+      fieldId: $fieldId,
+      value: {{
+        {fieldTypeName}: $value,
+      }}
+    }}) {{
+    projectV2Item {{
+      id
+    }}
+  }}
+}}
+"""
+
+
+def update_project_pr_custom_field(field_name: str, field_value, item_id: str, project: GhProject) -> None:
+    """Add a pull request to a project.
+
+    The project is a tuple: (orgname, number)
+    """
+    logger.info(f"Updating project {project} field {field_name} to {field_value}")
+
+    project_metadata = get_project_metadata(project)
+    target_field = None
+    for field in project_metadata["fields"]:
+        if field["name"] == field_name:
+            target_field = field
+            break
+    else:
+        logger.error(f"Could not find field with name: {field_name} in project: {project}")
+        return
+
+    # Update field value
+    variables = {
+        "projectId": project_metadata["id"],
+        "fieldId": target_field["id"],
+        "itemId": item_id,
+        "value": field_value,
+    }
+    field_type_name = target_field["dataType"].lower()
+    field_type = value_graphql_type(field_type_name)
+    graphql_query(
+        query=UPDATE_PROJECT_ITEM.format(fieldType=field_type, fieldTypeName=field_type_name),
+        variables=variables
+    )

openedx_webhooks/info.py

@@ -1,11 +1,12 @@
 """
 Get information about people, repos, orgs, pull requests, etc.
 """
+from collections import namedtuple
 import csv
 import fnmatch
 import logging
 import re
-from typing import Dict, Iterable, Optional
+from typing import Dict, Iterable, Literal, Optional
 
 import yaml
 from glom import glom
@@ -301,12 +302,46 @@ def get_bot_comments(prid: PrId) -> Iterable[PrCommentDict]:
             yield comment
 
 
+@memoize_timed(minutes=15)
 def get_catalog_info(repo_fullname: str) -> Dict:
     """Get the parsed catalog-info.yaml data from a repo, or {} if missing."""
     yml = read_github_file(repo_fullname, "catalog-info.yaml", not_there="{}")
     return yaml.safe_load(yml)
 
 
+@memoize_timed(minutes=60)
+def get_github_user_info(username: str) -> Dict | None:
+    """Get github user information"""
+    resp = get_github_session().get(f"/users/{username}")
+    if resp.ok:
+        return resp.json()
+    logger.error(f"Could not find user information for user: {username} on github")
+    return None
+
+
+Lifecycle = Literal["experimental", "production", "deprecated"]
+RepoSpec: (str | None, Lifecycle | None, bool) = namedtuple('RepoSpec', ['owner', 'lifecycle', 'is_owner_individual'])
+
+
+def get_repo_spec(repo_full_name: str) -> RepoSpec:
+    """
+    Get the owner of the repo from its catalog-info.yaml file.
+    """
+    catalog_info = get_catalog_info(repo_full_name)
+    if not catalog_info:
+        return RepoSpec(None, None, False)
+    owner = catalog_info["spec"].get("owner", "")
+    owner_type = None
+    is_owner_individual = False
+    if ":" in owner:
+        owner_type, owner = owner.split(":")
+    if owner_type == "group":
+        owner = f"openedx/{owner}"
+    elif owner_type == "user":
+        is_owner_individual = True
+    return RepoSpec(owner, catalog_info["spec"]["lifecycle"], is_owner_individual)
+
+
 def projects_for_pr(pull_request: PrDict) -> Iterable[GhProject]:
     """
     Get the projects a pull request should be added to.

openedx_webhooks/tasks/pr_tracking.py

@@ -39,12 +39,15 @@
 from openedx_webhooks.gh_projects import (
     add_pull_request_to_project,
     pull_request_projects,
+    update_project_pr_custom_field,
 )
 from openedx_webhooks.info import (
     NoJiraMapping,
     NoJiraServer,
     get_blended_project_id,
     get_bot_comments,
+    get_github_user_info,
+    get_repo_spec,
     is_bot_pull_request,
     is_draft_pull_request,
     is_internal_pull_request,
@@ -402,10 +405,40 @@ def _fix_ospr(self) -> None:
         self._fix_comments()
 
         # Check the GitHub projects.
+        self._fix_projects()
+
+    def _fix_projects(self) -> None:
+        """
+        Update projects for pr.
+        """
         for project in (self.desired.github_projects - self.current.github_projects):
-            self.actions.add_pull_request_to_project(
+            project_item_id = self.actions.add_pull_request_to_project(
                 pr_node_id=self.pr["node_id"], project=project
             )
+            if not project_item_id:
+                continue
+            self.actions.update_project_pr_custom_field(
+                field_name="Date opened",
+                field_value=self.pr["created_at"],
+                item_id=project_item_id,
+                project=project
+            )
+            # get base repo owner info
+            repo_spec = get_repo_spec(self.pr["base"]["repo"]["full_name"])
+            owner = repo_spec.owner
+            if not owner:
+                continue
+            # get user info if owner is an individual
+            if repo_spec.is_owner_individual:
+                owner_info = get_github_user_info(owner)
+                if owner_info:
+                    owner = f"{owner_info['name']} (@{owner})"
+            self.actions.update_project_pr_custom_field(
+                field_name="Repo Owner / Owning Team",
+                field_value=owner,
+                item_id=project_item_id,
+                project=project
+            )
 
     def _make_jira_issue(self, jira_nick) -> None:
         """
@@ -659,14 +692,24 @@ def update_labels_on_pull_request(self, *, labels: List[str]) -> None:
         resp = get_github_session().patch(url, json={"labels": labels})
         log_check_response(resp)
 
-    def add_pull_request_to_project(self, *, pr_node_id: str, project: GhProject) -> None:
+    def add_pull_request_to_project(self, *, pr_node_id: str, project: GhProject) -> str | None:
         """
         Add a pull request to a project.
         """
         try:
-            add_pull_request_to_project(self.prid, pr_node_id, project)
-        except Exception as exc:    # pylint: disable=broad-exception-caught
-            logger.exception(f"Couldn't add PR to project: {exc}")
+            return add_pull_request_to_project(self.prid, pr_node_id, project)
+        except Exception:    # pylint: disable=broad-exception-caught
+            logger.exception("Couldn't add PR to project")
+        return None
+
+    def update_project_pr_custom_field(self, *, field_name: str, field_value, item_id: str, project: GhProject) -> None:
+        """
+        Add a pull request to a project.
+        """
+        try:
+            update_project_pr_custom_field(field_name, field_value, item_id, project)
+        except Exception:    # pylint: disable=broad-exception-caught
+            logger.exception(f"Couldn't update: {field_name} for a PR in project")
 
     def set_cla_status(self, *, status: Dict[str, str]) -> None:
         set_cla_status_on_pr(self.prid.full_name, self.prid.number, status)

openedx_webhooks/types.py

@@ -17,6 +17,9 @@
 # A GitHub project: org name, and number.
 GhProject = Tuple[str, int]
 
+# A GitHub project metadata json object.
+GhPrMetaDict = Dict
+
 
 @dataclasses.dataclass(frozen=True)
 class PrId:

openedx_webhooks/utils.py

@@ -228,6 +228,15 @@ def jira_paginated_get(url, session=None,
             more_results = True  # just keep going until there are no more results.
 
 
+def value_graphql_type(field_type: str) -> str:
+    if field_type == "date":
+        return "Date"
+    elif field_type == "number":
+        return "Float"
+    else:
+        return "String"
+
+
 def graphql_query(query: str, variables: Dict = {}) -> Dict:    # pylint: disable=dangerous-default-value
     """
     Make a GraphQL query against GitHub.