Add create_django_issue.py script for GitHub actions cron

Signed-off-by: Andrew-Chen-Wang <acwangpython@gmail.com>

.github/workflows/django-issue-checker.yml

@@ -0,0 +1,29 @@
+# Creates a new issue for Major/Minor Django updates that keeps track
+# of all dependencies that need to be updated/merged in order for the
+# latest Django version to also be merged.
+name: Django Issue Checker
+
+on:
+  # Every day at midnight
+  schedule:
+  - cron: "0 3 * * *"
+  # Manual trigger
+  workflow_dispatch:
+
+
+jobs:
+  issue-manager:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2.2.2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Create Django Major Issue
+      run: python scripts/create_django_issue.py
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

requirements.txt

@@ -22,3 +22,4 @@ pyyaml==5.4.1
 # ------------------------------------------------------------------------------
 PyGithub==1.55
 jinja2==3.0.1
+requests==2.25.1

scripts/create_django_issue.py

@@ -0,0 +1,279 @@
+"""
+Creates an issue that generates a table for dependency checking whether
+all packages support the latest Django version. "Latest" does not include
+patches, only comparing major and minor version numbers.
+
+This script handles when there are multiple Django versions that need
+to keep up to date.
+"""
+
+import os
+from typing import Sequence, TYPE_CHECKING
+
+import requests
+import sys
+from pathlib import Path
+
+from github import Github
+
+
+if TYPE_CHECKING:
+    from github.Issue import Issue
+
+CURRENT_FILE = Path(__file__)
+ROOT = CURRENT_FILE.parents[1]
+REQUIREMENTS_DIR = ROOT / "{{cookiecutter.project_slug}}" / "requirements"
+GITHUB_REPO = "pydanny/cookiecutter-django"
+
+
+def get_package_info(package: str) -> dict:
+    # "django" converts to "Django" on redirect
+    r = requests.get(f"https://pypi.org/pypi/{package}/json", allow_redirects=True)
+    if not r.ok:
+        print(f"Couldn't find package: {package}")
+        sys.exit(1)
+    return r.json()
+
+
+def get_package_versions(package_info: dict, reverse=True, *, include_pre=False):
+    # Mostly used for the Django check really... to get the latest
+    # package version, you could simple do get_package_info()["info"]["version"]
+    releases: Sequence[str] = package_info["releases"].keys()
+    if not include_pre:
+        releases = [x for x in releases if x.replace(".", "").isdigit()]
+    return sorted(releases, reverse=reverse)
+
+
+def get_name_and_version(requirements_line: str) -> list[str, str]:
+    return requirements_line.split(" ", 1)[0].split("==")
+
+
+def get_all_latest_django_versions() -> tuple[str, list[str]]:
+    """
+    Grabs all Django versions that are worthy of a GitHub issue. Depends on
+    if Django versions has higher major version or minor version
+    """
+    base_txt = REQUIREMENTS_DIR / "base.txt"
+    with base_txt.open() as f:
+        for line in f.readlines():
+            if "django==" in line:
+                break
+        else:
+            print(f"django not found in {base_txt}")  # Huh...?
+            sys.exit(1)
+
+    # Begin parsing and verification
+    base_django_version = get_name_and_version(line)[1].split(".")
+    django_versions = get_package_versions(get_package_info("django"), include_pre=True)
+    _needed_django_versions: set[tuple] = set()
+    actual_needed_django_versions: list[str] = []
+    for x in django_versions:
+        _version = x.split(".")
+        # Compare if major is higher or if minor is higher iff major is the same
+        if (_version[0] > base_django_version[0]) or (
+            _version[0] == base_django_version[0]
+            and _version[1] > base_django_version[1]
+        ):
+            will_add = (_version[0], _version[1])
+            if will_add not in _needed_django_versions:
+                _needed_django_versions.add(will_add)
+                actual_needed_django_versions.append(x)
+
+    return line, actual_needed_django_versions
+
+
+def get_first_digit(tokens) -> str:
+    return next(item for item in tokens if item.isdigit())
+
+
+_TABLE_HEADER = """{file}.txt
+
+| Name | Version in Master | {dj_version} Compatible Version | OK |
+| ---- | :---------------: | :-----------------------------: | :-: |
+"""
+VITAL_BUT_UNKNOWN = [
+    "django-environ",  # not updated often
+    "pylint-django",  # classifier not included in setup.py
+]
+
+
+class GitHubManager:
+    def __init__(self, base_dj_version: str, needed_dj_versions: list[str]):
+        self.github = Github(os.getenv("GITHUB_TOKEN", None))
+        self.repo = self.github.get_repo(GITHUB_REPO)
+
+        self.base_dj_version = base_dj_version
+        self.needed_dj_versions = needed_dj_versions
+        # (major+minor) Version and description
+        self.existing_issues: dict[str, "Issue"] = {}
+
+        # Load all requirements from our requirements files and preload their
+        # package information like a cache:
+        self.requirements_files = ["base", "local", "production"]
+        # Format:
+        # requirement file name: {package name: (master_version, package_info)}
+        self.requirements: dict[str, dict[str, tuple[str, dict]]] = {
+            x: {} for x in self.requirements_files
+        }
+
+    def setup(self) -> None:
+        self.load_requirements()
+        self.load_existing_issues()
+
+    def load_requirements(self):
+        for requirements_file in self.requirements_files:
+            with (REQUIREMENTS_DIR / f"{requirements_file}.txt").open() as f:
+                for line in f.readlines():
+                    if "==" in line:
+                        name, version = get_name_and_version(line)
+                        self.requirements[requirements_file][name] = (
+                            version, get_package_info(name)
+                        )
+
+    def load_existing_issues(self):
+        """Closes the issue if the base Django version is greater than the needed"""
+        qualifiers = {
+            "repo": GITHUB_REPO,
+            "author": "actions-user",
+            "state": "open",
+            "is": "issue",
+            "in": "title",
+        }
+        issues = list(
+            self.github.search_issues(
+                "[Django Update]", "created", "desc", **qualifiers
+            )
+        )
+        for issue in issues:
+            try:
+                dj_version = get_first_digit(issue.title.split(" "))
+            except StopIteration:
+                try:
+                    # Some padding; randomly chose 4 to make sure we don't get a random
+                    # version number from a package that's not Django
+                    dj_version = get_first_digit(issue.body.split(" ", 4))
+                except StopIteration:
+                    print(
+                        f"Found issue {issue.title} that had an invalid syntax",
+                        "(Did not have a Django version number in the title or body's"
+                        f" first word. Issue number: [{issue.id}]({issue.url}))"
+                    )
+                    continue
+            if self.base_dj_version > dj_version:
+                issue.edit(state="closed")
+                print(f"Closed issue {issue.title} (ID: [{issue.id}]({issue.url}))")
+                try:
+                    self.needed_dj_versions.remove(dj_version)
+                except ValueError:
+                    print("Something weird happened. Continuing anyway (Warning ID: 1)")
+            else:
+                self.existing_issues[dj_version] = issue
+
+    def get_compatibility(
+        self, package_name: str, package_info: dict, needed_dj_version
+    ):
+        """
+        Verify compatibility via setup.py classifiers. If Django is not in the
+        classifiers, then default compatibility is n/a and OK is ✅.
+
+        If it's a package that's vital but known to not be updated often, we give it
+        a ❓. If a package has ❓ or 🕒, then we allow manual update. Automatic updates
+         only include ❌ and ✅.
+        """
+        # If issue previously existed, find package and skip any gtg, manually
+        # updated packages, or known releases that will happen but haven't yet
+        if issue := self.existing_issues.get(needed_dj_version):
+            if index := issue.body.find(package_name):
+                name, _current, prev_compat, ok = issue.body[index:].split("|", 4)[:4]
+                if ok in ("✅", "❓", "🕒"):
+                    return prev_compat, ok
+
+        if package_name in VITAL_BUT_UNKNOWN:
+            return "", "❓"
+
+        # Check classifiers if it includes Django
+        supported_dj_versions = []
+        for classifier in package_info["info"]["classifiers"]:
+            # Usually in the form of "Framework :: Django :: 3.2"
+            tokens = classifier.split(" ")
+            for token in tokens:
+                if token.lower() == "django":
+                    try:
+                        _version = get_first_digit(reversed(tokens))
+                    except StopIteration:
+                        pass
+                    else:
+                        supported_dj_versions.append(
+                            float(".".join(_version.split(".", 2)[:2]))
+                        )
+
+        if supported_dj_versions:
+            needed_dj_version = float(needed_dj_version)
+            if any(x >= needed_dj_version for x in supported_dj_versions):
+                return package_info["info"]["version"], "✅"
+            else:
+                return "", "❌"
+
+        # Django classifier DNE; assume it just isn't a Django lib
+        # Great exceptions include pylint-django, where we need to do this manually...
+        return "n/a", "✅"
+
+    HOME_PAGE_URL_KEYS = [
+        "home_page",
+        "project_url",
+        "docs_url",
+        "package_url",
+        "release_url",
+        "bugtrack_url",
+    ]
+
+    def _get_md_home_page_url(self, package_info: dict):
+        urls = [package_info["info"].get(x) for x in self.HOME_PAGE_URL_KEYS]
+        try:
+            return f"[{{}}]({next(item for item in urls if item)})"
+        except StopIteration:
+            return "{}"
+
+    def generate_markdown(self, needed_dj_version: str):
+        requirements = f"{needed_dj_version} requirements tables\n\n"
+        for _file in self.requirements_files:
+            requirements += (
+                _TABLE_HEADER.format_map(
+                    {"file": _file, "dj_version": needed_dj_version}
+                )
+            )
+            for package_name, (version, info) in self.requirements[_file].items():
+                compat_version, icon = self.get_compatibility(
+                    package_name, info, needed_dj_version
+                )
+                requirements += (
+                    f"|{self._get_md_home_page_url(info).format(package_name)}"
+                    f"|{version}|{compat_version}|{icon}|"
+                )
+        return requirements
+
+    def create_or_edit_issue(self, needed_dj_version, description):
+        if issue := self.existing_issues.get(str(needed_dj_version)):
+            issue.edit(body=description)
+        else:
+            self.repo.create_issue(
+                f"[Update Django] Django {needed_dj_version}", description
+            )
+
+    def generate(self):
+        for version in self.needed_dj_versions:
+            self.create_or_edit_issue(version, self.generate_markdown(version))
+
+
+def main() -> None:
+    # Check if there are any djs
+    current_dj, latest_djs = get_all_latest_django_versions()
+    if not latest_djs:
+        sys.exit(0)
+    manager = GitHubManager(current_dj, latest_djs)
+    manager.setup()
+    manager.generate()
+
+
+if __name__ == "__main__":
+    main()