Fix plugin list discovery and updates (#978)

Author: tobiasraabe

.github/workflows/update-plugin-list.yml

@@ -39,6 +39,5 @@ jobs:
           author: 'Tobias Raabe <tobiasraabe@users.noreply.github.com>'
           branch: update-plugin-list/patch
           delete-branch: true
-          branch-suffix: short-commit-hash
           title: '[automated] Update plugin list'
           body: '[automated] Update plugin list'

docs/source/plugin_list.md

@@ -13,12 +13,13 @@ Packages classified as inactive are excluded.
     conduct your own quality assessment before incorporating any of these plugins into your
     own projects.
 
-This list contains 5 plugins.
+This list contains 6 plugins.
 
-| name                                                     | summary                                    | last release | status    | requires       |
-| -------------------------------------------------------- | ------------------------------------------ | ------------ | --------- | -------------- |
-| [pytask-julia](https://pypi.org/project/pytask-julia/)   | A Pytask plugin for Julia                  | Oct 07, 2023 | 4 - Beta  | pytask >=0.4.0 |
-| [pytask-latex](https://pypi.org/project/pytask-latex/)   | Compile LaTeX documents with pytask.       | Jun 01, 2025 | 4 - Beta  | pytask>=0.4.0  |
-| [pytask-r](https://pypi.org/project/pytask-r/)           | Run R scripts with pytask.                 | Apr 20, 2024 | 4 - Beta  | pytask>=0.4.5  |
-| [pytask-stata](https://pypi.org/project/pytask-stata/)   | Execute do-files with Stata and pytask.    | Jul 26, 2025 | 4 - Beta  | pytask>=0.5.2  |
-| [pytask-vscode](https://pypi.org/project/pytask-vscode/) | Additional Logging for VS Code integration | Nov 21, 2023 | 3 - Alpha | pytask >=0.4.2 |
+| name                                                         | summary                                         | last release | status    | requires       |
+| ------------------------------------------------------------ | ----------------------------------------------- | ------------ | --------- | -------------- |
+| [pytask-julia](https://pypi.org/project/pytask-julia/)       | A Pytask plugin for Julia                       | Oct 07, 2023 | 4 - Beta  | pytask >=0.4.0 |
+| [pytask-latex](https://pypi.org/project/pytask-latex/)       | Compile LaTeX documents with pytask.            | Jun 01, 2025 | 4 - Beta  | pytask>=0.4.0  |
+| [pytask-parallel](https://pypi.org/project/pytask-parallel/) | Parallelize the execution of tasks with pytask. | Aug 23, 2026 | 4 - Beta  | pytask>=0.5.2  |
+| [pytask-r](https://pypi.org/project/pytask-r/)               | Run R scripts with pytask.                      | Apr 20, 2024 | 4 - Beta  | pytask>=0.4.5  |
+| [pytask-stata](https://pypi.org/project/pytask-stata/)       | Execute do-files with Stata and pytask.         | Jun 14, 2026 | 4 - Beta  | pytask>=0.5.2  |
+| [pytask-vscode](https://pypi.org/project/pytask-vscode/)     | Additional Logging for VS Code integration      | Nov 21, 2023 | 3 - Alpha | pytask >=0.4.2 |

scripts/update_plugin_list.py

@@ -32,13 +32,17 @@
 from __future__ import annotations
 
 import datetime
+import html.parser
 import pathlib
 import re
 from typing import TYPE_CHECKING
+from urllib.parse import unquote
+from urllib.parse import urlparse
 
 import httpx
 import packaging.version
 import wcwidth
+from packaging.utils import canonicalize_name
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -122,27 +126,61 @@ def _create_table(entries: list[dict[str, str]]) -> str:
     return "\n".join([header, separator, *rows])
 
 
+class _SimpleIndexParser(html.parser.HTMLParser):
+    """Extract normalized project names from the PyPI simple index."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.project_names: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag != "a":
+            return
+
+        href = dict(attrs).get("href")
+        if href is None:
+            return
+
+        path = urlparse(href).path.rstrip("/")
+        if not path.startswith("/simple/"):
+            return
+
+        project_name = unquote(path.removeprefix("/simple/"))
+        if "/" not in project_name and project_name:
+            self.project_names.append(canonicalize_name(project_name))
+
+
+def _iter_project_names(simple_index: str) -> list[str]:
+    """Return normalized project names from a PyPI simple index response."""
+    parser = _SimpleIndexParser()
+    parser.feed(simple_index)
+    parser.close()
+    return parser.project_names
+
+
 def _iter_plugins() -> Generator[dict[str, str], None, None]:  # noqa: C901
     """Iterate over all plugins and format entries."""
-    regex = r">([\d\w-]*)</a>"
     response = httpx.get("https://pypi.org/simple/", timeout=20)
+    response.raise_for_status()
 
-    matches = [
-        match
-        for match in re.finditer(regex, response.text)
-        if match.groups()[0].startswith("pytask-")
-        and match.groups()[0] not in _EXCLUDED_PACKAGES
+    project_names = [
+        project_name
+        for project_name in _iter_project_names(response.text)
+        if project_name.startswith("pytask-") and project_name not in _EXCLUDED_PACKAGES
     ]
 
-    for match in matches:
-        name = match.groups()[0]
-        response = httpx.get(f"https://pypi.org/pypi/{name}/json", timeout=20)
-        if response.status_code == 404:  # noqa: PLR2004
+    for project_name in project_names:
+        package_response = httpx.get(
+            f"https://pypi.org/pypi/{project_name}/json",
+            timeout=20,
+        )
+        if package_response.status_code == 404:  # noqa: PLR2004
             # Some packages might return a 404.
             continue
 
-        response.raise_for_status()
-        info = response.json()["info"]
+        package_response.raise_for_status()
+        package = package_response.json()
+        info = package["info"]
 
         if "Development Status :: 7 - Inactive" in info["classifiers"]:
             continue
@@ -171,7 +209,7 @@ def _version_sort_key(version_string: str) -> packaging.version.Version:
                 # Use a hard-coded pre-release version.
                 return packaging.version.Version("0.0.0alpha")
 
-        releases = response.json()["releases"]
+        releases = package["releases"]
 
         for release in sorted(releases, key=_version_sort_key, reverse=True):
             if releases[release]:
@@ -181,7 +219,9 @@ def _version_sort_key(version_string: str) -> packaging.version.Version:
                 last_release = release_date.strftime("%b %d, %Y")
                 break
 
-        name = f"[{info['name']}](https://pypi.org/project/{info['name']}/)"
+        # Use the canonical name from the simple-index link. Some projects have
+        # temporarily published metadata with a different name, e.g. pytask_stata.
+        name = f"[{project_name}](https://pypi.org/project/{project_name}/)"
         summary = ""
         if info["summary"]:
             summary = _escape_markdown(info["summary"].replace("\n", ""))