feat: allow publishing multiple packages in one PR (#1418)

Author: radu-mocanu

.github/scripts/check_version_uniqueness.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Ensure every changed package has a version not yet published to PyPI.
+
+For each package with file changes in the PR, we read its version from
+pyproject.toml and check whether that exact version exists on PyPI.
+If it does, the check fails — the developer must bump the version.
+
+This catches two scenarios:
+  1. Developer changed code but forgot to bump the version.
+  2. Two PRs raced to the same version — after rebase the first PR's
+     version is already on PyPI, so this check forces the second PR
+     to pick a new one.
+"""
+
+import os
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore[no-redef]
+
+
+def version_exists_on_pypi(package_name: str, version: str) -> bool | None:
+    url = f"https://pypi.org/pypi/{package_name}/{version}/json"
+    try:
+        req = urllib.request.Request(url, method="HEAD")
+        with urllib.request.urlopen(req, timeout=10):
+            return True
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return False
+        print(f"Warning: PyPI returned HTTP {e.code} for {package_name}=={version}", file=sys.stderr)
+        return None
+    except Exception as e:
+        print(f"Warning: Could not reach PyPI for {package_name}=={version}: {e}", file=sys.stderr)
+        return None
+
+
+def get_package_info(package_dir: str) -> tuple[str, str] | None:
+    pyproject = Path("packages") / package_dir / "pyproject.toml"
+    if not pyproject.exists():
+        return None
+    with open(pyproject, "rb") as f:
+        data = tomllib.load(f)
+    project = data.get("project", {})
+    name = project.get("name")
+    version = project.get("version")
+    if name and version:
+        return name, version
+    return None
+
+
+def get_changed_packages() -> list[str]:
+    base_sha = os.getenv("BASE_SHA", "")
+    head_sha = os.getenv("HEAD_SHA", "")
+    diff_spec = f"{base_sha}...{head_sha}" if base_sha and head_sha else "origin/main...HEAD"
+
+    try:
+        result = subprocess.run(
+            ["git", "diff", "--name-only", diff_spec],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"Error running git diff: {e}", file=sys.stderr)
+        return []
+
+    changed = set()
+    for file_path in result.stdout.strip().split("\n"):
+        if file_path.startswith("packages/"):
+            parts = file_path.split("/")
+            if len(parts) >= 2 and (Path("packages") / parts[1] / "pyproject.toml").exists():
+                changed.add(parts[1])
+    return sorted(changed)
+
+
+def main() -> int:
+    changed = get_changed_packages()
+    if not changed:
+        print("No changed packages detected.")
+        return 0
+
+    conflicts = []
+    failures = []
+    for pkg_dir in changed:
+        info = get_package_info(pkg_dir)
+        if not info:
+            continue
+
+        name, version = info
+        exists = version_exists_on_pypi(name, version)
+
+        if exists is True:
+            print(f"FAIL: {name}=={version} already exists on PyPI")
+            conflicts.append(f"{name}=={version}")
+        elif exists is False:
+            print(f"OK: {name}=={version} is available")
+        else:
+            print(f"FAIL: could not verify {name}=={version}")
+            failures.append(f"{name}=={version}")
+
+    success = len(conflicts) + len(failures) == 0
+    if not success:
+        if conflicts:
+            print(f"\nPlease bump the version in pyproject.toml for: {', '.join(conflicts)}", file=sys.stderr)
+        if failures:
+            print(f"\nError while trying to check the following packages on pypi index: {', '.join(failures)}. Please retry.", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/test_check_version_uniqueness.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Tests for check_version_uniqueness.py."""
+
+import os
+import urllib.error
+from unittest import mock
+
+import pytest
+
+from check_version_uniqueness import (
+    get_package_info,
+    main,
+    version_exists_on_pypi,
+)
+
+
+class TestVersionExistsOnPypi:
+    def test_returns_true_when_version_exists(self):
+        mock_response = mock.MagicMock()
+        mock_response.__enter__ = mock.MagicMock(return_value=mock_response)
+        mock_response.__exit__ = mock.MagicMock(return_value=False)
+
+        with mock.patch("urllib.request.urlopen", return_value=mock_response):
+            assert version_exists_on_pypi("some-package", "1.0.0") is True
+
+    def test_returns_false_when_404(self):
+        error = urllib.error.HTTPError(
+            url="", code=404, msg="Not Found", hdrs=None, fp=None  # type: ignore[arg-type]
+        )
+        with mock.patch("urllib.request.urlopen", side_effect=error):
+            assert version_exists_on_pypi("some-package", "9.9.9") is False
+
+    def test_returns_none_on_server_error(self):
+        error = urllib.error.HTTPError(
+            url="", code=500, msg="Server Error", hdrs=None, fp=None  # type: ignore[arg-type]
+        )
+        with mock.patch("urllib.request.urlopen", side_effect=error):
+            assert version_exists_on_pypi("some-package", "1.0.0") is None
+
+    def test_returns_none_on_network_error(self):
+        with mock.patch(
+            "urllib.request.urlopen", side_effect=ConnectionError("no network")
+        ):
+            assert version_exists_on_pypi("some-package", "1.0.0") is None
+
+
+class TestGetPackageInfo:
+    def test_reads_name_and_version(self, tmp_path):
+        pkg = tmp_path / "packages" / "my-pkg"
+        pkg.mkdir(parents=True)
+        (pkg / "pyproject.toml").write_text(
+            '[project]\nname = "my-pkg"\nversion = "1.2.3"\n'
+        )
+        with mock.patch(
+            "check_version_uniqueness.Path",
+            side_effect=lambda p: tmp_path / p if p == "packages" else __import__("pathlib").Path(p),
+        ):
+            assert get_package_info("my-pkg") == ("my-pkg", "1.2.3")
+
+    def test_returns_none_for_missing_file(self):
+        assert get_package_info("nonexistent-package-xyz") is None
+
+
+class TestMain:
+    def _run(self, changed, package_info, pypi_result):
+        patches = [
+            mock.patch("check_version_uniqueness.get_changed_packages", return_value=changed),
+            mock.patch("check_version_uniqueness.get_package_info", side_effect=lambda d: package_info.get(d)),
+            mock.patch("check_version_uniqueness.version_exists_on_pypi", return_value=pypi_result),
+        ]
+        with patches[0], patches[1], patches[2]:
+            os.environ.pop("BASE_SHA", None)
+            os.environ.pop("HEAD_SHA", None)
+            return main()
+
+    def test_passes_when_version_not_on_pypi(self):
+        assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "2.0.0")}, False) == 0
+
+    def test_fails_when_version_exists_on_pypi(self):
+        assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "2.0.0")}, True) == 1
+
+    def test_fails_on_network_error(self):
+        assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "2.0.0")}, None) == 1
+
+    def test_no_changed_packages(self):
+        assert self._run([], {}, False) == 0
+
+    def test_fails_when_version_unchanged_but_on_pypi(self):
+        """The key scenario: code changed, version not bumped, already on PyPI."""
+        assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "1.0.0")}, True) == 1
+
+    def test_multiple_packages_one_conflict(self):
+        def pypi_check(name, version):
+            return name == "pkg-a"
+
+        with (
+            mock.patch("check_version_uniqueness.get_changed_packages", return_value=["pkg-a", "pkg-b"]),
+            mock.patch(
+                "check_version_uniqueness.get_package_info",
+                side_effect=lambda d: {"pkg-a": ("pkg-a", "1.0.0"), "pkg-b": ("pkg-b", "2.0.0")}.get(d),
+            ),
+            mock.patch("check_version_uniqueness.version_exists_on_pypi", side_effect=pypi_check),
+        ):
+            os.environ.pop("BASE_SHA", None)
+            os.environ.pop("HEAD_SHA", None)
+            assert main() == 1

.github/scripts/wait_for_pypi.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Wait for a package version to become available on PyPI.
+
+Usage: python wait_for_pypi.py <package-directory-name>
+
+Reads the package name and version from packages/<dir>/pyproject.toml,
+then polls PyPI until the version appears or a timeout is reached.
+"""
+
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore[no-redef]
+
+MAX_WAIT = 300  # 5 minutes
+POLL_INTERVAL = 15  # seconds
+
+
+def version_exists_on_pypi(package_name: str, version: str) -> bool:
+    """Check if a specific version of a package exists on PyPI."""
+    url = f"https://pypi.org/pypi/{package_name}/{version}/json"
+    try:
+        req = urllib.request.Request(url, method="HEAD")
+        with urllib.request.urlopen(req, timeout=10):
+            return True
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return False
+        print(f"  Warning: PyPI returned HTTP {e.code}", file=sys.stderr)
+        return False
+    except Exception as e:
+        print(f"  Warning: Could not reach PyPI: {e}", file=sys.stderr)
+        return False
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <package-directory>", file=sys.stderr)
+        return 1
+
+    directory = sys.argv[1]
+    pyproject = Path("packages") / directory / "pyproject.toml"
+
+    if not pyproject.exists():
+        print(f"ERROR: {pyproject} not found", file=sys.stderr)
+        return 1
+
+    with open(pyproject, "rb") as f:
+        data = tomllib.load(f)
+
+    name = data["project"]["name"]
+    version = data["project"]["version"]
+
+    print(f"Waiting for {name}=={version} to appear on PyPI...")
+    elapsed = 0
+    while elapsed < MAX_WAIT:
+        if version_exists_on_pypi(name, version):
+            print(f"{name}=={version} is now available on PyPI.")
+            return 0
+        time.sleep(POLL_INTERVAL)
+        elapsed += POLL_INTERVAL
+        print(f"  Still waiting... ({elapsed}s/{MAX_WAIT}s)")
+
+    print(
+        f"ERROR: {name}=={version} did not appear on PyPI within {MAX_WAIT}s",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())