feat: enable pyproject.toml as single source of truth for Python version

Author: Jan Winkler (jwi)

.bazelrc.deleted_packages

@@ -45,3 +45,7 @@ common --deleted_packages=tests/modules/other/nspkg_single
 common --deleted_packages=tests/modules/other/simple_v1
 common --deleted_packages=tests/modules/other/simple_v2
 common --deleted_packages=tests/modules/other/with_external_data
+common --deleted_packages=tests/pyproject/compile_requirements_test
+common --deleted_packages=tests/pyproject/pip_integration_test
+common --deleted_packages=tests/pyproject/priority_test
+common --deleted_packages=tests/pyproject/python_toolchain_test

.gitattributes

@@ -1,2 +1,3 @@
 python/features.bzl export-subst
 tools/publish/*.txt linguist-generated=true
+requirements_lock.txt linguist-generated=true

.gitignore

@@ -48,8 +48,10 @@ user.bazelrc
 # CLion
 .clwb
 
-# Python cache
+# Python artifacts
 **/__pycache__/
+*.egg
+*.egg-info
 
 # MODULE.bazel.lock is ignored for now as per recommendation from upstream.
 # See https://github.com/bazelbuild/bazel/issues/20369

CHANGELOG.md

@@ -79,6 +79,7 @@ END_UNRELEASED_TEMPLATE
 ### Fixed
 * (runfiles) Fixed `CurrentRepository()` raising `ValueError` on Windows.
   ([#3579](https://github.com/bazel-contrib/rules_python/issues/3579))
+* (pip) Made `config_settings` optional in `pip.default()` when not using platform-specific configurations.
 * (tests) No more coverage warnings are being printed if there are no sources.
   ([#2762](https://github.com/bazel-contrib/rules_python/issues/2762))
 * (gazelle) Ancestor `conftest.py` files are added in addition to sibling `conftest.py`.
@@ -95,6 +96,8 @@ END_UNRELEASED_TEMPLATE
 
 {#v0-0-0-added}
 ### Added
+* (pip,python) Added `pyproject_toml` attribute to `pip.default()` and `python.defaults()`
+  to read Python version from pyproject.toml `requires-python` field (must be `==X.Y.Z` format).
 * (binaries/tests) {obj}`--debugger`: allows specifying an extra dependency
   to add to binaries/tests for custom debuggers.
 * (binaries/tests) Build information is now included in binaries and tests.

python/private/BUILD.bazel

@@ -29,7 +29,10 @@ package(
 
 licenses(["notice"])
 
-exports_files(["runtime_env_toolchain_interpreter.sh"])
+exports_files([
+    "runtime_env_toolchain_interpreter.sh",
+    "pyproject_version_extractor.py",
+])
 
 filegroup(
     name = "distribution",
@@ -272,6 +275,8 @@ bzl_library(
     deps = [
         ":full_version_bzl",
         ":platform_info_bzl",
+        ":pyproject_repo_bzl",
+        ":pyproject_utils_bzl",
         ":python_register_toolchains_bzl",
         ":pythons_hub_bzl",
         ":repo_utils_bzl",
@@ -752,6 +757,16 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "pyproject_repo_bzl",
+    srcs = ["pyproject_repo.bzl"],
+)
+
+bzl_library(
+    name = "pyproject_utils_bzl",
+    srcs = ["pyproject_utils.bzl"],
+)
+
 # Needed to define bzl_library targets for docgen. (We don't define the
 # bzl_library target here because it'd give our users a transitive dependency
 # on Skylib.)

python/private/pypi/BUILD.bazel

@@ -127,6 +127,7 @@ bzl_library(
         ":whl_library_bzl",
         "//python/private:auth_bzl",
         "//python/private:normalize_name_bzl",
+        "//python/private:pyproject_utils_bzl",
         "//python/private:repo_utils_bzl",
         "@bazel_features//:features",
         "@pythons_hub//:interpreters_bzl",

python/private/pypi/extension.bzl

@@ -19,6 +19,7 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING")
 load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:pyproject_utils.bzl", "read_pyproject_version")
 load("//python/private:repo_utils.bzl", "repo_utils")
 load(":evaluate_markers.bzl", EVALUATE_MARKERS_SRCS = "SRCS")
 load(":hub_builder.bzl", "hub_builder")
@@ -86,12 +87,23 @@ def build_config(
     """
     defaults = {
         "platforms": {},
+        "python_version": None,
     }
     for mod in module_ctx.modules:
         if not (mod.is_root or mod.name == "rules_python"):
             continue
 
         for tag in mod.tags.default:
+            pyproject_toml = getattr(tag, "pyproject_toml", None)
+            if pyproject_toml:
+                pyproject_version = read_pyproject_version(
+                    module_ctx,
+                    pyproject_toml,
+                    logger = None,
+                )
+                if pyproject_version:
+                    defaults["python_version"] = pyproject_version
+
             platform = tag.platform
             if platform:
                 specific_config = defaults["platforms"].setdefault(platform, {})
@@ -125,6 +137,7 @@ def build_config(
     return struct(
         auth_patterns = defaults.get("auth_patterns", {}),
         netrc = defaults.get("netrc", None),
+        python_version = defaults.get("python_version", None),
         platforms = {
             name: _plat(**values)
             for name, values in defaults["platforms"].items()
@@ -155,6 +168,8 @@ def parse_modules(
     Returns:
         A struct with the following attributes:
     """
+    config = build_config(module_ctx = module_ctx, enable_pipstar = enable_pipstar, enable_pipstar_extract = enable_pipstar_extract)
+
     whl_mods = {}
     for mod in module_ctx.modules:
         for whl_mod in mod.tags.whl_mods:
@@ -186,8 +201,6 @@ You cannot use both the additive_build_content and additive_build_content_file a
                 srcs_exclude_glob = whl_mod.srcs_exclude_glob,
             )
 
-    config = build_config(module_ctx = module_ctx, enable_pipstar = enable_pipstar, enable_pipstar_extract = enable_pipstar_extract)
-
     # TODO @aignas 2025-06-03: Merge override API with the builder?
     _overriden_whl_set = {}
     whl_overrides = {}
@@ -228,6 +241,10 @@ You cannot use both the additive_build_content and additive_build_content_file a
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
+            python_version = pip_attr.python_version or config.python_version
+            if not python_version:
+                _fail("pip.parse() requires either python_version attribute or pip.default(pyproject_toml=...) to be set")
+
             hub_name = pip_attr.hub_name
             if hub_name not in pip_hub_map:
                 builder = hub_builder(
@@ -264,6 +281,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
             builder.pip_parse(
                 module_ctx,
                 pip_attr = pip_attr,
+                python_version = python_version,
             )
 
     # Keeps track of all the hub's whl repos across the different versions.
@@ -409,7 +427,7 @@ Either this or {attr}`env` `platform_machine` key should be specified.
 """,
     ),
     "config_settings": attr.label_list(
-        mandatory = True,
+        mandatory = False,
         doc = """\
 The list of labels to `config_setting` targets that need to be matched for the platform to be
 selected.
@@ -471,6 +489,22 @@ If you are defining custom platforms in your project and don't want things to cl
 [isolation] feature.
 
 [isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate
+""",
+    ),
+    "pyproject_toml": attr.label(
+        mandatory = False,
+        allow_single_file = True,
+        doc = """\
+Label pointing to pyproject.toml file to read the default Python version from.
+When specified, reads the `requires-python` field from pyproject.toml and uses
+it as the default python_version for all `pip.parse()` calls that don't
+explicitly specify one.
+
+The version must be specified as `==X.Y.Z` (exact version with full semver).
+This is designed to work with dependency management tools like Renovate.
+
+:::{versionadded} 1.8.0
+:::
 """,
     ),
     "whl_abi_tags": attr.string_list(
@@ -651,7 +685,7 @@ find in case extra indexes are specified.
             default = True,
         ),
         "python_version": attr.string(
-            mandatory = True,
+            mandatory = False,
             doc = """
 The Python version the dependencies are targetting, in Major.Minor format
 (e.g., "3.11") or patch level granularity (e.g. "3.11.1").

python/private/pypi/hub_builder.bzl

@@ -147,8 +147,8 @@ def _build(self):
         whl_libraries = self._whl_libraries,
     )
 
-def _pip_parse(self, module_ctx, pip_attr):
-    python_version = pip_attr.python_version
+def _pip_parse(self, module_ctx, pip_attr, python_version = None):
+    python_version = python_version or pip_attr.python_version
     if python_version in self._platforms:
         fail((
             "Duplicate pip python version '{version}' for hub " +
@@ -197,8 +197,9 @@ def _pip_parse(self, module_ctx, pip_attr):
         self,
         module_ctx,
         pip_attr = pip_attr,
-        enable_pipstar = self._config.enable_pipstar or self._get_index_urls.get(pip_attr.python_version),
-        enable_pipstar_extract = self._config.enable_pipstar_extract or self._get_index_urls.get(pip_attr.python_version),
+        python_version = python_version,
+        enable_pipstar = self._config.enable_pipstar or self._get_index_urls.get(python_version),
+        enable_pipstar_extract = self._config.enable_pipstar_extract or self._get_index_urls.get(python_version),
     )
 
 ### end of PUBLIC methods
@@ -410,11 +411,11 @@ def _set_get_index_urls(self, pip_attr):
     )
     return True
 
-def _detect_interpreter(self, pip_attr):
+def _detect_interpreter(self, pip_attr, python_version):
     python_interpreter_target = pip_attr.python_interpreter_target
     if python_interpreter_target == None and not pip_attr.python_interpreter:
         python_name = "python_{}_host".format(
-            pip_attr.python_version.replace(".", "_"),
+            python_version.replace(".", "_"),
         )
         if python_name not in self._available_interpreters:
             fail((
@@ -424,7 +425,7 @@ def _detect_interpreter(self, pip_attr):
                 "Expected to find {python_name} among registered versions:\n  {labels}"
             ).format(
                 hub_name = self.name,
-                version = pip_attr.python_version,
+                version = python_version,
                 python_name = python_name,
                 labels = "  \n".join(self._available_interpreters),
             ))
@@ -488,17 +489,17 @@ def _platforms(module_ctx, *, python_version, config, target_platforms):
         )
     return platforms
 
-def _evaluate_markers(self, pip_attr, enable_pipstar):
+def _evaluate_markers(self, pip_attr, python_version, enable_pipstar):
     if self._evaluate_markers_fn:
         return self._evaluate_markers_fn
 
     if enable_pipstar:
         return lambda _, requirements: evaluate_markers_star(
             requirements = requirements,
-            platforms = self._platforms[pip_attr.python_version],
+            platforms = self._platforms[python_version],
         )
 
-    interpreter = _detect_interpreter(self, pip_attr)
+    interpreter = _detect_interpreter(self, pip_attr, python_version)
 
     # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
     # in the PATH or if specified as a label. We will configure the env
@@ -518,7 +519,7 @@ def _evaluate_markers(self, pip_attr, enable_pipstar):
         module_ctx,
         requirements = {
             k: {
-                p: self._platforms[pip_attr.python_version][p].triple
+                p: self._platforms[python_version][p].triple
                 for p in plats
             }
             for k, plats in requirements.items()
@@ -534,6 +535,7 @@ def _create_whl_repos(
         module_ctx,
         *,
         pip_attr,
+        python_version,
         enable_pipstar = False,
         enable_pipstar_extract = False):
     """create all of the whl repositories
@@ -542,11 +544,12 @@ def _create_whl_repos(
         self: the builder.
         module_ctx: {type}`module_ctx`.
         pip_attr: {type}`struct` - the struct that comes from the tag class iteration.
+        python_version: {type}`str` - the resolved python version for this pip.parse call.
         enable_pipstar: {type}`bool` - enable the pipstar or not.
         enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not.
     """
     logger = self._logger
-    platforms = self._platforms[pip_attr.python_version]
+    platforms = self._platforms[python_version]
     requirements_by_platform = parse_requirements(
         module_ctx,
         requirements_by_platform = requirements_files_by_platform(
@@ -558,15 +561,15 @@ def _create_whl_repos(
             extra_pip_args = pip_attr.extra_pip_args,
             platforms = sorted(platforms),  # here we only need keys
             python_version = full_version(
-                version = pip_attr.python_version,
+                version = python_version,
                 minor_mapping = self._minor_mapping,
             ),
             logger = logger,
         ),
         platforms = platforms,
         extra_pip_args = pip_attr.extra_pip_args,
-        get_index_urls = self._get_index_urls.get(pip_attr.python_version),
-        evaluate_markers = _evaluate_markers(self, pip_attr, enable_pipstar),
+        get_index_urls = self._get_index_urls.get(python_version),
+        evaluate_markers = _evaluate_markers(self, pip_attr, python_version, enable_pipstar),
         logger = logger,
     )
 
@@ -588,7 +591,7 @@ def _create_whl_repos(
         enable_pipstar = enable_pipstar,
     )
 
-    interpreter = _detect_interpreter(self, pip_attr)
+    interpreter = _detect_interpreter(self, pip_attr, python_version)
 
     for whl in requirements_by_platform:
         whl_library_args = common_args | _whl_library_args(
@@ -602,17 +605,17 @@ def _create_whl_repos(
                 whl_library_args = whl_library_args,
                 download_only = pip_attr.download_only,
                 netrc = self._config.netrc or pip_attr.netrc,
-                use_downloader = _use_downloader(self, pip_attr.python_version, whl.name),
+                use_downloader = _use_downloader(self, python_version, whl.name),
                 auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns,
-                python_version = _major_minor_version(pip_attr.python_version),
+                python_version = _major_minor_version(python_version),
                 is_multiple_versions = whl.is_multiple_versions,
                 interpreter = interpreter,
                 enable_pipstar = enable_pipstar,
                 enable_pipstar_extract = enable_pipstar_extract,
             )
             _add_whl_library(
                 self,
-                python_version = pip_attr.python_version,
+                python_version = python_version,
                 whl = whl,
                 repo = repo,
                 enable_pipstar = enable_pipstar,