feat: source resolver configuration

Introduce new configuration for source resolver and downloads. The new
system uses profiles for common tasks:

- `pypi-sdist`: resolve versions of sdists from PyPI, download sdist
- `pypi-prebuilt`: resolve versions of platform wheels from PyPI,
  download pre-built wheel from PyPI.
- `pypi-download`: resolve versions of any package from PyPI, download
  from external URL (with `{version}` variable)
- `pypi-git`: resolve versions for any package from PyPI, git clone
  (with `{version}` variable)
- `versionmap-git`: resolve with a version map, git clone
- `github-tag-download`: resolve from GitHub tag, download tarball
- `github-tag-git`: resolve from GitHub tag, git clone
- `gitlab-tag-download`: resolve from GitLab tag, download tarball
- `gitlab-tag-git`: resolve from GitLab tag, git clone
- `not-available`: block resolving
- `hook`: call Fromager hooks

The new settings will eventually replace `download_source`,
`resolver_dist`, and `git_options` top-level options as well as
`wheel_server_url` and `pre_built` flags for variants.

Signed-off-by: Christian Heimes <cheimes@redhat.com>

Author: tiran

docs/reference/config-reference.rst

@@ -1,6 +1,8 @@
 Configuration Reference
 =======================
 
+.. currentmodule:: fromager.packagesettings
+
 Per-package Settings
 --------------------
 
@@ -24,6 +26,45 @@ For example `flash_attn.yaml`.
 
 .. autopydantic_model:: fromager.packagesettings.SbomSettings
 
+Source Resolver
+^^^^^^^^^^^^^^^
+
+.. autopydantic_model:: fromager.packagesettings.PyPISDistResolver
+   :inherited-members: AbstractPyPIResolver
+
+.. autopydantic_model:: fromager.packagesettings.PyPIPrebuiltResolver
+   :inherited-members: AbstractPyPIResolver
+
+.. autopydantic_model:: fromager.packagesettings.PyPIDownloadResolver
+   :inherited-members: AbstractPyPIResolver
+
+.. autopydantic_model:: fromager.packagesettings.PyPIGitResolver
+   :inherited-members: AbstractPyPIResolver
+
+.. autopydantic_model:: fromager.packagesettings.VersionMapResolver
+
+.. autopydantic_model:: fromager.packagesettings.GitHubTagDownloadResolver
+   :inherited-members: AbstractGitSourceResolver
+
+.. autopydantic_model:: fromager.packagesettings.GitHubTagCloneResolver
+   :inherited-members: AbstractGitSourceResolver
+
+.. autopydantic_model:: fromager.packagesettings.GitLabTagDownloadResolver
+   :inherited-members: AbstractGitSourceResolver
+
+.. autopydantic_model:: fromager.packagesettings.GitLabTagCloneResolver
+   :inherited-members: AbstractGitSourceResolver
+
+.. autopydantic_model:: fromager.packagesettings.NotAvailableResolver
+
+.. autopydantic_model:: fromager.packagesettings.HookResolver
+
+.. autoclass:: fromager.packagesettings.BuildSDist
+
+   .. autoattribute:: pep517
+   .. autoattribute:: tarball
+
+
 Global Settings
 ---------------
 

src/fromager/packagesettings/__init__.py

@@ -13,6 +13,21 @@
     VariantInfo,
 )
 from ._pbi import PackageBuildInfo
+from ._resolver import (
+    DEFAULT_TAG_MATCHER,
+    BuildSDist,
+    GitHubTagCloneResolver,
+    GitHubTagDownloadResolver,
+    GitLabTagCloneResolver,
+    GitLabTagDownloadResolver,
+    HookResolver,
+    NotAvailableResolver,
+    PyPIDownloadResolver,
+    PyPIGitResolver,
+    PyPIPrebuiltResolver,
+    PyPISDistResolver,
+    VersionMapResolver,
+)
 from ._settings import Settings, SettingsFile
 from ._templates import substitute_template
 from ._typedefs import (
@@ -32,22 +47,34 @@
 )
 
 __all__ = (
+    "DEFAULT_TAG_MATCHER",
     "MODEL_CONFIG",
     "Annotations",
     "BuildDirectory",
     "BuildOptions",
+    "BuildSDist",
     "DownloadSource",
     "EnvKey",
     "EnvVars",
+    "GitHubTagCloneResolver",
+    "GitHubTagDownloadResolver",
+    "GitLabTagCloneResolver",
+    "GitLabTagDownloadResolver",
     "GitOptions",
     "GlobalChangelog",
+    "HookResolver",
+    "NotAvailableResolver",
     "Package",
     "PackageBuildInfo",
     "PackageSettings",
     "PackageVersion",
     "PatchMap",
     "ProjectOverride",
     "PurlConfig",
+    "PyPIDownloadResolver",
+    "PyPIGitResolver",
+    "PyPIPrebuiltResolver",
+    "PyPISDistResolver",
     "RawAnnotations",
     "ResolverDist",
     "SbomSettings",
@@ -57,6 +84,7 @@
     "Variant",
     "VariantChangelog",
     "VariantInfo",
+    "VersionMapResolver",
     "default_update_extra_environ",
     "get_extra_environ",
     "substitute_template",

src/fromager/packagesettings/_models.py

@@ -16,6 +16,7 @@
 from pydantic import Field
 from pydantic_core import core_schema
 
+from ._resolver import SourceResolver
 from ._typedefs import (
     MODEL_CONFIG,
     BuildDirectory,
@@ -313,7 +314,7 @@ class VariantInfo(pydantic.BaseModel):
         VAR1: "value 1"
         VAR2: "2.0
       wheel_server_url: https://pypi.org/simple/
-      pre_build: False
+      pre_built: False
     """
 
     model_config = MODEL_CONFIG
@@ -334,6 +335,12 @@ class VariantInfo(pydantic.BaseModel):
     pre_built: bool = False
     """Use pre-built wheel from index server?"""
 
+    source: typing.Annotated[
+        SourceResolver | None,
+        pydantic.Field(default=None, discriminator="provider"),
+    ]
+    """Source resolver and downloader"""
+
 
 class GitOptions(pydantic.BaseModel):
     """Git repository cloning options
@@ -458,6 +465,12 @@ class PackageSettings(pydantic.BaseModel):
     project_override: ProjectOverride = Field(default_factory=ProjectOverride)
     """Patch project settings"""
 
+    source: typing.Annotated[
+        SourceResolver | None,
+        pydantic.Field(default=None, discriminator="provider"),
+    ]
+    """Source resolver and downloader"""
+
     variants: Mapping[Variant, VariantInfo] = Field(default_factory=dict)
     """Variant configuration"""
 

src/fromager/packagesettings/_pbi.py

@@ -21,6 +21,7 @@
     PurlConfig,
     VariantInfo,
 )
+from ._resolver import SourceResolver
 from ._templates import _resolve_template, substitute_template
 from ._typedefs import Annotations, PackageVersion, PatchMap, Template, Variant
 
@@ -176,6 +177,14 @@ def pre_built(self) -> bool:
             return vi.pre_built
         return False
 
+    @property
+    def source_resolver(self) -> SourceResolver | None:
+        """Get source resolver settings (variant or global)"""
+        vi = self._ps.variants.get(self.variant)
+        if vi is not None and vi.source is not None:
+            return vi.source
+        return self._ps.source
+
     @property
     def wheel_server_url(self) -> str | None:
         """Alternative package index for pre-build wheel"""