Split 'build and publish' to 'build' and 'publish_and_verify'. Add (package) repository credentials provider. Extend protocol between WM and ER to request and return response in multiple formats at once.

Author: Aksem

.github/workflows/ci-cd.yml

@@ -1,9 +1,19 @@
+# TODO: check formatting
+# TODO: lock files on all platforms
+# TODO: test with all supported python versions
 name: CI
 
 on:
   merge_group:
   push:
   pull_request:
+  workflow_dispatch:
+    inputs:
+      publish_testpypi:
+        description: 'Publish to TestPyPI'
+        required: true
+        type: boolean
+        default: false
 
 defaults:
   run:
@@ -58,102 +68,61 @@ jobs:
           python -m finecode prepare-envs
         shell: bash
 
-      # TODO: install all other supported python versions. Version can be extracted from finecode
+      - name: Lint
+        run: |
+          source .venvs/dev_workspace/bin/activate
+          python -m finecode run lint
+        shell: bash
+
+      - name: Build artifacts
+        id: build
+        if: runner.os == 'Linux'
+        run: |
+          source .venvs/dev_workspace/bin/activate
+          python -m finecode run build_artifact
+        shell: bash
 
-      # - name: Lint
+      # - name: Run unit tests
+      #   if: ${{ !cancelled() }}
       #   run: |
-      #     poetry run python -m finecode run lint
+      #     source .venvs/dev_workspace/bin/activate
+      #     python -m finecode run test
       #   shell: bash
 
-      - name: Build all packages
-        if: runner.os == 'Linux'
+      - name: Publish to TestPyPI and verify
+        if: runner.os == 'Linux' && github.event_name == 'workflow_dispatch' && inputs.publish_testpypi
+        env:
+          FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__REPOSITORIES: '[{"name": "testpypi", "url": "https://test.pypi.org/"}]'
+          FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__CREDENTIALS_BY_REPOSITORY: '{"testpypi": {"username": "${{ secrets.TESTPYPI_USERNAME }}", "password": "${{ secrets.TESTPYPI_PASSWORD }}"}}'
         run: |
           source .venvs/dev_workspace/bin/activate
-          python -m finecode run build
+          python -m finecode run \
+            --map-payload-fields="src-artifact-def-path,dist-artifact-paths" \
+            publish_and_verify_artifact \
+            --src-artifact-def-path="build_artifact.src_artifact_def_path" \
+            --dist-artifact-paths="build_artifact.build_output_paths"
         shell: bash
 
-      - name: Collect all distribution packages
-        if: runner.os == 'Linux'
+      - name: Publish to PyPI and verify
+        if: runner.os == 'Linux' && startsWith(github.ref, 'refs/tags/')
+        env:
+          FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__REPOSITORIES: '[{"name": "pypi", "url": "https://pypi.org/"}]'
+          FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__CREDENTIALS_BY_REPOSITORY: '{"pypi": {"username": "${{ secrets.PYPI_USERNAME }}", "password": "${{ secrets.PYPI_PASSWORD }}"}}'
         run: |
-          # TODO: finecode action to copy only updated packages in dist
-          mkdir -p dist
-          cp finecode_extension_api/dist/* dist/
-          cp extensions/fine_python_ast/dist/* dist/
-          cp extensions/fine_python_black/dist/* dist/
-          cp extensions/fine_python_flake8/dist/* dist/
-          cp extensions/fine_python_isort/dist/* dist/
-          cp extensions/fine_python_module_exports/dist/* dist/
-          cp extensions/fine_python_mypy/dist/* dist/
-          cp presets/fine_python_format/dist/* dist/
-          cp presets/fine_python_lint/dist/* dist/
-          cp presets/fine_python_recommended/dist/* dist/
+          # TODO: make sure git tag exists (for manual trigger)
+          source .venvs/dev_workspace/bin/activate
+          python -m finecode run \
+            --map-payload-fields="src-artifact-def-path,dist-artifact-paths" \
+            publish_and_verify_artifact \
+            --src-artifact-def-path="build_artifact.src_artifact_def_path" \
+            --dist-artifact-paths="build_artifact.build_output_paths"
+
         shell: bash
 
+      # TODO: try to replace by finecode action
       - name: Store the distribution packages
         uses: actions/upload-artifact@v4
         if: runner.os == 'Linux'
         with:
           name: python-package-distributions
           path: dist/
-
-      # - name: Run unit tests
-      #   if: ${{ !cancelled() }}
-      #   run: |
-      #     poetry run python -m pytest tests/
-      #   shell: bash
-
-  publish-to-pypi:
-    name: >-
-      Publish Python 🐍 distribution 📦 to PyPI
-    if: startsWith(github.ref, 'refs/tags/')
-    needs:
-    - build
-    runs-on: ubuntu-24.04
-    environment:
-      name: pypi
-      url: https://pypi.org/p/finecode
-    permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        # temporary skip existing packages, because not always all packages at once
-        # are updated.
-        # TODO: implement publishing only of changed in finecode
-        skip-existing: true
-
-  publish-to-testpypi:
-    name: Publish Python 🐍 distribution 📦 to TestPyPI
-    needs:
-    - build
-    runs-on: ubuntu-24.04
-
-    environment:
-      name: testpypi
-      url: https://test.pypi.org/p/finecode
-
-    permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution 📦 to TestPyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        repository-url: https://test.pypi.org/legacy/
-        verbose: true
-        # temporary skip existing packages, because not always all packages at once
-        # are updated.
-        # TODO: implement publishing only of changed in finecode
-        skip-existing: true

extensions/fine_python_package_info/fine_python_package_info/__init__.py

@@ -1,4 +1,6 @@
 from .build_artifact_py_handler import BuildArtifactPyHandler
+from .get_dist_artifact_version_py_handler import \
+    GetDistArtifactVersionPyHandler
 from .get_src_artifact_registries_py_handler import \
     GetSrcArtifactRegistriesPyHandler
 from .get_src_artifact_version_py_handler import GetSrcArtifactVersionPyHandler
@@ -14,6 +16,7 @@
 
 __all__ = [
     "BuildArtifactPyHandler",
+    "GetDistArtifactVersionPyHandler",
     "GroupSrcArtifactFilesByLangPythonHandler",
     "ListSrcArtifactFilesByLangPythonHandler",
     "PyPackageLayoutInfoProvider",

extensions/fine_python_package_info/fine_python_package_info/build_artifact_py_handler.py

@@ -91,5 +91,6 @@ async def run(
         self.logger.info(f"Build completed. Output: {build_output_paths}")
 
         return build_artifact_action.BuildArtifactRunResult(
-            build_output_paths=build_output_paths
+            src_artifact_def_path=src_artifact_def_path,
+            build_output_paths=build_output_paths,
         )

extensions/fine_python_package_info/fine_python_package_info/get_dist_artifact_version_py_handler.py

@@ -0,0 +1,60 @@
+import dataclasses
+
+from finecode_extension_api import code_action
+from finecode_extension_api.actions import \
+    get_dist_artifact_version as get_dist_artifact_version_action
+from finecode_extension_api.interfaces import ilogger
+
+
+@dataclasses.dataclass
+class GetDistArtifactVersionPyHandlerConfig(code_action.ActionHandlerConfig): ...
+
+
+class GetDistArtifactVersionPyHandler(
+    code_action.ActionHandler[
+        get_dist_artifact_version_action.GetDistArtifactVersionAction,
+        GetDistArtifactVersionPyHandlerConfig,
+    ]
+):
+    def __init__(
+        self,
+        config: GetDistArtifactVersionPyHandlerConfig,
+        logger: ilogger.ILogger,
+    ) -> None:
+        self.config = config
+        self.logger = logger
+
+    async def run(
+        self,
+        payload: get_dist_artifact_version_action.GetDistArtifactVersionRunPayload,
+        run_context: get_dist_artifact_version_action.GetDistArtifactVersionRunContext,
+    ) -> get_dist_artifact_version_action.GetDistArtifactVersionRunResult:
+        filename = payload.dist_artifact_path.name
+        version = self._extract_version_from_filename(filename)
+
+        if version is None:
+            raise code_action.ActionFailedException(
+                f"Could not extract version from dist filename: {filename}"
+            )
+
+        return get_dist_artifact_version_action.GetDistArtifactVersionRunResult(
+            version=version
+        )
+
+    def _extract_version_from_filename(self, filename: str) -> str | None:
+        if filename.endswith('.whl'):
+            # Wheel: name-version-python-abi-platform.whl
+            parts = filename[:-4].split('-')
+            if len(parts) >= 5:
+                return parts[1]
+        elif filename.endswith('.tar.gz'):
+            # Source dist: name-version.tar.gz
+            parts = filename[:-7].split('-')
+            if len(parts) >= 2:
+                return parts[1]
+        elif filename.endswith('.zip'):
+            # Source dist: name-version.zip
+            parts = filename[:-4].split('-')
+            if len(parts) >= 2:
+                return parts[1]
+        return None

extensions/fine_python_package_info/fine_python_package_info/get_src_artifact_registries_py_handler.py

@@ -3,7 +3,10 @@
 from finecode_extension_api import code_action
 from finecode_extension_api.actions import \
     get_src_artifact_registries as get_src_artifact_registries_action
-from finecode_extension_api.interfaces import ilogger, iprojectinfoprovider
+from finecode_extension_api.interfaces import (
+    ilogger,
+    irepositorycredentialsprovider,
+)
 
 
 @dataclasses.dataclass
@@ -19,65 +22,24 @@ class GetSrcArtifactRegistriesPyHandler(
     def __init__(
         self,
         config: GetSrcArtifactRegistriesPyHandlerConfig,
-        project_info_provider: iprojectinfoprovider.IProjectInfoProvider,
+        repository_credentials_provider: irepositorycredentialsprovider.IRepositoryCredentialsProvider,
         logger: ilogger.ILogger,
     ) -> None:
         self.config = config
-        self.project_info_provider = project_info_provider
+        self.repository_credentials_provider = repository_credentials_provider
         self.logger = logger
 
     async def run(
         self,
         payload: get_src_artifact_registries_action.GetSrcArtifactRegistriesRunPayload,
         run_context: get_src_artifact_registries_action.GetSrcArtifactRegistriesRunContext,
     ) -> get_src_artifact_registries_action.GetSrcArtifactRegistriesRunResult:
-        src_artifact_raw_def = await self.project_info_provider.get_project_raw_config(
-            project_def_path=payload.src_artifact_def_path
-        )
-
-        # Registries are in tool.finecode.registries
-        tool_config = src_artifact_raw_def.get("tool", {})
-        finecode_config = tool_config.get("finecode", {})
-        registries_raw = finecode_config.get("registries", [])
-
-        if not isinstance(registries_raw, list):
-            raise code_action.ActionFailedException(
-                f"tool.finecode.registries in {payload.src_artifact_def_path} expected to be a list, but is {type(registries_raw)}"
-            )
-
-        registries = []
-        for idx, registry_dict in enumerate(registries_raw):
-            if not isinstance(registry_dict, dict):
-                raise code_action.ActionFailedException(
-                    f"Registry at index {idx} in {payload.src_artifact_def_path} expected to be a dict, but is {type(registry_dict)}"
-                )
-
-            url = registry_dict.get("url")
-            name = registry_dict.get("name")
-
-            if url is None:
-                raise code_action.ActionFailedException(
-                    f"Registry at index {idx} in {payload.src_artifact_def_path} is missing 'url' field"
-                )
-
-            if name is None:
-                raise code_action.ActionFailedException(
-                    f"Registry at index {idx} in {payload.src_artifact_def_path} is missing 'name' field"
-                )
-
-            if not isinstance(url, str):
-                raise code_action.ActionFailedException(
-                    f"Registry url at index {idx} in {payload.src_artifact_def_path} expected to be a string, but is {type(url)}"
-                )
-
-            if not isinstance(name, str):
-                raise code_action.ActionFailedException(
-                    f"Registry name at index {idx} in {payload.src_artifact_def_path} expected to be a string, but is {type(name)}"
-                )
+        repositories = self.repository_credentials_provider.get_all_repositories()
 
-            registries.append(
-                get_src_artifact_registries_action.Registry(url=url, name=name)
-            )
+        registries = [
+            get_src_artifact_registries_action.Registry(url=repo.url, name=repo.name)
+            for repo in repositories
+        ]
 
         return get_src_artifact_registries_action.GetSrcArtifactRegistriesRunResult(
             registries=registries

extensions/fine_python_package_info/fine_python_package_info/is_artifact_published_to_registry_py_handler.py

@@ -88,8 +88,8 @@ async def run(
                 f"Registry '{payload.registry_name}' not found in configuration"
             )
 
-        # Check if package version exists using PyPI JSON API
-        check_url = f"{registry_url.rstrip('/')}/{package_name}/"
+        # Check if package version exists using PyPI Simple API
+        check_url = f"{registry_url.rstrip('/')}/simple/{package_name}/"
 
         self.logger.debug(
             f"Checking if {package_name} {payload.version} is published to {payload.registry_name} at {check_url}"
@@ -98,12 +98,19 @@ async def run(
         try:
             async with self.http_client.session() as session:
                 response = await session.get(check_url, headers={"Accept": "application/vnd.pypi.simple.v1+json"}, timeout=10.0)
-                response_json = response.json()
         except Exception as exception:
             raise code_action.ActionFailedException(
                 f"Error checking publication status: {str(exception)}"
             ) from exception
-        
+
+        if response.status_code == 404:
+            # Package does not exist in the registry yet
+            is_published_by_dist_path = {dist_path: False for dist_path in payload.dist_artifact_paths}
+            return is_artifact_published_to_registry_action.IsArtifactPublishedToRegistryRunResult(
+                is_published_by_dist_path=is_published_by_dist_path
+            )
+
+        response_json = response.json()
         version_list = response_json.get('versions', None)
         if version_list is None:
             raise code_action.ActionFailedException("No 'versions' key in response from registry")