Merge branch 'master' into develop

Author: Panaetius

CHANGES.rst

@@ -30,6 +30,8 @@ Bug Fixes
 -  **cli:** fix special paths in workflow files and bump `toil` / `cwltool`
    (`#3489 <https://github.com/SwissDataScienceCenter/renku-python/issues/3489>`__)
    (`28086cf <https://github.com/SwissDataScienceCenter/renku-python/commit/28086cf1361c86109c8e0e1c59c5704a5a663f30>`__)
+- **cli:**: fix wrong plan ids in plans coming from workflow files
+   (`#3511 <https://github.com/SwissDataScienceCenter/renku-python/pull/3511>`__)
 -  **service:** fix working with branches
    (`#3472 <https://github.com/SwissDataScienceCenter/renku-python/issues/3472>`__)
    (`0eaf204 <https://github.com/SwissDataScienceCenter/renku-python/commit/0eaf204365d38bbf82bd2d0df357abbf61c18548>`__)

renku/command/schema/workflow_file.py

@@ -15,6 +15,9 @@
 # limitations under the License.
 """Represent workflow file run templates."""
 
+from itertools import chain
+from typing import List, Union
+
 import marshmallow
 
 from renku.command.schema.calamus import fields, prov, renku, schema
@@ -33,6 +36,26 @@ class Meta:
         model = WorkflowFilePlan
         unknown = marshmallow.EXCLUDE
 
+    @marshmallow.pre_dump(pass_many=True)
+    def fix_ids(self, objs: Union[WorkflowFilePlan, List[WorkflowFilePlan]], many, **kwargs):
+        """Renku up to 2.4.1 had a bug that created wrong ids for workflow file entities, this fixes those on export."""
+
+        def _replace_id(obj):
+            obj.unfreeze()
+            obj.id = obj.id.replace("//plans/", "/")
+
+            for child in chain(obj.inputs, obj.outputs, obj.parameters):
+                child.id = child.id.replace("//plans/", "/")
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_id(obj)
+            return objs
+
+        _replace_id(objs)
+        return objs
+
 
 class WorkflowFileCompositePlanSchema(CompositePlanSchema):
     """Plan schema."""
@@ -46,3 +69,20 @@ class Meta:
 
     path = fields.String(prov.atLocation)
     plans = fields.Nested(renku.hasSubprocess, WorkflowFilePlanSchema, many=True)
+
+    @marshmallow.pre_dump(pass_many=True)
+    def fix_ids(self, objs, many, **kwargs):
+        """Renku up to 2.4.1 had a bug that created wrong ids for workflow file entities, this fixes those on export."""
+
+        def _replace_id(obj):
+            obj.unfreeze()
+            obj.id = obj.id.replace("//plans/", "/")
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_id(obj)
+            return objs
+
+        _replace_id(objs)
+        return objs

renku/domain_model/workflow/workflow_file.py

@@ -37,19 +37,30 @@ def __init__(self, *, path: Union[Path, str], **kwargs):
         self.path: str = str(path)
 
     @staticmethod
-    def generate_id(path: Optional[Union[Path, str]] = None, sequence: Optional[int] = None, **_) -> str:
+    def generate_id(
+        path: Optional[Union[Path, str]] = None, sequence: Optional[int] = None, uuid_only: bool = False, **_
+    ) -> str:
         """Generate an identifier for Plan."""
         assert path, "Path is needed to generate id for WorkflowFileCompositePlan"
 
         # NOTE: Workflow file's root composite plan's ID is generated only based on the file's path. The ID might be
         # changed later if the plan is a derivative
         key = f"{path}" if sequence is None else f"{path}::{sequence}"
         key_bytes = key.encode("utf-8")
-        return CompositePlan.generate_id(uuid=hashlib.md5(key_bytes).hexdigest()[:32])  # nosec
+
+        uuid = hashlib.md5(key_bytes).hexdigest()[:32]  # nosec
+
+        if uuid_only:
+            return uuid
+        return CompositePlan.generate_id(uuid=uuid)
 
     def assign_new_id(self, *, sequence: Optional[int] = None, **_) -> str:
         """Assign a new UUID or a deterministic."""
-        new_id = uuid.uuid4().hex if sequence is None else self.generate_id(path=self.path, sequence=sequence)
+        new_id = (
+            uuid.uuid4().hex
+            if sequence is None
+            else self.generate_id(path=self.path, sequence=sequence, uuid_only=True)
+        )
         return super().assign_new_id(uuid=new_id)
 
     def is_equal_to(self, other: WorkflowFileCompositePlan) -> bool:
@@ -67,15 +78,22 @@ def __init__(self, *, path: Union[Path, str], **kwargs):
 
     @staticmethod
     def generate_id(
-        path: Optional[Union[Path, str]] = None, name: Optional[str] = None, sequence: Optional[int] = None, **_
+        path: Optional[Union[Path, str]] = None,
+        name: Optional[str] = None,
+        sequence: Optional[int] = None,
+        uuid_only: bool = False,
+        **_,
     ) -> str:
         """Generate an identifier for Plan."""
         assert path, "Path is needed to generate id for WorkflowFilePlan"
         assert name, "Name is needed to generate id for WorkflowFilePlan"
 
         key = f"{path}::{name}" if sequence is None else f"{path}::{name}::{sequence}"
         key_bytes = key.encode("utf-8")
-        return Plan.generate_id(uuid=hashlib.md5(key_bytes).hexdigest()[:32])  # nosec
+        uuid = hashlib.md5(key_bytes).hexdigest()[:32]  # nosec
+        if uuid_only:
+            return uuid
+        return Plan.generate_id(uuid=uuid)
 
     @staticmethod
     def validate_name(name: str):
@@ -92,7 +110,7 @@ def assign_new_id(self, *, sequence: Optional[int] = None, **_) -> str:
         new_id = (
             uuid.uuid4().hex
             if sequence is None
-            else self.generate_id(path=self.path, name=self.name, sequence=sequence)
+            else self.generate_id(path=self.path, name=self.name, sequence=sequence, uuid_only=True)
         )
         return super().assign_new_id(uuid=new_id)