second last review

Author: lorenzorubi-db

knowledge_base/pydabs_job_with_for_each/databricks.yml

@@ -2,7 +2,6 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: pydabs_job_with_foreach
-  uuid: 3874a19c-7ea5-401d-bca2-9bd1f9d3efbf
 
 python:
   venv_path: .venv

knowledge_base/pydabs_job_with_for_each/resources/for_each_simple.py

@@ -1,16 +1,8 @@
-from databricks.bundles.jobs import (
-    Job,
-    Task,
-    NotebookTask,
-    ForEachTask,
-    TaskDependency,
-    JobEnvironment,
-    Environment,
-)
+from databricks.bundles.jobs import Job, Task, NotebookTask, ForEachTask, TaskDependency
 
 generate_items = Task(
     task_key="generate_items",
-    notebook_task=NotebookTask(notebook_path="src/foreach/generate_items.ipynb"),
+    notebook_task=NotebookTask(notebook_path="src/foreach/generate_items.py"),
 )
 
 process_item = Task(
@@ -20,7 +12,7 @@
         task=Task(
             task_key="process_item_iteration",
             notebook_task=NotebookTask(
-                notebook_path="src/foreach/process_item.ipynb",
+                notebook_path="src/foreach/process_item.py",
                 base_parameters={"item": "{{input}}"},
             ),
         ),

knowledge_base/pydabs_job_with_for_each/src/foreach/generate_items.ipynb

@@ -0,0 +1,5 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+
+items = [1, 2, 3]
+dbutils.jobs.taskValues.set(key="items", value=items)

knowledge_base/pydabs_job_with_for_each/src/foreach/generate_items.py

@@ -0,0 +1,8 @@
+# Databricks notebook source
+
+# Runs once per item in the for-each. Do not call dbutils.jobs.taskValues.set() here.\n",
+from databricks.sdk.runtime import dbutils
+
+# Current iteration value passed from the for-each task (base_parameters: item = {{input}})\n",
+current_item = dbutils.widgets.get("item")
+print(f"Processing item: {current_item}")

knowledge_base/pydabs_job_with_for_each/src/foreach/process_item.ipynb

@@ -2,7 +2,6 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: pydabs_job_with_task_values
-  uuid: 3874a19c-7ea5-401d-bca2-9bd1f9d3efbf
 
 python:
   venv_path: .venv

knowledge_base/pydabs_job_with_for_each/src/foreach/process_item.py

@@ -1,3 +1,4 @@
+# Databricks notebook source
 from databricks.sdk.runtime import dbutils
 
 val = dbutils.jobs.taskValues.get(taskKey="producer", key="answer", debugValue=None)

knowledge_base/pydabs_job_with_task_values/databricks.yml

@@ -1,3 +1,4 @@
+# Databricks notebook source
 from databricks.sdk.runtime import dbutils
 
 value = 42