update file arrival trigger example

Author: zanitarahimi

knowledge_base/pydabs_job_file_arrival/README.md

@@ -10,7 +10,7 @@ The Lakeflow Job is configured with:
 - **Automatic processing**: When files are detected, the job automatically runs and processes them
 
 * `src/`: Notebook source code for this project.
-  * `src/process_files.ipynb`: Processes newly arrived files from the volume path.
+  * `src/process_files.py`: Processes newly arrived files from the volume path.
 * `resources/`:  Resource configurations (jobs, pipelines, etc.)
   * `resources/file_arrival.py`: PyDABs job with file arrival trigger configuration.
 

knowledge_base/pydabs_job_file_arrival/resources/file_arrival.py

@@ -1,29 +1,16 @@
-from databricks.bundles.jobs import (
-    FileArrivalTriggerConfiguration,
-    Job,
-    NotebookTask,
-    Task,
-    TriggerSettings,
+from databricks.bundles.jobs import FileArrivalTriggerConfiguration, Job, Task, NotebookTask
+
+process_files = Task(
+    task_key="process_files",
+    notebook_task=NotebookTask(notebook_path="src/files/process_files.py"),
 )
 
-pydabs_job_file_arrival = Job(
-    name="pydabs_job_file_arrival",
-    tasks=[
-        Task(
-            task_key="process_new_files",
-            notebook_task=NotebookTask(
-                notebook_path="src/process_files.ipynb",
-                base_parameters={
-                    "file_arrival_location": "{{job.trigger.file_arrival.location}}"
-                },
-            ),
-        )
-    ],
-    trigger=TriggerSettings(
-        file_arrival=FileArrivalTriggerConfiguration(
-            url="/Volumes/your_catalog/your_schema/your_volume/",
-            min_time_between_triggers_seconds=60,
-            wait_after_last_change_seconds=90,
-        ),
+job = Job(
+    name="file_arrival_example",
+    trigger=FileArrivalTriggerConfiguration(
+        url="Volumes/main.raw.incoming",  # UC volume or external location
+        min_time_between_triggers_seconds=60,
+        wait_after_last_change_seconds=90,
     ),
+    tasks=[process_files],
 )

knowledge_base/pydabs_job_file_arrival/src/process_files.ipynb

@@ -0,0 +1,5 @@
+from pyspark.sql import functions as F
+
+df = spark.read.format("cloudFiles") \
+    .option("cloudFiles.format", "csv") \
+    .load("/Volumes/main/raw/incoming")