Django to 6.0 support

.envs/.local/.django

@@ -11,3 +11,4 @@ CELERY_RESULT_BACKEND="redis://redis:6379/0"
 CELERY_FLOWER_USER=debug
 CELERY_FLOWER_PASSWORD=debug
 
+DJANGO_STRUCTLOG_CACHE="redis://redis:6379/0"

.github/workflows/main.yml

@@ -32,6 +32,7 @@ jobs:
       - name: Test demo app
         env:
           CELERY_BROKER_URL: redis://0.0.0.0:6379
+          DJANGO_STRUCTLOG_CACHE: redis://0.0.0.0:6379
           DJANGO_SETTINGS_MODULE: config.settings.test_demo_app
         run: pytest --cov=./django_structlog_demo_project --cov-append django_structlog_demo_project
       - uses: codecov/codecov-action@v5

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
         args: [--no-incremental]
         additional_dependencies: [
             celery-types==0.23.0,
-            "django-stubs[compatible-mypy]==5.2.5",
+            "django-stubs[compatible-mypy]==5.2.8",
             structlog==25.2.0,
             django-extensions==4.1,
             django-ipware==7.0.1,

README.rst

@@ -328,7 +328,7 @@ In your formatters, add the ``foreign_pre_chain`` section, and then add ``struct
                ],
            },
        },
-       ...
+       #...
     }
 
 
@@ -357,7 +357,7 @@ Flat lines file (\ ``logs/flat_lines.log``\ )
 Json file (\ ``logs/json.log``\ )
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. code-block:: json
+.. code-block:: jsonl
 
    {"request_id": "c53dff1d-3fc5-4257-a78a-9a567c937561", "user_id": 1, "ip": "0.0.0.0", "request": "GET /", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", "event": "request_started", "timestamp": "2019-04-13T19:39:29.321453Z", "logger": "django_structlog.middlewares.request", "level": "info"}
    {"request_id": "c53dff1d-3fc5-4257-a78a-9a567c937561", "user_id": 1, "ip": "0.0.0.0", "code": 200, "event": "request_finished", "timestamp": "2019-04-13T19:39:29.345207Z", "logger": "django_structlog.middlewares.request", "level": "info"}
@@ -624,7 +624,7 @@ Changes you need to do
                ],
            },
        },
-       ...
+       #...
     }
 
 
@@ -662,8 +662,8 @@ Note: For the moment redis is needed to run the tests. The easiest way is to sta
 
    docker compose up -d redis
    pip install -r requirements.txt
-   env CELERY_BROKER_URL=redis://0.0.0.0:6379 DJANGO_SETTINGS_MODULE=config.settings.test pytest test_app
-   env CELERY_BROKER_URL=redis://0.0.0.0:6379 DJANGO_SETTINGS_MODULE=config.settings.test_demo_app pytest django_structlog_demo_project
+   env DJANGO_SETTINGS_MODULE=config.settings.test pytest test_app
+   env DJANGO_SETTINGS_MODULE=config.settings.test_demo_app pytest django_structlog_demo_project
    docker compose stop redis
 
 .. inclusion-marker-running-tests-end

compose/local/django/entrypoint

@@ -4,12 +4,6 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-
-
-# N.B. If only .env files supported variable expansion...
-export CELERY_BROKER_URL="${CELERY_BROKER_URL}"
-
-
 if [ -z "${POSTGRES_USER}" ]; then
     base_postgres_image_default_user='postgres'
     export POSTGRES_USER="${base_postgres_image_default_user}"

config/settings/base.py

@@ -263,8 +263,27 @@ def serve(self, request):
 # https://django-allauth.readthedocs.io/en/latest/configuration.html
 ACCOUNT_ADAPTER = "django_structlog_demo_project.users.adapters.AccountAdapter"
 
-
-# Your stuff...
+# Django's Tasks Framework
 # ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/topics/tasks/
+TASKS = {
+    "default": {
+        # This task backend runs synchronously on the same thread.
+        # This makes it appear that "django_structlog" works with Django task framework,
+        # but it's only because they are in the same thread-local context.
+        "BACKEND": "django.tasks.backends.immediate.ImmediateBackend",
+    },
+}
+
+CACHES = {
+    "default": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+    },
+    "django_structlog": {
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "LOCATION": env("DJANGO_STRUCTLOG_CACHE"),
+    },
+}
+
 
 INSTALLED_APPS += ["django_structlog"]

config/settings/local.py

@@ -18,16 +18,6 @@
 
 IS_WORKER = env.bool("IS_WORKER", default=False)
 
-# CACHES
-# ------------------------------------------------------------------------------
-# https://docs.djangoproject.com/en/dev/ref/settings/#caches
-CACHES = {
-    "default": {
-        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
-        "LOCATION": "",
-    }
-}
-
 # TEMPLATES
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#templates
@@ -168,3 +158,4 @@
 
 DJANGO_STRUCTLOG_CELERY_ENABLED = True
 DJANGO_STRUCTLOG_COMMAND_LOGGING_ENABLED = True
+DJANGO_STRUCTLOG_DJANGO_TASKS_ENABLED = True

config/settings/test.py

@@ -47,7 +47,11 @@
     "default": {
         "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
         "LOCATION": "",
-    }
+    },
+    "django_structlog": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "LOCATION": "",
+    },
 }
 
 # EMAIL

config/urls.py

@@ -20,6 +20,12 @@ def uncaught_exception_view(request):
     ),
     re_path(r"^failing_task$", views.enqueue_failing_task, name="enqueue_failing_task"),
     re_path(r"^nesting_task$", views.enqueue_nesting_task, name="enqueue_nesting_task"),
+    re_path(r"^django_task$", views.enqueue_django_task, name="enqueue_django_task"),
+    re_path(
+        r"^django_failing_task$",
+        views.enqueue_django_failing_task,
+        name="enqueue_django_failing_task",
+    ),
     re_path(r"^unknown_task$", views.enqueue_unknown_task, name="enqueue_unknown_task"),
     re_path(
         r"^rejected_task$", views.enqueue_rejected_task, name="enqueue_rejected_task"

django_structlog/app_settings.py

@@ -11,6 +11,10 @@ class AppSettings:
     def CELERY_ENABLED(self) -> bool:
         return getattr(settings, self.PREFIX + "CELERY_ENABLED", False)
 
+    @property
+    def DJANGO_TASKS_ENABLED(self) -> bool:
+        return getattr(settings, self.PREFIX + "DJANGO_TASKS_ENABLED", False)
+
     @property
     def IP_LOGGING_ENABLED(self) -> bool:
         return getattr(settings, self.PREFIX + "IP_LOGGING_ENABLED", True)

django_structlog/apps.py

@@ -18,3 +18,9 @@ def ready(self) -> None:
 
             self._django_command_receiver = DjangoCommandReceiver()
             self._django_command_receiver.connect_signals()
+
+        if app_settings.DJANGO_TASKS_ENABLED:
+            from .tasks.receivers import DjangoTaskReceiver
+
+            self._django_task_receiver = DjangoTaskReceiver()
+            self._django_task_receiver.connect_signals()

django_structlog/celery/receivers.py

@@ -15,6 +15,7 @@
     task_unknown,
 )
 
+from ..tasks.receivers import BaseTaskReceiver
 from . import signals
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -23,7 +24,7 @@
 logger = structlog.getLogger(__name__)
 
 
-class CeleryReceiver:
+class CeleryReceiver(BaseTaskReceiver):
     _priority: Optional[str]
 
     def __init__(self) -> None:
@@ -41,10 +42,7 @@ def receiver_before_task_publish(
         if current_app.conf.task_protocol < 2:
             return
 
-        context = structlog.contextvars.get_merged_contextvars(logger)
-        if "task_id" in context:
-            context["parent_task_id"] = context.pop("task_id")
-
+        context = self.get_task_context()
         signals.modify_context_before_task_publish.send(
             sender=self.receiver_before_task_publish,
             context=context,
@@ -90,7 +88,7 @@ def receiver_task_prerun(
         structlog.contextvars.clear_contextvars()
         structlog.contextvars.bind_contextvars(task_id=task_id)
         metadata = getattr(task.request, "__django_structlog__", {})
-        structlog.contextvars.bind_contextvars(**metadata)
+        self.bind_context(metadata)
         signals.bind_extra_task_metadata.send(
             sender=self.receiver_task_prerun, task=task, logger=logger
         )

django_structlog/tasks/receivers.py

@@ -0,0 +1,102 @@
+from typing import TYPE_CHECKING, Any, Type
+
+import django
+import structlog
+from django.core.cache import caches
+
+from django_structlog.tasks import signals
+
+logger = structlog.getLogger(__name__)
+
+if TYPE_CHECKING:  # pragma: no cover
+    if django.VERSION >= (6, 0):
+        from django.tasks.base import TaskResult  # type: ignore[import-untyped]
+    else:
+        TaskResult = Any
+
+
+class BaseTaskReceiver:
+    def get_task_context(self) -> dict[str, Any]:
+        context = structlog.contextvars.get_merged_contextvars(logger)
+        if "task_id" in context:
+            context["parent_task_id"] = context.pop("task_id")
+        return context
+
+    def bind_context(self, context: dict[Any, Any] | Any) -> None:
+        structlog.contextvars.bind_contextvars(**context)
+
+
+class DjangoTaskReceiver(BaseTaskReceiver):
+
+    def __init__(self) -> None:
+        self.cache = caches["django_structlog"]
+
+    def receiver_task_enqueued(
+        self,
+        sender: Type[Any],
+        task_result: "TaskResult",
+        **kwargs: dict[str, str],
+    ) -> None:
+        context = self.get_task_context()
+        logger.info(
+            "task_enqueued",
+            task_id=task_result.id,
+            task_name=task_result.task.module_path,
+        )
+        signals.modify_context_before_task_publish.send(
+            sender=self.receiver_task_enqueued,
+            context=context,
+        )
+        self.cache.set(task_result.id, context)
+
+    def receiver_task_started(
+        self,
+        sender: Type[Any],
+        task_result: "TaskResult",
+        **kwargs: dict[str, str],
+    ) -> None:
+        structlog.contextvars.clear_contextvars()
+        structlog.contextvars.bind_contextvars(task_id=task_result.id)
+        context = self.cache.get(task_result.id, default={})
+        self.bind_context(context)
+        signals.bind_extra_task_metadata.send(
+            sender=self.receiver_task_started, task_result=task_result, logger=logger
+        )
+        logger.info(
+            "task_started",
+            task=task_result.task.module_path,
+        )
+
+    def receiver_task_finished(
+        self,
+        sender: Type[Any],
+        task_result: "TaskResult",
+        **kwargs: dict[str, str],
+    ) -> None:
+        from django.tasks.base import TaskResultStatus
+
+        log_vars: dict[str, Any] = {}
+        if task_result.status == TaskResultStatus.SUCCESSFUL:
+            signals.pre_task_succeeded.send(
+                sender=self.receiver_task_finished,
+                logger=logger,
+                task_result=task_result,
+            )
+            logger.info("task_succeeded", **log_vars)
+        elif task_result.status == TaskResultStatus.FAILED:
+            if task_result.errors:
+                last_error = task_result.errors[-1]  # Get the last error
+                log_vars["exception_class"] = last_error.exception_class_path
+                log_vars["traceback"] = last_error.traceback
+            logger.error("task_failed", **log_vars)
+
+    def connect_signals(self) -> None:
+        from django.tasks.signals import (  # type: ignore[import-untyped]
+            task_enqueued,
+            task_finished,
+            task_started,
+        )
+
+        task_started.connect(self.receiver_task_started)
+        task_finished.connect(self.receiver_task_finished)
+        task_enqueued.connect(self.receiver_task_enqueued)

django_structlog/tasks/signals.py

@@ -0,0 +1,57 @@
+import django.dispatch
+
+bind_extra_task_metadata = django.dispatch.Signal()
+""" Signal to add extra ``structlog`` bindings from ``celery``'s task.
+
+:param task: the celery task being run
+:param logger: the logger to bind more metadata or override existing bound metadata
+
+>>> from django.dispatch import receiver
+>>> from django_structlog.celery import signals
+>>> import structlog
+>>>
+>>> @receiver(signals.bind_extra_task_metadata)
+... def receiver_bind_extra_task_metadata(sender, signal, task=None, logger=None, **kwargs):
+...     structlog.contextvars.bind_contextvars(correlation_id=task.request.correlation_id)
+
+"""
+
+
+modify_context_before_task_publish = django.dispatch.Signal()
+""" Signal to modify context passed over to ``celery`` task's context. You must modify the ``context`` dict.
+
+:param context: the context dict that will be passed over to the task runner's logger
+:param task_routing_key: routing key of the task
+:param task_properties: task's message properties
+
+>>> from django.dispatch import receiver
+>>> from django_structlog.celery import signals
+>>>
+>>> @receiver(signals.modify_context_before_task_publish)
+... def receiver_modify_context_before_task_publish(sender, signal, context, task_routing_key=None, task_properties=None, **kwargs):
+...     keys_to_keep = {"request_id", "parent_task_id"}
+...     new_dict = {
+...         key_to_keep: context[key_to_keep]
+...         for key_to_keep in keys_to_keep
+...         if key_to_keep in context
+...     }
+...     context.clear()
+...     context.update(new_dict)
+
+"""
+
+pre_task_succeeded = django.dispatch.Signal()
+""" Signal to add ``structlog`` bindings from ``celery``'s successful task.
+
+:param logger: the logger to bind more metadata or override existing bound metadata
+:param result: result of the succeeding task
+
+>>> from django.dispatch import receiver
+>>> from django_structlog.celery import signals
+>>> import structlog
+>>>
+>>> @receiver(signals.pre_task_succeeded)
+... def receiver_pre_task_succeeded(sender, signal, logger=None, result=None, **kwargs):
+...     structlog.contextvars.bind_contextvars(result=str(result))
+
+"""