Add project notification settings maintenance cleanup

src/Exceptionless.Core/Bootstrapper.cs

@@ -111,6 +111,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
             handlers.Register<RemoveStacksWorkItem>(s.GetRequiredService<RemoveStacksWorkItemHandler>);
             handlers.Register<SetLocationFromGeoWorkItem>(s.GetRequiredService<SetLocationFromGeoWorkItemHandler>);
             handlers.Register<SetProjectIsConfiguredWorkItem>(s.GetRequiredService<SetProjectIsConfiguredWorkItemHandler>);
+            handlers.Register<UpdateProjectNotificationSettingsWorkItem>(s.GetRequiredService<UpdateProjectNotificationSettingsWorkItemHandler>);
             handlers.Register<UserMaintenanceWorkItem>(s.GetRequiredService<UserMaintenanceWorkItemHandler>);
             return handlers;
         });

src/Exceptionless.Core/Jobs/WorkItemHandlers/UpdateProjectNotificationSettingsWorkItemHandler.cs

@@ -0,0 +1,103 @@
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Repositories;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+public class UpdateProjectNotificationSettingsWorkItemHandler : WorkItemHandlerBase
+{
+    private const int BATCH_SIZE = 50;
+
+    private readonly IOrganizationRepository _organizationRepository;
+    private readonly OrganizationService _organizationService;
+    private readonly ILockProvider _lockProvider;
+    private readonly TimeProvider _timeProvider;
+
+    public UpdateProjectNotificationSettingsWorkItemHandler(
+        IOrganizationRepository organizationRepository,
+        OrganizationService organizationService,
+        ILockProvider lockProvider,
+        TimeProvider timeProvider,
+        ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _organizationRepository = organizationRepository;
+        _organizationService = organizationService;
+        _lockProvider = lockProvider;
+        _timeProvider = timeProvider;
+    }
+
+    public override Task<ILock> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = new())
+    {
+        return _lockProvider.AcquireAsync(nameof(UpdateProjectNotificationSettingsWorkItemHandler), TimeSpan.FromMinutes(15), cancellationToken);
+    }
+
+    public override async Task HandleItemAsync(WorkItemContext context)
+    {
+        var workItem = context.GetData<UpdateProjectNotificationSettingsWorkItem>();
+        Log.LogInformation("Received update project notification settings work item. Organization={Organization}", workItem.OrganizationId);
+
+        long totalNotificationSettingsRemoved = 0;
+        long organizationsProcessed = 0;
+
+        if (!String.IsNullOrEmpty(workItem.OrganizationId))
+        {
+            await context.ReportProgressAsync(0, $"Starting project notification settings update for organization {workItem.OrganizationId}");
+
+            var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId);
+            if (organization is null)
+            {
+                Log.LogWarning("Organization {Organization} not found", workItem.OrganizationId);
+                return;
+            }
+
+            totalNotificationSettingsRemoved += await _organizationService.CleanupProjectNotificationSettingsAsync(
+                organization,
+                [],
+                context.CancellationToken,
+                context.RenewLockAsync);
+            organizationsProcessed++;
+        }
+        else
+        {
+            await context.ReportProgressAsync(0, "Starting project notification settings update for all organizations");
+
+            var results = await _organizationRepository.FindAsync(
+                q => q.Include(o => o.Id),
+                o => o.SearchAfterPaging().PageLimit(BATCH_SIZE));
+
+            long totalOrganizations = results.Total;
+
+            while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
+            {
+                foreach (var organization in results.Documents)
+                {
+                    totalNotificationSettingsRemoved += await _organizationService.CleanupProjectNotificationSettingsAsync(
+                        organization,
+                        [],
+                        context.CancellationToken,
+                        context.RenewLockAsync);
+                    organizationsProcessed++;
+                }
+
+                int percentage = totalOrganizations > 0
+                    ? (int)Math.Min(99, organizationsProcessed * 100.0 / totalOrganizations)
+                    : 99;
+                await context.ReportProgressAsync(percentage, $"Processed {organizationsProcessed}/{totalOrganizations} organizations, removed {totalNotificationSettingsRemoved} invalid notification settings");
+
+                await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider);
+
+                if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync())
+                    break;
+
+                if (results.Documents.Count > 0)
+                    await context.RenewLockAsync();
+            }
+        }
+
+        Log.LogInformation("Project notification settings update complete. Organizations processed: {OrganizationsProcessed}, invalid notification settings removed: {RemovedNotificationSettings}", organizationsProcessed, totalNotificationSettingsRemoved);
+    }
+}

src/Exceptionless.Core/Models/WorkItems/UpdateProjectNotificationSettingsWorkItem.cs

@@ -0,0 +1,6 @@
+namespace Exceptionless.Core.Models.WorkItems;
+
+public record UpdateProjectNotificationSettingsWorkItem
+{
+    public string? OrganizationId { get; init; }
+}

src/Exceptionless.Core/Services/OrganizationService.cs

@@ -1,6 +1,6 @@
+using Exceptionless.Core.Extensions;
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Repositories;
-using Foundatio.Caching;
 using Foundatio.Extensions.Hosting.Startup;
 using Foundatio.Repositories;
 using Foundatio.Repositories.Models;
@@ -11,22 +11,23 @@ namespace Exceptionless.Core.Services;
 
 public class OrganizationService : IStartupAction
 {
+    private const int BATCH_SIZE = 50;
     private readonly IOrganizationRepository _organizationRepository;
+    private readonly IProjectRepository _projectRepository;
     private readonly ITokenRepository _tokenRepository;
     private readonly IUserRepository _userRepository;
     private readonly IWebHookRepository _webHookRepository;
-    private readonly ICacheClient _cache;
     private readonly AppOptions _appOptions;
     private readonly UsageService _usageService;
     private readonly ILogger _logger;
 
-    public OrganizationService(IOrganizationRepository organizationRepository, ITokenRepository tokenRepository, IUserRepository userRepository, IWebHookRepository webHookRepository, ICacheClient cache, AppOptions appOptions, UsageService usageService, ILoggerFactory loggerFactory)
+    public OrganizationService(IOrganizationRepository organizationRepository, IProjectRepository projectRepository, ITokenRepository tokenRepository, IUserRepository userRepository, IWebHookRepository webHookRepository, AppOptions appOptions, UsageService usageService, ILoggerFactory loggerFactory)
     {
         _organizationRepository = organizationRepository;
+        _projectRepository = projectRepository;
         _tokenRepository = tokenRepository;
         _userRepository = userRepository;
         _webHookRepository = webHookRepository;
-        _cache = cache;
         _appOptions = appOptions;
         _usageService = usageService;
         _logger = loggerFactory.CreateLogger<OrganizationService>();
@@ -72,36 +73,111 @@ public async Task CancelSubscriptionsAsync(Organization organization)
 
     public async Task<long> RemoveUsersAsync(Organization organization, string? currentUserId)
     {
-        var users = await _userRepository.GetByOrganizationIdAsync(organization.Id, o => o.PageLimit(1000));
-        foreach (var user in users.Documents)
+        long totalUsersAffected = 0;
+        var userResults = await _userRepository.GetByOrganizationIdAsync(organization.Id, o => o.SearchAfterPaging().PageLimit(BATCH_SIZE));
+
+        while (userResults.Documents.Count > 0)
         {
-            // delete the user if they are not associated to any other organizations and they are not the current user
-            if (user.OrganizationIds.All(oid => String.Equals(oid, organization.Id)) && !String.Equals(user.Id, currentUserId))
+            var usersToDelete = new List<User>(userResults.Documents.Count);
+            var usersToUpdate = new List<User>(userResults.Documents.Count);
+
+            foreach (var user in userResults.Documents)
             {
-                _logger.LogInformation("Removing user {User} as they do not belong to any other organizations", user.Id);
-                await _userRepository.RemoveAsync(user.Id);
+                // delete the user if they are not associated to any other organizations and they are not the current user
+                if (user.OrganizationIds.All(oid => String.Equals(oid, organization.Id)) && !String.Equals(user.Id, currentUserId))
+                {
+                    _logger.LogInformation("Removing user {User} as they do not belong to any other organizations", user.Id);
+                    usersToDelete.Add(user);
+                }
+                else
+                {
+                    _logger.LogInformation("Removing user {User} from organization: {OrganizationName} ({Organization})", user.Id, organization.Name, organization.Id);
+                    user.OrganizationIds.Remove(organization.Id);
+                    usersToUpdate.Add(user);
+                }
             }
-            else
+
+            if (usersToDelete.Count > 0)
+                await _userRepository.RemoveAsync(usersToDelete);
+
+            if (usersToUpdate.Count > 0)
+                await _userRepository.SaveAsync(usersToUpdate, o => o.Cache());
+
+            totalUsersAffected += usersToDelete.Count + usersToUpdate.Count;
+
+            if (!await userResults.NextPageAsync())
+                break;
+        }
+
+        return totalUsersAffected;
+    }
+
+    public Task<long> CleanupProjectNotificationSettingsAsync(Organization organization, IReadOnlyCollection<string> userIdsToRemove, CancellationToken cancellationToken = default, Func<Task>? renewWorkItemLockAsync = null)
+    {
+        ArgumentNullException.ThrowIfNull(organization);
+        ArgumentNullException.ThrowIfNull(userIdsToRemove);
+
+        return CleanupProjectNotificationSettingsAsync(organization.Id, userIdsToRemove, cancellationToken, renewWorkItemLockAsync);
+    }
+
+    private async Task<long> CleanupProjectNotificationSettingsAsync(string organizationId, IReadOnlyCollection<string> userIdsToRemove, CancellationToken cancellationToken, Func<Task>? renewWorkItemLockAsync)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(organizationId);
+        ArgumentNullException.ThrowIfNull(userIdsToRemove);
+
+        using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organizationId));
+
+        var userIdsToRemoveSet = userIdsToRemove.Count == 0
+            ? new HashSet<string>(StringComparer.Ordinal)
+            : new HashSet<string>(userIdsToRemove, StringComparer.Ordinal);
+
+        long removed = 0;
+        var projectResults = await _projectRepository.GetByOrganizationIdAsync(organizationId, o => o.SearchAfterPaging().PageLimit(BATCH_SIZE));
+        while (projectResults.Documents.Count > 0 && !cancellationToken.IsCancellationRequested)
+        {
+            var candidateUserIds = projectResults.Documents
+                .SelectMany(project => project.NotificationSettings.Keys)
+                .Where(key => !IsNotificationIntegrationKey(key))
+                .ToHashSet(StringComparer.Ordinal);
+
+            var validUserIds = await GetValidNotificationUserIdsAsync(organizationId, candidateUserIds, cancellationToken);
+            var projectsToSave = new List<Project>(projectResults.Documents.Count);
+
+            foreach (var project in projectResults.Documents)
             {
-                _logger.LogInformation("Removing user {User} from organization: {OrganizationName} ({OrganizationId})", user.Id, organization.Name, organization.Id);
-                user.OrganizationIds.Remove(organization.Id);
+                int removedFromProject = RemoveInvalidNotificationSettings(project, validUserIds, userIdsToRemoveSet);
+                if (removedFromProject <= 0)
+                    continue;
 
-                await _userRepository.SaveAsync(user, o => o.Cache());
+                removed += removedFromProject;
+                projectsToSave.Add(project);
             }
+
+            if (projectsToSave.Count > 0)
+                await _projectRepository.SaveAsync(projectsToSave);
+
+            if (renewWorkItemLockAsync is not null)
+                await renewWorkItemLockAsync();
+
+            if (!await projectResults.NextPageAsync())
+                break;
         }
 
-        return users.Documents.Count;
+        if (removed > 0)
+            _logger.LogInformation("Removed {Count} invalid notification settings", removed);
+
+        return removed;
     }
 
     public Task<long> RemoveTokensAsync(Organization organization)
     {
-        _logger.LogInformation("Removing tokens for {OrganizationName} ({OrganizationId})", organization.Name, organization.Id);
+        _logger.LogInformation("Removing tokens for {OrganizationName} ({Organization})", organization.Name, organization.Id);
         return _tokenRepository.RemoveAllByOrganizationIdAsync(organization.Id);
     }
 
     public Task<long> RemoveWebHooksAsync(Organization organization)
     {
-        _logger.LogInformation("Removing web hooks for {OrganizationName} ({OrganizationId})", organization.Name, organization.Id);
+        _logger.LogInformation("Removing web hooks for {OrganizationName} ({Organization})", organization.Name, organization.Id);
         return _webHookRepository.RemoveAllByOrganizationIdAsync(organization.Id);
     }
 
@@ -114,8 +190,41 @@ public async Task SoftDeleteOrganizationAsync(Organization organization, string
         await RemoveWebHooksAsync(organization);
         await CancelSubscriptionsAsync(organization);
         await RemoveUsersAsync(organization, currentUserId);
+        await CleanupProjectNotificationSettingsAsync(organization, []);
 
         organization.IsDeleted = true;
         await _organizationRepository.SaveAsync(organization);
     }
+
+    private async Task<HashSet<string>> GetValidNotificationUserIdsAsync(string organizationId, IReadOnlyCollection<string> userIds, CancellationToken cancellationToken)
+    {
+        var validUserIds = new HashSet<string>(StringComparer.Ordinal);
+        if (userIds.Count == 0)
+            return validUserIds;
+
+        foreach (string[] batch in userIds.Chunk(BATCH_SIZE))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var users = await _userRepository.GetByIdsAsync(batch);
+            validUserIds.UnionWith(users.Where(user => user.OrganizationIds.Contains(organizationId)).Select(user => user.Id));
+        }
+
+        return validUserIds;
+    }
+
+    private static int RemoveInvalidNotificationSettings(Project project, IReadOnlySet<string> validUserIds, IReadOnlySet<string> userIdsToRemove)
+    {
+        var keysToRemove = project.NotificationSettings.Keys
+            .Where(key => !IsNotificationIntegrationKey(key) && (userIdsToRemove.Contains(key) || !validUserIds.Contains(key)))
+            .ToList();
+
+        foreach (var key in keysToRemove)
+            project.NotificationSettings.Remove(key);
+
+        return keysToRemove.Count;
+    }
+
+    private static bool IsNotificationIntegrationKey(string key) =>
+        String.Equals(key, Project.NotificationIntegrations.Slack, StringComparison.OrdinalIgnoreCase);
 }

src/Exceptionless.Web/Controllers/AdminController.cs

@@ -163,45 +163,51 @@ public async Task<IActionResult> RunJobAsync(string name, DateTime? utcStart = n
 
         switch (name.ToLowerInvariant())
         {
+            case "fix-stack-stats":
+                var defaultUtcStart = new DateTime(2026, 2, 10, 0, 0, 0, DateTimeKind.Utc);
+                var effectiveUtcStart = utcStart ?? defaultUtcStart;
+
+                if (utcEnd.HasValue && utcEnd.Value.IsBefore(effectiveUtcStart))
+                {
+                    ModelState.AddModelError(nameof(utcEnd), "utcEnd must be greater than or equal to utcStart.");
+                    return ValidationProblem(ModelState);
+                }
+
+                await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem
+                {
+                    UtcStart = effectiveUtcStart,
+                    UtcEnd = utcEnd,
+                    OrganizationId = organizationId
+                });
+                break;
+            case "increment-project-configuration-version":
+                await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { IncrementConfigurationVersion = true });
+                break;
             case "indexes":
                 if (!_appOptions.ElasticsearchOptions.DisableIndexConfiguration)
                     await _configuration.ConfigureIndexesAsync(beginReindexingOutdated: false);
                 break;
-            case "update-organization-plans":
-                await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { UpgradePlans = true });
+            case "normalize-user-email-address":
+                await _workItemQueue.EnqueueAsync(new UserMaintenanceWorkItem { Normalize = true });
                 break;
             case "remove-old-organization-usage":
                 await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { RemoveOldUsageStats = true });
                 break;
-            case "update-project-default-bot-lists":
-                await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { UpdateDefaultBotList = true, IncrementConfigurationVersion = true });
-                break;
-            case "increment-project-configuration-version":
-                await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { IncrementConfigurationVersion = true });
-                break;
             case "remove-old-project-usage":
                 await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { RemoveOldUsageStats = true });
                 break;
-            case "normalize-user-email-address":
-                await _workItemQueue.EnqueueAsync(new UserMaintenanceWorkItem { Normalize = true });
-                break;
             case "reset-verify-email-address-token-and-expiration":
                 await _workItemQueue.EnqueueAsync(new UserMaintenanceWorkItem { ResetVerifyEmailAddressToken = true });
                 break;
-            case "fix-stack-stats":
-                var defaultUtcStart = new DateTime(2026, 2, 10, 0, 0, 0, DateTimeKind.Utc);
-                var effectiveUtcStart = utcStart ?? defaultUtcStart;
-
-                if (utcEnd.HasValue && utcEnd.Value.IsBefore(effectiveUtcStart))
-                {
-                    ModelState.AddModelError(nameof(utcEnd), "utcEnd must be greater than or equal to utcStart.");
-                    return ValidationProblem(ModelState);
-                }
-
-                await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem
+            case "update-organization-plans":
+                await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { UpgradePlans = true });
+                break;
+            case "update-project-default-bot-lists":
+                await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { UpdateDefaultBotList = true, IncrementConfigurationVersion = true });
+                break;
+            case "update-project-notification-settings":
+                await _workItemQueue.EnqueueAsync(new UpdateProjectNotificationSettingsWorkItem
                 {
-                    UtcStart = effectiveUtcStart,
-                    UtcEnd = utcEnd,
                     OrganizationId = organizationId
                 });
                 break;