nexus/backend/Services/StaleTaskRecoveryBackgroundService.cs

using Microsoft.Extensions.Options;

namespace Nexus.Api.Services;

public sealed class StaleTaskRecoveryBackgroundService(
    IServiceScopeFactory scopeFactory,
    IOptionsMonitor<StaleTaskRecoveryOptions> optionsMonitor,
    ILogger<StaleTaskRecoveryBackgroundService> logger) : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                var flaggedCount = await RunWatchdogOnceAsync(stoppingToken);
                if (flaggedCount > 0)
                    logger.LogInformation("Stall watchdog flagged {FlaggedCount} stalled task(s) for Iris.", flaggedCount);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                break;
            }
            catch (Exception ex)
            {
                logger.LogError(ex, "Stale task recovery run failed.");
            }

            try
            {
                await Task.Delay(optionsMonitor.CurrentValue.GetInterval(), stoppingToken);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                break;
            }
        }
    }

    public async Task<int> RunWatchdogOnceAsync(CancellationToken ct = default)
    {
        await using var scope = scopeFactory.CreateAsyncScope();
        var recoveryService = scope.ServiceProvider.GetRequiredService<IStaleTaskRecoveryService>();
        return await recoveryService.FlagStalledInProgressTasksAsync(optionsMonitor.CurrentValue.GetStalledThreshold(), ct);
    }
}