nexus/backend-tests/StaleTaskRecoveryTests.cs

using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nexus.Api.Data;
using Nexus.Api.Models;
using Nexus.Api.Repositories;
using Nexus.Api.Services;
using Xunit;

namespace Nexus.Api.Tests;

public sealed class StaleTaskRecoveryTests
{
    [Fact]
    public async Task ResetStaleInProgressTasksAsync_OnlyResetsStaleInProgressTasks_AndWritesActivity()
    {
        await using var fixture = await TaskWorkflowFixture.CreateAsync();
        var staleTimestamp = DateTimeOffset.UtcNow.AddHours(-3);

        var staleInProgress = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Stale in progress",
            State = "In progress",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        }, CancellationToken.None);

        await fixture.ActivityRepository.AddAsync(new ActivityEvent
        {
            Type = "comment",
            Message = "Previous agent note",
            TaskId = staleInProgress.Id,
            CreatedAt = staleTimestamp.AddMinutes(15)
        }, CancellationToken.None);

        var staleBlocked = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Blocked task",
            State = "Blocked",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        }, CancellationToken.None);

        var staleReview = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Review task",
            State = "Review",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        }, CancellationToken.None);

        var staleDone = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Done task",
            State = "Done",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        }, CancellationToken.None);

        var staleBacklog = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Backlog task",
            State = "Backlog",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        }, CancellationToken.None);

        var freshInProgress = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Fresh in progress",
            State = "In progress",
            Source = "iris",
            UpdatedAt = DateTimeOffset.UtcNow.AddMinutes(-30),
            CreatedAt = staleTimestamp
        }, CancellationToken.None);

        var resetCount = await fixture.StaleTaskRecoveryService.ResetStaleInProgressTasksAsync(TimeSpan.FromHours(2), CancellationToken.None);

        Assert.Equal(1, resetCount);
        Assert.Equal("Backlog", (await fixture.TaskService.GetByIdAsync(staleInProgress.Id, CancellationToken.None))!.State);
        Assert.Equal("Blocked", (await fixture.TaskService.GetByIdAsync(staleBlocked.Id, CancellationToken.None))!.State);
        Assert.Equal("Review", (await fixture.TaskService.GetByIdAsync(staleReview.Id, CancellationToken.None))!.State);
        Assert.Equal("Done", (await fixture.TaskService.GetByIdAsync(staleDone.Id, CancellationToken.None))!.State);
        Assert.Equal("Backlog", (await fixture.TaskService.GetByIdAsync(staleBacklog.Id, CancellationToken.None))!.State);
        Assert.Equal("In progress", (await fixture.TaskService.GetByIdAsync(freshInProgress.Id, CancellationToken.None))!.State);

        var activity = await fixture.TaskService.GetTaskActivityAsync(staleInProgress.Id, CancellationToken.None);
        var resetActivity = activity.FirstOrDefault(entry => entry.Message.Contains("stale recovery", StringComparison.Ordinal));

        Assert.NotNull(resetActivity);
        Assert.Contains("reason=stale-recovery", resetActivity!.Message, StringComparison.Ordinal);
        Assert.Contains("previous status In progress", resetActivity.Message, StringComparison.Ordinal);
        Assert.Contains("stale reference", resetActivity.Message, StringComparison.Ordinal);
        Assert.Contains("last activity", resetActivity.Message, StringComparison.Ordinal);
        Assert.Contains("new status Backlog", resetActivity.Message, StringComparison.Ordinal);
    }

    [Fact]
    public async Task ResetStaleInProgressTasksAsync_RevalidatesCurrentTaskBeforeReset()
    {
        var staleTimestamp = DateTimeOffset.UtcNow.AddHours(-3);
        var taskId = Guid.NewGuid();
        var staleCandidate = new WorkTask
        {
            Id = taskId,
            Title = "Changed during recovery scan",
            State = "In progress",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        };
        var currentTask = new WorkTask
        {
            Id = taskId,
            Title = "Changed during recovery scan",
            State = "Review",
            Source = "iris",
            UpdatedAt = staleTimestamp,
            CreatedAt = staleTimestamp
        };
        var taskRepository = new FakeTaskRepository(staleCandidate, currentTask);
        var activityRepository = new FakeActivityRepository();
        var liveUpdateService = new FakeLiveUpdateService();
        var recoveryService = new StaleTaskRecoveryService(
            taskRepository,
            activityRepository,
            liveUpdateService,
            new FakeNotificationService());

        var resetCount = await recoveryService.ResetStaleInProgressTasksAsync(TimeSpan.FromHours(2), CancellationToken.None);

        Assert.Equal(0, resetCount);
        Assert.Equal("Review", currentTask.State);
        Assert.Equal(0, taskRepository.ResetCount);
        Assert.Empty(activityRepository.Added);
        Assert.Equal(0, liveUpdateService.PublishCount);
    }

    [Fact]
    public async Task FlagStalledInProgressTasksAsync_FlagsStalledTask_NotifiesIris_WithoutResetting()
    {
        await using var fixture = await TaskWorkflowFixture.CreateAsync();
        var stalledTimestamp = DateTimeOffset.UtcNow.AddHours(-3);

        var stalled = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Stalled agent task",
            State = "In progress",
            Source = "iris",
            UpdatedAt = stalledTimestamp,
            CreatedAt = stalledTimestamp
        }, CancellationToken.None);

        var fresh = await fixture.TaskRepository.AddAsync(new WorkTask
        {
            Title = "Fresh in progress",
            State = "In progress",
            Source = "iris",
            UpdatedAt = DateTimeOffset.UtcNow.AddMinutes(-5),
            CreatedAt = stalledTimestamp
        }, CancellationToken.None);

        var flagged = await fixture.StaleTaskRecoveryService.FlagStalledInProgressTasksAsync(
            TimeSpan.FromMinutes(40), CancellationToken.None);

        Assert.Equal(1, flagged);
        // Nicht-destruktiv: bleibt In progress, kein Reset auf Backlog.
        Assert.Equal("In progress", (await fixture.TaskService.GetByIdAsync(stalled.Id, CancellationToken.None))!.State);
        Assert.Equal("In progress", (await fixture.TaskService.GetByIdAsync(fresh.Id, CancellationToken.None))!.State);

        var activity = await fixture.TaskService.GetTaskActivityAsync(stalled.Id, CancellationToken.None);
        Assert.Contains(activity, entry => string.Equals(entry.Type, "stalled", StringComparison.OrdinalIgnoreCase));

        var irisNotifications = await fixture.NotificationService.GetForUserAsync("iris", 50, false, CancellationToken.None);
        Assert.Contains(irisNotifications, n => n.Type == "task_stalled" && n.TaskId == stalled.Id);

        // Idempotent: erneuter Lauf meldet denselben Hänger nicht nochmal.
        var flaggedAgain = await fixture.StaleTaskRecoveryService.FlagStalledInProgressTasksAsync(
            TimeSpan.FromMinutes(40), CancellationToken.None);
        Assert.Equal(0, flaggedAgain);
    }

    [Fact]
    public async Task BackgroundService_RunWatchdogOnceAsync_UsesStalledThreshold_AndFlags()
    {
        var fakeRecoveryService = new FakeStaleTaskRecoveryService();
        var services = new ServiceCollection();
        services.AddScoped<IStaleTaskRecoveryService>(_ => fakeRecoveryService);

        await using var provider = services.BuildServiceProvider();
        var backgroundService = new StaleTaskRecoveryBackgroundService(
            provider.GetRequiredService<IServiceScopeFactory>(),
            new TestOptionsMonitor<StaleTaskRecoveryOptions>(new StaleTaskRecoveryOptions
            {
                StalledMinutes = 45,
                IntervalMinutes = 10
            }),
            NullLogger<StaleTaskRecoveryBackgroundService>.Instance);

        var flaggedCount = await backgroundService.RunWatchdogOnceAsync(CancellationToken.None);

        Assert.Equal(1, fakeRecoveryService.FlagCallCount);
        Assert.Equal(0, fakeRecoveryService.ResetCallCount);
        Assert.Equal(TimeSpan.FromMinutes(45), fakeRecoveryService.LastThreshold);
        Assert.Equal(7, flaggedCount);
    }

    [Fact]
    public async Task BackgroundService_StartAsync_RunsWatchdogWithoutWaitingForFullInterval()
    {
        var fakeRecoveryService = new FakeStaleTaskRecoveryService();
        var firstCall = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
        fakeRecoveryService.OnCall = () => firstCall.TrySetResult(true);

        var services = new ServiceCollection();
        services.AddScoped<IStaleTaskRecoveryService>(_ => fakeRecoveryService);

        await using var provider = services.BuildServiceProvider();
        var backgroundService = new StaleTaskRecoveryBackgroundService(
            provider.GetRequiredService<IServiceScopeFactory>(),
            new TestOptionsMonitor<StaleTaskRecoveryOptions>(new StaleTaskRecoveryOptions
            {
                StalledMinutes = 40,
                IntervalMinutes = 10
            }),
            NullLogger<StaleTaskRecoveryBackgroundService>.Instance);

        using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
        await backgroundService.StartAsync(cts.Token);
        await firstCall.Task.WaitAsync(cts.Token);
        await backgroundService.StopAsync(CancellationToken.None);

        Assert.True(fakeRecoveryService.FlagCallCount >= 1);
        Assert.Equal(TimeSpan.FromMinutes(40), fakeRecoveryService.LastThreshold);
    }

    [Fact]
    public void TaskRecoveryOptions_BindsStaleHoursFromEnvironmentOverride()
    {
        const string key = "TaskRecovery__StaleHours";
        var originalValue = Environment.GetEnvironmentVariable(key);

        try
        {
            Environment.SetEnvironmentVariable(key, "5");

            var configuration = new ConfigurationBuilder()
                .AddInMemoryCollection(new Dictionary<string, string?>
                {
                    [$"{StaleTaskRecoveryOptions.SectionName}:StaleHours"] = "2",
                    [$"{StaleTaskRecoveryOptions.SectionName}:IntervalMinutes"] = "30"
                })
                .AddEnvironmentVariables()
                .Build();

            var options = configuration.GetSection(StaleTaskRecoveryOptions.SectionName).Get<StaleTaskRecoveryOptions>();

            Assert.NotNull(options);
            Assert.Equal(5, options!.StaleHours);
            Assert.Equal(30, options.IntervalMinutes);
        }
        finally
        {
            Environment.SetEnvironmentVariable(key, originalValue);
        }
    }
}

file sealed class FakeTaskRepository(WorkTask staleCandidate, WorkTask currentTask) : ITaskRepository
{
    public int ResetCount { get; private set; }

    public Task<List<WorkTask>> GetAllAsync(CancellationToken ct = default)
        => Task.FromResult(new List<WorkTask> { staleCandidate });

    public ValueTask<WorkTask?> GetByIdAsync(Guid id, CancellationToken ct = default)
        => ValueTask.FromResult<WorkTask?>(id == currentTask.Id ? currentTask : null);

    public Task<bool> TryResetStaleInProgressToBacklogAsync(
        Guid id,
        DateTimeOffset staleBefore,
        DateTimeOffset updatedAt,
        CancellationToken ct = default)
    {
        if (id != currentTask.Id
            || !string.Equals(currentTask.State, "In progress", StringComparison.OrdinalIgnoreCase)
            || currentTask.UpdatedAt >= staleBefore)
        {
            return Task.FromResult(false);
        }

        ResetCount++;
        currentTask.State = "Backlog";
        currentTask.UpdatedAt = updatedAt;
        return Task.FromResult(true);
    }

    public Task UpdateAsync(WorkTask task, CancellationToken ct = default)
    {
        return Task.CompletedTask;
    }

    public Task<List<WorkTask>> GetPendingApprovalAsync(CancellationToken ct = default)
        => Task.FromResult(new List<WorkTask>());

    public Task<WorkTask> AddAsync(WorkTask task, CancellationToken ct = default)
        => Task.FromResult(task);

    public Task DeleteAsync(WorkTask task, CancellationToken ct = default)
        => Task.CompletedTask;

    public Task<int> CountAsync(CancellationToken ct = default)
        => Task.FromResult(0);

    public Task<int> CountByStateAsync(string state, CancellationToken ct = default)
        => Task.FromResult(0);

    public Task<WorkTask?> GetLastBlockedAsync(CancellationToken ct = default)
        => Task.FromResult<WorkTask?>(null);
}

file sealed class FakeActivityRepository : IActivityRepository
{
    public List<ActivityEvent> Added { get; } = [];

    public Task<List<ActivityEvent>> GetRecentAsync(int take, CancellationToken ct = default)
        => Task.FromResult(new List<ActivityEvent>());

    public Task<List<ActivityEvent>> GetRecentForTasksAsync(IEnumerable<Guid> taskIds, CancellationToken ct = default)
        => Task.FromResult(new List<ActivityEvent>());

    public Task<(List<ActivityEvent> Items, int TotalCount)> GetPagedAsync(
        string? type,
        string? sort,
        int page,
        int pageSize,
        CancellationToken ct = default)
        => Task.FromResult((new List<ActivityEvent>(), 0));

    public Task<List<ActivityEvent>> GetByAgentAsync(string agentId, int take, CancellationToken ct = default)
        => Task.FromResult(new List<ActivityEvent>());

    public Task<ActivityEvent> AddAsync(ActivityEvent activity, CancellationToken ct = default)
    {
        Added.Add(activity);
        return Task.FromResult(activity);
    }
}

file sealed class FakeLiveUpdateService : ILiveUpdateService
{
    public int PublishCount { get; private set; }
    public long CurrentSequence => PublishCount;

    public Task<LiveUpdateSubscription> SubscribeAsync(long? afterSequence = null, CancellationToken ct = default)
        => throw new NotSupportedException();

    public LiveUpdateEnvelope Publish(string type, object payload, string channel = "dashboard")
    {
        PublishCount++;
        return new LiveUpdateEnvelope(type, DateTimeOffset.UtcNow, payload, PublishCount, channel);
    }
}

file sealed class FakeStaleTaskRecoveryService : IStaleTaskRecoveryService
{
    public int FlagCallCount { get; private set; }
    public int ResetCallCount { get; private set; }
    public TimeSpan LastThreshold { get; private set; }
    public Action? OnCall { get; set; }

    public Task<int> FlagStalledInProgressTasksAsync(TimeSpan stalledThreshold, CancellationToken ct = default)
    {
        FlagCallCount++;
        LastThreshold = stalledThreshold;
        OnCall?.Invoke();
        return Task.FromResult(7);
    }

    public Task<int> ResetStaleInProgressTasksAsync(TimeSpan staleThreshold, CancellationToken ct = default)
    {
        ResetCallCount++;
        LastThreshold = staleThreshold;
        OnCall?.Invoke();
        return Task.FromResult(7);
    }
}

file sealed class FakeNotificationService : INotificationService
{
    public List<Notification> Created { get; } = [];

    public Task<Notification> CreateAsync(string type, string title, string? message, string forUser, Guid? taskId = null, CancellationToken ct = default)
    {
        var notification = new Notification { Type = type, Title = title, Message = message, ForUser = forUser, TaskId = taskId };
        Created.Add(notification);
        return Task.FromResult(notification);
    }

    public Task<IReadOnlyList<Notification>> GetForUserAsync(string forUser, int limit = 50, bool unreadOnly = false, CancellationToken ct = default)
        => Task.FromResult<IReadOnlyList<Notification>>(Created.Where(n => n.ForUser == forUser).ToList());

    public Task<bool> MarkAsReadAsync(Guid id, CancellationToken ct = default) => Task.FromResult(true);

    public Task<int> MarkAllAsReadAsync(string forUser, CancellationToken ct = default) => Task.FromResult(0);

    public Task<int> GetUnreadCountAsync(string forUser, CancellationToken ct = default) => Task.FromResult(0);

    public Task<NotificationSnapshotDto> GetSnapshotAsync(string forUser, int limit = 50, bool unreadOnly = false, CancellationToken ct = default)
        => Task.FromResult(new NotificationSnapshotDto([], 0, forUser));
}

file sealed class TestOptionsMonitor<T>(T currentValue) : IOptionsMonitor<T>
{
    public T CurrentValue { get; private set; } = currentValue;

    public T Get(string? name) => CurrentValue;

    public IDisposable? OnChange(Action<T, string?> listener) => null;
}