using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Nexus.Api.Data; using Nexus.Api.Models; using Nexus.Api.Repositories; using Nexus.Api.Services; using Xunit; namespace Nexus.Api.Tests; public sealed class StaleTaskRecoveryTests { [Fact] public async Task ResetStaleInProgressTasksAsync_OnlyResetsStaleInProgressTasks_AndWritesActivity() { await using var fixture = await TaskWorkflowFixture.CreateAsync(); var staleTimestamp = DateTimeOffset.UtcNow.AddHours(-3); var staleInProgress = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Stale in progress", State = "In progress", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }, CancellationToken.None); await fixture.ActivityRepository.AddAsync(new ActivityEvent { Type = "comment", Message = "Previous agent note", TaskId = staleInProgress.Id, CreatedAt = staleTimestamp.AddMinutes(15) }, CancellationToken.None); var staleBlocked = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Blocked task", State = "Blocked", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }, CancellationToken.None); var staleReview = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Review task", State = "Review", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }, CancellationToken.None); var staleDone = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Done task", State = "Done", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }, CancellationToken.None); var staleBacklog = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Backlog task", State = "Backlog", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }, CancellationToken.None); var freshInProgress = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Fresh in progress", State = "In progress", Source = "iris", UpdatedAt = DateTimeOffset.UtcNow.AddMinutes(-30), CreatedAt = staleTimestamp }, CancellationToken.None); var resetCount = await fixture.StaleTaskRecoveryService.ResetStaleInProgressTasksAsync(TimeSpan.FromHours(2), CancellationToken.None); Assert.Equal(1, resetCount); Assert.Equal("Backlog", (await fixture.TaskService.GetByIdAsync(staleInProgress.Id, CancellationToken.None))!.State); Assert.Equal("Blocked", (await fixture.TaskService.GetByIdAsync(staleBlocked.Id, CancellationToken.None))!.State); Assert.Equal("Review", (await fixture.TaskService.GetByIdAsync(staleReview.Id, CancellationToken.None))!.State); Assert.Equal("Done", (await fixture.TaskService.GetByIdAsync(staleDone.Id, CancellationToken.None))!.State); Assert.Equal("Backlog", (await fixture.TaskService.GetByIdAsync(staleBacklog.Id, CancellationToken.None))!.State); Assert.Equal("In progress", (await fixture.TaskService.GetByIdAsync(freshInProgress.Id, CancellationToken.None))!.State); var activity = await fixture.TaskService.GetTaskActivityAsync(staleInProgress.Id, CancellationToken.None); var resetActivity = activity.FirstOrDefault(entry => entry.Message.Contains("stale recovery", StringComparison.Ordinal)); Assert.NotNull(resetActivity); Assert.Contains("reason=stale-recovery", resetActivity!.Message, StringComparison.Ordinal); Assert.Contains("previous status In progress", resetActivity.Message, StringComparison.Ordinal); Assert.Contains("stale reference", resetActivity.Message, StringComparison.Ordinal); Assert.Contains("last activity", resetActivity.Message, StringComparison.Ordinal); Assert.Contains("new status Backlog", resetActivity.Message, StringComparison.Ordinal); } [Fact] public async Task ResetStaleInProgressTasksAsync_RevalidatesCurrentTaskBeforeReset() { var staleTimestamp = DateTimeOffset.UtcNow.AddHours(-3); var taskId = Guid.NewGuid(); var staleCandidate = new WorkTask { Id = taskId, Title = "Changed during recovery scan", State = "In progress", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }; var currentTask = new WorkTask { Id = taskId, Title = "Changed during recovery scan", State = "Review", Source = "iris", UpdatedAt = staleTimestamp, CreatedAt = staleTimestamp }; var taskRepository = new FakeTaskRepository(staleCandidate, currentTask); var activityRepository = new FakeActivityRepository(); var liveUpdateService = new FakeLiveUpdateService(); var recoveryService = new StaleTaskRecoveryService( taskRepository, activityRepository, liveUpdateService, new FakeNotificationService()); var resetCount = await recoveryService.ResetStaleInProgressTasksAsync(TimeSpan.FromHours(2), CancellationToken.None); Assert.Equal(0, resetCount); Assert.Equal("Review", currentTask.State); Assert.Equal(0, taskRepository.ResetCount); Assert.Empty(activityRepository.Added); Assert.Equal(0, liveUpdateService.PublishCount); } [Fact] public async Task FlagStalledInProgressTasksAsync_FlagsStalledTask_NotifiesIris_WithoutResetting() { await using var fixture = await TaskWorkflowFixture.CreateAsync(); var stalledTimestamp = DateTimeOffset.UtcNow.AddHours(-3); var stalled = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Stalled agent task", State = "In progress", Source = "iris", UpdatedAt = stalledTimestamp, CreatedAt = stalledTimestamp }, CancellationToken.None); var fresh = await fixture.TaskRepository.AddAsync(new WorkTask { Title = "Fresh in progress", State = "In progress", Source = "iris", UpdatedAt = DateTimeOffset.UtcNow.AddMinutes(-5), CreatedAt = stalledTimestamp }, CancellationToken.None); var flagged = await fixture.StaleTaskRecoveryService.FlagStalledInProgressTasksAsync( TimeSpan.FromMinutes(40), CancellationToken.None); Assert.Equal(1, flagged); // Nicht-destruktiv: bleibt In progress, kein Reset auf Backlog. Assert.Equal("In progress", (await fixture.TaskService.GetByIdAsync(stalled.Id, CancellationToken.None))!.State); Assert.Equal("In progress", (await fixture.TaskService.GetByIdAsync(fresh.Id, CancellationToken.None))!.State); var activity = await fixture.TaskService.GetTaskActivityAsync(stalled.Id, CancellationToken.None); Assert.Contains(activity, entry => string.Equals(entry.Type, "stalled", StringComparison.OrdinalIgnoreCase)); var irisNotifications = await fixture.NotificationService.GetForUserAsync("iris", 50, false, CancellationToken.None); Assert.Contains(irisNotifications, n => n.Type == "task_stalled" && n.TaskId == stalled.Id); // Idempotent: erneuter Lauf meldet denselben Hänger nicht nochmal. var flaggedAgain = await fixture.StaleTaskRecoveryService.FlagStalledInProgressTasksAsync( TimeSpan.FromMinutes(40), CancellationToken.None); Assert.Equal(0, flaggedAgain); } [Fact] public async Task BackgroundService_RunWatchdogOnceAsync_UsesStalledThreshold_AndFlags() { var fakeRecoveryService = new FakeStaleTaskRecoveryService(); var services = new ServiceCollection(); services.AddScoped(_ => fakeRecoveryService); await using var provider = services.BuildServiceProvider(); var backgroundService = new StaleTaskRecoveryBackgroundService( provider.GetRequiredService(), new TestOptionsMonitor(new StaleTaskRecoveryOptions { StalledMinutes = 45, IntervalMinutes = 10 }), NullLogger.Instance); var flaggedCount = await backgroundService.RunWatchdogOnceAsync(CancellationToken.None); Assert.Equal(1, fakeRecoveryService.FlagCallCount); Assert.Equal(0, fakeRecoveryService.ResetCallCount); Assert.Equal(TimeSpan.FromMinutes(45), fakeRecoveryService.LastThreshold); Assert.Equal(7, flaggedCount); } [Fact] public async Task BackgroundService_StartAsync_RunsWatchdogWithoutWaitingForFullInterval() { var fakeRecoveryService = new FakeStaleTaskRecoveryService(); var firstCall = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); fakeRecoveryService.OnCall = () => firstCall.TrySetResult(true); var services = new ServiceCollection(); services.AddScoped(_ => fakeRecoveryService); await using var provider = services.BuildServiceProvider(); var backgroundService = new StaleTaskRecoveryBackgroundService( provider.GetRequiredService(), new TestOptionsMonitor(new StaleTaskRecoveryOptions { StalledMinutes = 40, IntervalMinutes = 10 }), NullLogger.Instance); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); await backgroundService.StartAsync(cts.Token); await firstCall.Task.WaitAsync(cts.Token); await backgroundService.StopAsync(CancellationToken.None); Assert.True(fakeRecoveryService.FlagCallCount >= 1); Assert.Equal(TimeSpan.FromMinutes(40), fakeRecoveryService.LastThreshold); } [Fact] public void TaskRecoveryOptions_BindsStaleHoursFromEnvironmentOverride() { const string key = "TaskRecovery__StaleHours"; var originalValue = Environment.GetEnvironmentVariable(key); try { Environment.SetEnvironmentVariable(key, "5"); var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { [$"{StaleTaskRecoveryOptions.SectionName}:StaleHours"] = "2", [$"{StaleTaskRecoveryOptions.SectionName}:IntervalMinutes"] = "30" }) .AddEnvironmentVariables() .Build(); var options = configuration.GetSection(StaleTaskRecoveryOptions.SectionName).Get(); Assert.NotNull(options); Assert.Equal(5, options!.StaleHours); Assert.Equal(30, options.IntervalMinutes); } finally { Environment.SetEnvironmentVariable(key, originalValue); } } } file sealed class FakeTaskRepository(WorkTask staleCandidate, WorkTask currentTask) : ITaskRepository { public int ResetCount { get; private set; } public Task> GetAllAsync(CancellationToken ct = default) => Task.FromResult(new List { staleCandidate }); public ValueTask GetByIdAsync(Guid id, CancellationToken ct = default) => ValueTask.FromResult(id == currentTask.Id ? currentTask : null); public Task TryResetStaleInProgressToBacklogAsync( Guid id, DateTimeOffset staleBefore, DateTimeOffset updatedAt, CancellationToken ct = default) { if (id != currentTask.Id || !string.Equals(currentTask.State, "In progress", StringComparison.OrdinalIgnoreCase) || currentTask.UpdatedAt >= staleBefore) { return Task.FromResult(false); } ResetCount++; currentTask.State = "Backlog"; currentTask.UpdatedAt = updatedAt; return Task.FromResult(true); } public Task UpdateAsync(WorkTask task, CancellationToken ct = default) { return Task.CompletedTask; } public Task> GetPendingApprovalAsync(CancellationToken ct = default) => Task.FromResult(new List()); public Task AddAsync(WorkTask task, CancellationToken ct = default) => Task.FromResult(task); public Task DeleteAsync(WorkTask task, CancellationToken ct = default) => Task.CompletedTask; public Task CountAsync(CancellationToken ct = default) => Task.FromResult(0); public Task CountByStateAsync(string state, CancellationToken ct = default) => Task.FromResult(0); public Task GetLastBlockedAsync(CancellationToken ct = default) => Task.FromResult(null); } file sealed class FakeActivityRepository : IActivityRepository { public List Added { get; } = []; public Task> GetRecentAsync(int take, CancellationToken ct = default) => Task.FromResult(new List()); public Task> GetRecentForTasksAsync(IEnumerable taskIds, CancellationToken ct = default) => Task.FromResult(new List()); public Task<(List Items, int TotalCount)> GetPagedAsync( string? type, string? sort, int page, int pageSize, CancellationToken ct = default) => Task.FromResult((new List(), 0)); public Task> GetByAgentAsync(string agentId, int take, CancellationToken ct = default) => Task.FromResult(new List()); public Task AddAsync(ActivityEvent activity, CancellationToken ct = default) { Added.Add(activity); return Task.FromResult(activity); } } file sealed class FakeLiveUpdateService : ILiveUpdateService { public int PublishCount { get; private set; } public long CurrentSequence => PublishCount; public Task SubscribeAsync(long? afterSequence = null, CancellationToken ct = default) => throw new NotSupportedException(); public LiveUpdateEnvelope Publish(string type, object payload, string channel = "dashboard") { PublishCount++; return new LiveUpdateEnvelope(type, DateTimeOffset.UtcNow, payload, PublishCount, channel); } } file sealed class FakeStaleTaskRecoveryService : IStaleTaskRecoveryService { public int FlagCallCount { get; private set; } public int ResetCallCount { get; private set; } public TimeSpan LastThreshold { get; private set; } public Action? OnCall { get; set; } public Task FlagStalledInProgressTasksAsync(TimeSpan stalledThreshold, CancellationToken ct = default) { FlagCallCount++; LastThreshold = stalledThreshold; OnCall?.Invoke(); return Task.FromResult(7); } public Task ResetStaleInProgressTasksAsync(TimeSpan staleThreshold, CancellationToken ct = default) { ResetCallCount++; LastThreshold = staleThreshold; OnCall?.Invoke(); return Task.FromResult(7); } } file sealed class FakeNotificationService : INotificationService { public List Created { get; } = []; public Task CreateAsync(string type, string title, string? message, string forUser, Guid? taskId = null, CancellationToken ct = default) { var notification = new Notification { Type = type, Title = title, Message = message, ForUser = forUser, TaskId = taskId }; Created.Add(notification); return Task.FromResult(notification); } public Task> GetForUserAsync(string forUser, int limit = 50, bool unreadOnly = false, CancellationToken ct = default) => Task.FromResult>(Created.Where(n => n.ForUser == forUser).ToList()); public Task MarkAsReadAsync(Guid id, CancellationToken ct = default) => Task.FromResult(true); public Task MarkAllAsReadAsync(string forUser, CancellationToken ct = default) => Task.FromResult(0); public Task GetUnreadCountAsync(string forUser, CancellationToken ct = default) => Task.FromResult(0); public Task GetSnapshotAsync(string forUser, int limit = 50, bool unreadOnly = false, CancellationToken ct = default) => Task.FromResult(new NotificationSnapshotDto([], 0, forUser)); } file sealed class TestOptionsMonitor(T currentValue) : IOptionsMonitor { public T CurrentValue { get; private set; } = currentValue; public T Get(string? name) => CurrentValue; public IDisposable? OnChange(Action listener) => null; }