nexus/backend-tests/OpenClawRunServiceTests.cs

using System.Security.Claims;
using System.Text.Json.Nodes;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using Nexus.Api.Controllers;
using Nexus.Api.Data;
using Nexus.Api.Models;
using Nexus.Api.Repositories;
using Nexus.Api.Services;
using Xunit;

namespace Nexus.Api.Tests;

public sealed class OpenClawRunServiceTests
{
    [Fact]
    public async Task ListCursor_DoesNotSkipOrDuplicateRunsWithTheSameCreatedAt()
    {
        await using var harness = await RunHarness.CreateAsync();
        var timestamp = new DateTimeOffset(
            2026,
            7,
            31,
            12,
            0,
            0,
            TimeSpan.Zero);
        var runs = Enumerable.Range(0, 3)
            .Select(index => new OpenClawRun
            {
                Id = Guid.NewGuid(),
                Title = $"Run {index}",
                Prompt = $"Inspect {index}",
                AgentId = "iris",
                SessionKey = "agent:iris:main",
                Status = OpenClawRunStates.Completed,
                StartIdempotencyKey = $"list-cursor-{index}",
                CorrelationId = $"correlation-{index}",
                Actor = "owner",
                CreatedAt = timestamp,
                UpdatedAt = timestamp
            })
            .ToList();
        harness.Db.OpenClawRuns.AddRange(runs);
        await harness.Db.SaveChangesAsync();
        harness.Db.ChangeTracker.Clear();

        var first = await harness.Service.GetAsync(new OpenClawRunQuery(Limit: 2));
        var second = await harness.Service.GetAsync(
            new OpenClawRunQuery(Limit: 2, Cursor: first.NextCursor));

        Assert.Equal(2, first.Items.Count);
        Assert.NotNull(first.NextCursor);
        Assert.Matches("^[A-Za-z0-9_-]+$", first.NextCursor);
        Assert.True(
            OpenClawRunCursorCodec.TryDecode(
                first.NextCursor,
                out var cursorPosition));
        Assert.Equal(first.Items[^1].CreatedAt, cursorPosition.CreatedAt);
        Assert.Equal(first.Items[^1].Id, cursorPosition.Id);
        Assert.Single(second.Items);
        Assert.Null(second.NextCursor);
        Assert.Equal(
            3,
            first.Items
                .Concat(second.Items)
                .Select(item => item.Id)
                .Distinct()
                .Count());
    }

    [Fact]
    public async Task ListCursor_AcceptsLegacyUtcTicksCursor()
    {
        await using var harness = await RunHarness.CreateAsync();
        var boundary = new DateTimeOffset(
            2026,
            7,
            31,
            12,
            0,
            0,
            TimeSpan.Zero);
        harness.Db.OpenClawRuns.AddRange(
            CreateListedRun("newer", boundary.AddMinutes(1)),
            CreateListedRun("older", boundary.AddMinutes(-1)));
        await harness.Db.SaveChangesAsync();
        harness.Db.ChangeTracker.Clear();

        var page = await harness.Service.GetAsync(
            new OpenClawRunQuery(
                Limit: 10,
                Cursor: boundary.UtcTicks.ToString(
                    System.Globalization.CultureInfo.InvariantCulture)));

        var run = Assert.Single(page.Items);
        Assert.Equal("older", run.Title);
    }

    [Fact]
    public async Task Start_PersistsCorrelations_AndReplaysIdempotently()
    {
        await using var harness = await RunHarness.CreateAsync();
        var project = new Project { Name = "Nexus" };
        var task = new WorkTask { Title = "Connect OpenClaw", ProjectId = project.Id };
        harness.Db.Projects.Add(project);
        harness.Db.Tasks.Add(task);
        await harness.Db.SaveChangesAsync();
        harness.Gateway.StartResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            true,
            OpenClawRunStates.Running,
            "started",
            "oc-run-1"));
        var request = new StartOpenClawRunRequest(
            "Inspect the deployment",
            "iris",
            "agent:iris:main",
            "Deployment inspection",
            task.Id,
            project.Id);
        var invocation = Invocation("start-key", "corr-1", "owner-1");

        var first = await harness.Service.StartAsync(request, invocation);
        var replay = await harness.Service.StartAsync(request, invocation);

        Assert.True(first.Ok);
        Assert.Equal("oc-run-1", first.Run.OpenClawRunId);
        Assert.Equal(task.Id, first.Run.TaskId);
        Assert.Equal(project.Id, first.Run.ProjectId);
        Assert.Equal("corr-1", first.Run.CorrelationId);
        Assert.Equal("owner-1", first.Run.Actor);
        Assert.Equal("idempotent_replay", replay.State);
        Assert.Single(harness.Gateway.StartCalls);

        var persisted = await harness.Db.OpenClawRuns.SingleAsync();
        Assert.Equal(OpenClawRunStates.Running, persisted.Status);
        Assert.Equal(2, await harness.Db.OpenClawRunHistory.CountAsync());
        Assert.All(
            await harness.Db.OpenClawRunHistory.ToListAsync(),
            item => Assert.Equal("corr-1", item.CorrelationId));
    }

    [Fact]
    public async Task Stop_UsesExactRunAndDoesNotRepeatGatewayMutation()
    {
        await using var harness = await RunHarness.CreateAsync();
        harness.Gateway.StartResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            true,
            OpenClawRunStates.Running,
            "started",
            "oc-run-stop"));
        var started = await harness.Service.StartAsync(
            Request(),
            Invocation("start-stop", "corr-stop", "owner"));
        harness.Gateway.StopResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            true,
            OpenClawRunStates.Stopped,
            "stopped",
            "oc-run-stop"));
        var stopInvocation = Invocation("stop-key", "corr-stop-2", "owner");

        var stopped = await harness.Service.StopAsync(
            started.Run.Id,
            "operator request",
            stopInvocation);
        var replay = await harness.Service.StopAsync(
            started.Run.Id,
            "operator request",
            stopInvocation);

        Assert.NotNull(stopped);
        Assert.True(stopped!.Ok);
        Assert.Equal(OpenClawRunStates.Stopped, stopped.Run.Status);
        Assert.Equal("idempotent_replay", replay?.State);
        Assert.Single(harness.Gateway.StopCalls);
        Assert.Contains(
            await harness.Db.OpenClawRunHistory.ToListAsync(),
            item => item.Action == "stop_requested"
                    && item.IdempotencyKey == "stop-key"
                    && item.Actor == "owner");
    }

    [Fact]
    public async Task Retry_CreatesCorrelatedChild_WhileResumeIsTruthfullyUnsupported()
    {
        await using var harness = await RunHarness.CreateAsync();
        harness.Gateway.StartResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            false,
            OpenClawRunStates.Failed,
            "provider failed"));
        var source = await harness.Service.StartAsync(
            Request(),
            Invocation("start-failed", "corr-source", "owner"));
        harness.Gateway.StartResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            true,
            OpenClawRunStates.Running,
            "retry started",
            "oc-run-retry"));

        var resume = await harness.Service.ResumeAsync(
            source.Run.Id,
            null,
            Invocation("resume-key", "corr-resume", "owner"));
        var retry = await harness.Service.RetryAsync(
            source.Run.Id,
            "retry after provider recovery",
            Invocation("retry-key", "corr-retry", "owner"));

        Assert.NotNull(resume);
        Assert.False(resume!.Ok);
        Assert.Equal(OpenClawRunStates.Unsupported, resume.State);
        Assert.False(resume.Run.CanResume);
        Assert.NotNull(retry?.ResultRun);
        Assert.True(retry!.Ok);
        Assert.Equal(source.Run.Id, retry.ResultRun!.RetriedFromRunId);
        Assert.Equal(source.Run.SessionKey, retry.ResultRun.SessionKey);
        Assert.Equal("corr-retry", retry.ResultRun.CorrelationId);
        Assert.Equal(2, harness.Gateway.StartCalls.Count);
    }

    [Fact]
    public async Task Reconcile_ProjectsTerminalState_AndPerRunSequenceGap()
    {
        await using var harness = await RunHarness.CreateAsync();
        harness.Gateway.StartResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            true,
            OpenClawRunStates.Running,
            "started",
            "oc-run-event"));
        var started = await harness.Service.StartAsync(
            Request(),
            Invocation("start-event", "corr-event", "owner"));
        var now = DateTimeOffset.UtcNow;

        await harness.Service.ReconcileAsync(ChatEvent("oc-run-event", "delta", 1, now));
        await harness.Service.ReconcileAsync(ChatEvent("oc-run-event", "final", 3, now.AddSeconds(1)));
        await harness.Service.ReconcileAsync(ChatEvent("oc-run-event", "final", 3, now.AddSeconds(1)));

        harness.Db.ChangeTracker.Clear();
        var run = await harness.Db.OpenClawRuns.SingleAsync(item => item.Id == started.Run.Id);
        Assert.Equal(OpenClawRunStates.Completed, run.Status);
        Assert.Equal(3, run.LastGatewaySequence);
        Assert.True(run.SequenceGapDetected);
        Assert.NotNull(run.FinishedAt);
        var gatewayTransitions = await harness.Db.OpenClawRunHistory
            .Where(item => item.Action == "gateway_event")
            .ToListAsync();
        var gatewayTransition = Assert.Single(gatewayTransitions);
        Assert.True(gatewayTransition.SequenceGapDetected);
        Assert.Equal(OpenClawRunStates.Completed, gatewayTransition.ToStatus);
    }

    [Fact]
    public async Task Start_RejectsUnknownTaskCorrelationBeforeGatewayCall()
    {
        await using var harness = await RunHarness.CreateAsync();
        var request = Request() with { TaskId = Guid.NewGuid() };

        var exception = await Assert.ThrowsAsync<OpenClawRunValidationException>(
            () => harness.Service.StartAsync(
                request,
                Invocation("unknown-task", "corr", "owner")));

        Assert.Equal("taskId", exception.Field);
        Assert.Empty(harness.Gateway.StartCalls);
        Assert.Empty(await harness.Db.OpenClawRuns.ToListAsync());
    }

    [Fact]
    public async Task Start_ReplaysUnacknowledgedDispatchWithTheSameIdempotencyKey()
    {
        await using var harness = await RunHarness.CreateAsync();
        var request = Request();
        var pending = new OpenClawRun
        {
            Title = request.Title!,
            Prompt = request.Prompt,
            AgentId = request.AgentId,
            SessionKey = request.SessionKey,
            Status = OpenClawRunStates.Dispatching,
            StartIdempotencyKey = "recover-key",
            CorrelationId = "original-correlation",
            Actor = "owner"
        };
        harness.Db.OpenClawRuns.Add(pending);
        await harness.Db.SaveChangesAsync();
        harness.Db.ChangeTracker.Clear();
        harness.Gateway.StartResults.Enqueue(new OpenClawRunGatewayResult(
            true,
            true,
            OpenClawRunStates.Running,
            "in flight",
            "oc-recovered"));

        var result = await harness.Service.StartAsync(
            request,
            Invocation("recover-key", "recovery-correlation", "owner"));

        Assert.True(result.Ok);
        Assert.Equal("oc-recovered", result.Run.OpenClawRunId);
        Assert.Single(harness.Gateway.StartCalls);
        var recovered = await harness.Db.OpenClawRuns.SingleAsync();
        Assert.Equal(OpenClawRunStates.Running, recovered.Status);
        Assert.Contains(
            await harness.Db.OpenClawRunHistory.ToListAsync(),
            item => item.Action == "start_recovery"
                    && item.CorrelationId == "recovery-correlation");
    }

    [Fact]
    public async Task Controller_RequiresIdempotencyHeader_AndDerivesActorFromPrincipal()
    {
        var service = new CapturingRunService();
        var controller = new OpenClawRunsController(service);
        var context = new DefaultHttpContext();
        context.TraceIdentifier = "trace-http";
        context.User = new ClaimsPrincipal(
            new ClaimsIdentity(
                [new Claim("sub", "owner-subject"), new Claim(ClaimTypes.Role, "owner")],
                "test"));
        controller.ControllerContext = new ControllerContext { HttpContext = context };

        var missingHeader = await controller.Start(Request(), CancellationToken.None);
        Assert.IsType<BadRequestObjectResult>(missingHeader.Result);

        context.Request.Headers["Idempotency-Key"] = "controller-key";
        context.Request.Headers["X-Correlation-ID"] = "controller-correlation";
        var accepted = await controller.Start(Request(), CancellationToken.None);

        Assert.IsType<CreatedAtRouteResult>(accepted.Result);
        Assert.Equal("owner-subject", service.Invocation?.Actor);
        Assert.Equal("controller-key", service.Invocation?.IdempotencyKey);
        Assert.Equal("controller-correlation", service.Invocation?.CorrelationId);

        context.Request.Headers["traceparent"] = "not-a-w3c-traceparent";
        var invalidTrace = await controller.Start(Request(), CancellationToken.None);
        Assert.IsType<BadRequestObjectResult>(invalidTrace.Result);

        context.Request.Headers.Remove("traceparent");
        service.StartOk = false;
        var durablyBlocked = await controller.Start(Request(), CancellationToken.None);
        Assert.IsType<AcceptedAtRouteResult>(durablyBlocked.Result);
    }

    [Fact]
    public void Controller_MutationsRequireOwnerRole()
    {
        var classAuthorization = typeof(OpenClawRunsController)
            .GetCustomAttributes(typeof(AuthorizeAttribute), inherit: true)
            .Cast<AuthorizeAttribute>()
            .Single(attribute => string.IsNullOrWhiteSpace(attribute.Roles));
        var startAuthorization = typeof(OpenClawRunsController)
            .GetMethod(nameof(OpenClawRunsController.Start))!
            .GetCustomAttributes(typeof(AuthorizeAttribute), inherit: true)
            .Cast<AuthorizeAttribute>()
            .Single();

        Assert.NotNull(classAuthorization);
        Assert.Equal("owner", startAuthorization.Roles);
    }

    private static StartOpenClawRunRequest Request()
        => new(
            "Inspect and report the current state.",
            "iris",
            "agent:iris:main",
            "Inspect state");

    private static OpenClawRun CreateListedRun(
        string title,
        DateTimeOffset createdAt)
        => new()
        {
            Title = title,
            Prompt = $"Inspect {title}",
            AgentId = "iris",
            SessionKey = "agent:iris:main",
            Status = OpenClawRunStates.Completed,
            StartIdempotencyKey = $"list-{title}",
            CorrelationId = $"correlation-{title}",
            Actor = "owner",
            CreatedAt = createdAt,
            UpdatedAt = createdAt
        };

    private static OpenClawInvocationMetadata Invocation(
        string idempotencyKey,
        string correlationId,
        string actor)
        => new(
            idempotencyKey,
            correlationId,
            actor,
            "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01");

    private static GatewayEventEnvelope ChatEvent(
        string runId,
        string state,
        long sequence,
        DateTimeOffset receivedAt)
        => new(
            "chat",
            new JsonObject
            {
                ["runId"] = runId,
                ["sessionKey"] = "agent:iris:main",
                ["state"] = state,
                ["seq"] = sequence
            },
            sequence,
            sequence,
            receivedAt);

    private sealed class RunHarness : IAsyncDisposable
    {
        private RunHarness(
            NexusDbContext db,
            FakeRunGateway gateway,
            OpenClawRunService service)
        {
            Db = db;
            Gateway = gateway;
            Service = service;
        }

        public NexusDbContext Db { get; }
        public FakeRunGateway Gateway { get; }
        public OpenClawRunService Service { get; }

        public static async Task<RunHarness> CreateAsync()
        {
            var options = new DbContextOptionsBuilder<NexusDbContext>()
                .UseInMemoryDatabase($"openclaw-runs-{Guid.NewGuid():N}")
                .Options;
            var db = new NexusDbContext(options);
            await db.Database.EnsureCreatedAsync();
            var gateway = new FakeRunGateway();
            var repository = new OpenClawRunRepository(db);
            return new RunHarness(db, gateway, new OpenClawRunService(repository, gateway));
        }

        public ValueTask DisposeAsync() => Db.DisposeAsync();
    }

    private sealed class FakeRunGateway : IOpenClawRunGateway
    {
        public Queue<OpenClawRunGatewayResult> StartResults { get; } = new();
        public Queue<OpenClawRunGatewayResult> StopResults { get; } = new();
        public List<(Guid RunId, OpenClawInvocationMetadata Invocation)> StartCalls { get; } = [];
        public List<(Guid RunId, OpenClawInvocationMetadata Invocation)> StopCalls { get; } = [];

        public Task<OpenClawRunGatewayResult> StartAsync(
            OpenClawRun run,
            OpenClawInvocationMetadata invocation,
            CancellationToken cancellationToken = default)
        {
            StartCalls.Add((run.Id, invocation));
            return Task.FromResult(StartResults.Count > 0
                ? StartResults.Dequeue()
                : new OpenClawRunGatewayResult(
                    true,
                    true,
                    OpenClawRunStates.Running,
                    "started",
                    $"oc-{run.Id:N}"));
        }

        public Task<OpenClawRunGatewayResult> StopAsync(
            OpenClawRun run,
            OpenClawInvocationMetadata invocation,
            CancellationToken cancellationToken = default)
        {
            StopCalls.Add((run.Id, invocation));
            return Task.FromResult(StopResults.Count > 0
                ? StopResults.Dequeue()
                : new OpenClawRunGatewayResult(
                    true,
                    true,
                    OpenClawRunStates.Stopped,
                    "stopped",
                    run.OpenClawRunId));
        }

        public Task<OpenClawRunGatewayResult> GetHistoryAsync(
            OpenClawRun run,
            int limit,
            CancellationToken cancellationToken = default)
            => Task.FromResult(new OpenClawRunGatewayResult(
                true,
                true,
                "available",
                "history",
                run.OpenClawRunId,
                new JsonObject { ["messages"] = new JsonArray() }));
    }

    private sealed class CapturingRunService : IOpenClawRunService
    {
        public OpenClawInvocationMetadata? Invocation { get; private set; }
        public bool StartOk { get; set; } = true;

        public Task<OpenClawRunOperationDto> StartAsync(
            StartOpenClawRunRequest request,
            OpenClawInvocationMetadata invocation,
            CancellationToken cancellationToken = default)
        {
            Invocation = invocation;
            var now = DateTimeOffset.UtcNow;
            var run = new OpenClawRunDto(
                Guid.NewGuid(),
                request.Title ?? "Run",
                request.Prompt,
                request.AgentId,
                request.SessionKey,
                StartOk ? OpenClawRunStates.Running : OpenClawRunStates.Blocked,
                request.TaskId,
                request.ProjectId,
                "oc-run",
                null,
                invocation.CorrelationId,
                invocation.Actor,
                null,
                null,
                false,
                StartOk,
                !StartOk,
                false,
                "unsupported",
                now,
                now,
                now,
                null);
            return Task.FromResult(new OpenClawRunOperationDto(
                StartOk,
                StartOk ? OpenClawRunStates.Running : OpenClawRunStates.Blocked,
                StartOk ? "started" : "gateway disconnected",
                run,
                null,
                now));
        }

        public Task<OpenClawRunCollectionDto> GetAsync(
            OpenClawRunQuery query,
            CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
        public Task<OpenClawRunDto?> GetByIdAsync(Guid id, CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
        public Task<OpenClawRunOperationDto?> StopAsync(
            Guid id,
            string? reason,
            OpenClawInvocationMetadata invocation,
            CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
        public Task<OpenClawRunOperationDto?> ResumeAsync(
            Guid id,
            string? reason,
            OpenClawInvocationMetadata invocation,
            CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
        public Task<OpenClawRunOperationDto?> RetryAsync(
            Guid id,
            string? reason,
            OpenClawInvocationMetadata invocation,
            CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
        public Task<OpenClawRunHistoryResponse?> GetHistoryAsync(
            Guid id,
            int gatewayLimit = 200,
            CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
        public Task ReconcileAsync(
            GatewayEventEnvelope gatewayEvent,
            CancellationToken cancellationToken = default)
            => throw new NotSupportedException();
    }
}