feat: ship agent-first mission control v0.2.57

2026-07-31 22:39:47 +02:00
parent 3bc7622977
commit f5552218bc
535 changed files with 95242 additions and 8791 deletions
@@ -0,0 +1,59 @@
+[CmdletBinding()]
+param(
+    [string]$Config = "evals/promptfoo/promptfooconfig.yaml",
+    [switch]$ValidateOnly,
+    [ValidateRange(1, 100)]
+    [int]$Repeat = 1
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
+$configPath = if ([System.IO.Path]::IsPathRooted($Config)) {
+    $Config
+} else {
+    Join-Path $repoRoot $Config
+}
+
+if (-not (Test-Path -LiteralPath $configPath -PathType Leaf)) {
+    throw "Promptfoo config not found: $configPath"
+}
+
+$providerPath = Join-Path (Split-Path -Parent $configPath) "nexus-provider.cjs"
+if (-not (Test-Path -LiteralPath $providerPath -PathType Leaf)) {
+    throw "Promptfoo provider not found: $providerPath"
+}
+
+& node --check $providerPath
+if ($LASTEXITCODE -ne 0) {
+    throw "Promptfoo provider syntax validation failed."
+}
+
+if (-not $ValidateOnly) {
+    if ($env:NEXUS_EVAL_ALLOW_PROPOSALS -ne "1") {
+        throw "Set NEXUS_EVAL_ALLOW_PROPOSALS=1 after selecting an isolated test database."
+    }
+    if ([string]::IsNullOrWhiteSpace($env:NEXUS_EVAL_BEARER_TOKEN)) {
+        throw "Set NEXUS_EVAL_BEARER_TOKEN to a fresh owner JWT. The script never prints it."
+    }
+}
+
+Push-Location $repoRoot
+try {
+    if ($ValidateOnly) {
+        & npx --yes "promptfoo@0.121.19" validate --config $configPath
+    } else {
+        & npx --yes "promptfoo@0.121.19" eval `
+            --config $configPath `
+            --max-concurrency 1 `
+            --repeat $Repeat `
+            --no-cache
+    }
+    if ($LASTEXITCODE -ne 0) {
+        $operation = if ($ValidateOnly) { "validation" } else { "evaluation" }
+        throw "Promptfoo $operation failed with exit code $LASTEXITCODE."
+    }
+} finally {
+    Pop-Location
+}