AzuTear
60 lines
1.7 KiB
PowerShell
60 lines
1.7 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[string]$Config = "evals/promptfoo/promptfooconfig.yaml",
|
|
[switch]$ValidateOnly,
|
|
[ValidateRange(1, 100)]
|
|
[int]$Repeat = 1
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
|
|
$configPath = if ([System.IO.Path]::IsPathRooted($Config)) {
|
|
$Config
|
|
} else {
|
|
Join-Path $repoRoot $Config
|
|
}
|
|
|
|
if (-not (Test-Path -LiteralPath $configPath -PathType Leaf)) {
|
|
throw "Promptfoo config not found: $configPath"
|
|
}
|
|
|
|
$providerPath = Join-Path (Split-Path -Parent $configPath) "nexus-provider.cjs"
|
|
if (-not (Test-Path -LiteralPath $providerPath -PathType Leaf)) {
|
|
throw "Promptfoo provider not found: $providerPath"
|
|
}
|
|
|
|
& node --check $providerPath
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Promptfoo provider syntax validation failed."
|
|
}
|
|
|
|
if (-not $ValidateOnly) {
|
|
if ($env:NEXUS_EVAL_ALLOW_PROPOSALS -ne "1") {
|
|
throw "Set NEXUS_EVAL_ALLOW_PROPOSALS=1 after selecting an isolated test database."
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace($env:NEXUS_EVAL_BEARER_TOKEN)) {
|
|
throw "Set NEXUS_EVAL_BEARER_TOKEN to a fresh owner JWT. The script never prints it."
|
|
}
|
|
}
|
|
|
|
Push-Location $repoRoot
|
|
try {
|
|
if ($ValidateOnly) {
|
|
& npx --yes "promptfoo@0.121.19" validate --config $configPath
|
|
} else {
|
|
& npx --yes "promptfoo@0.121.19" eval `
|
|
--config $configPath `
|
|
--max-concurrency 1 `
|
|
--repeat $Repeat `
|
|
--no-cache
|
|
}
|
|
if ($LASTEXITCODE -ne 0) {
|
|
$operation = if ($ValidateOnly) { "validation" } else { "evaluation" }
|
|
throw "Promptfoo $operation failed with exit code $LASTEXITCODE."
|
|
}
|
|
} finally {
|
|
Pop-Location
|
|
}
|