749 lines
35 KiB
Go
749 lines
35 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
func TestHotPathLightLocalTransition(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
t.Run(endpoint, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
final := fixture.run()
|
|
if final.Code != http.StatusOK || !strings.Contains(final.Body.String(), "review-resolution-visible") {
|
|
t.Fatalf("final response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
history, _ := json.Marshal(fixture.history)
|
|
if !strings.Contains(string(history), "local-complete-visible") {
|
|
t.Fatalf("local completion was not visible before review: history=%s", history)
|
|
}
|
|
fixture.assertCleanupCommitted(7)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHotPathStageInputIsolation(t *testing.T) {
|
|
paths := newReservedPaths("req_stage_isolation")
|
|
selector := hotPathStageCorrelation{StageID: "stg_selector", ResponseID: "provider:selector.actual/1", RunID: "run-selector", ProviderID: "provider.actual", Terminal: "stop,done\"quoted\""}
|
|
local := hotPathStageCorrelation{StageID: "stg_local", ResponseID: "provider:local.actual/2", RunID: "run-local", ProviderID: "provider.actual", Terminal: "tool_calls,stop"}
|
|
localInput := buildLocalStageInput("immutable user task", paths, selector)
|
|
reviewInput := buildReviewStageInput("immutable user task", paths, selector, local)
|
|
|
|
for _, input := range []hotPathStageInput{localInput, reviewInput} {
|
|
phase := hotPathPhaseLocalActive
|
|
if input.Role == "review" {
|
|
phase = hotPathPhaseReviewActive
|
|
}
|
|
prompt, err := input.prompt(phase)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{"PLAN_FILE_SECRET", "credential-secret", "previous internal prompt", "provider-target.internal"} {
|
|
if strings.Contains(prompt, forbidden) {
|
|
t.Fatalf("stage prompt leaked %q: %s", forbidden, prompt)
|
|
}
|
|
}
|
|
if !strings.Contains(prompt, "immutable user task") || !strings.Contains(prompt, paths.PlanPath) || !strings.Contains(prompt, paths.ReviewPath) {
|
|
t.Fatalf("stage prompt omitted immutable input: %s", prompt)
|
|
}
|
|
|
|
// Exact committed selector correlation must be present for both roles.
|
|
if !strings.Contains(prompt, "Committed selector stage success:") {
|
|
t.Fatalf("prompt missing committed selector correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, selector.StageID) || !strings.Contains(prompt, selector.RunID) {
|
|
t.Fatalf("prompt omitted exact selector correlation fields: %s", prompt)
|
|
}
|
|
|
|
// Verify serialized JSON block decoding and single-line format
|
|
selHeaderIdx := strings.Index(prompt, "Committed selector stage success:\n")
|
|
if selHeaderIdx == -1 {
|
|
t.Fatalf("prompt missing selector header format")
|
|
}
|
|
selJSONLine := prompt[selHeaderIdx+len("Committed selector stage success:\n"):]
|
|
if newlineIdx := strings.IndexByte(selJSONLine, '\n'); newlineIdx != -1 {
|
|
selJSONLine = selJSONLine[:newlineIdx]
|
|
}
|
|
var selDecoded correlationPromptValue
|
|
if err := json.Unmarshal([]byte(selJSONLine), &selDecoded); err != nil {
|
|
t.Fatalf("failed to decode selector correlation JSON line %q: %v", selJSONLine, err)
|
|
}
|
|
if selDecoded.StageID != selector.StageID || selDecoded.ResponseID != selector.ResponseID || selDecoded.RunID != selector.RunID || selDecoded.ProviderID != selector.ProviderID || selDecoded.Terminal != selector.Terminal {
|
|
t.Fatalf("decoded selector correlation mismatch: got %#v want %#v", selDecoded, selector)
|
|
}
|
|
|
|
// Local stage must NOT carry a local correlation.
|
|
if input.Role == "local" {
|
|
if strings.Contains(prompt, "Committed local stage success:") {
|
|
t.Fatalf("local prompt leaked local correlation: %s", prompt)
|
|
}
|
|
if strings.Contains(prompt, local.StageID) {
|
|
t.Fatalf("local prompt contained local correlation fields: %s", prompt)
|
|
}
|
|
}
|
|
|
|
// Review stage must carry both selector and local correlations.
|
|
if input.Role == "review" {
|
|
if !strings.Contains(prompt, "Committed local stage success:") {
|
|
t.Fatalf("review prompt missing committed local correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, local.StageID) || !strings.Contains(prompt, local.RunID) {
|
|
t.Fatalf("review prompt omitted exact local correlation fields: %s", prompt)
|
|
}
|
|
|
|
locHeaderIdx := strings.Index(prompt, "Committed local stage success:\n")
|
|
if locHeaderIdx == -1 {
|
|
t.Fatalf("prompt missing local header format")
|
|
}
|
|
locJSONLine := prompt[locHeaderIdx+len("Committed local stage success:\n"):]
|
|
if newlineIdx := strings.IndexByte(locJSONLine, '\n'); newlineIdx != -1 {
|
|
locJSONLine = locJSONLine[:newlineIdx]
|
|
}
|
|
var locDecoded correlationPromptValue
|
|
if err := json.Unmarshal([]byte(locJSONLine), &locDecoded); err != nil {
|
|
t.Fatalf("failed to decode local correlation JSON line %q: %v", locJSONLine, err)
|
|
}
|
|
if locDecoded.StageID != local.StageID || locDecoded.ResponseID != local.ResponseID || locDecoded.RunID != local.RunID || locDecoded.ProviderID != local.ProviderID || locDecoded.Terminal != local.Terminal {
|
|
t.Fatalf("decoded local correlation mismatch: got %#v want %#v", locDecoded, local)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test invalid correlation field values fail closed for opaque fields.
|
|
invalidOpaqueValues := []string{
|
|
"",
|
|
"invalid\nvalue",
|
|
"invalid\rvalue",
|
|
"invalid\tvalue",
|
|
strings.Repeat("a", 257),
|
|
}
|
|
|
|
for _, invalid := range invalidOpaqueValues {
|
|
// Mutate Selector ResponseID
|
|
selBadResponse := selector
|
|
selBadResponse.ResponseID = invalid
|
|
inputBadSelResponse := buildLocalStageInput("immutable user task", paths, selBadResponse)
|
|
if p, err := inputBadSelResponse.prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector ResponseID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Selector ProviderID
|
|
selBadProvider := selector
|
|
selBadProvider.ProviderID = invalid
|
|
inputBadSelProvider := buildLocalStageInput("immutable user task", paths, selBadProvider)
|
|
if p, err := inputBadSelProvider.prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector ProviderID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Selector Terminal
|
|
selBadTerminal := selector
|
|
selBadTerminal.Terminal = invalid
|
|
inputBadSelTerminal := buildLocalStageInput("immutable user task", paths, selBadTerminal)
|
|
if p, err := inputBadSelTerminal.prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector Terminal %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Local ResponseID in review stage
|
|
localBadResponse := local
|
|
localBadResponse.ResponseID = invalid
|
|
inputBadLocalResponse := buildReviewStageInput("immutable user task", paths, selector, localBadResponse)
|
|
if p, err := inputBadLocalResponse.prompt(hotPathPhaseReviewActive); err == nil || p != "" {
|
|
t.Fatalf("local ResponseID %q accepted in review stage: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Local ProviderID in review stage
|
|
localBadProvider := local
|
|
localBadProvider.ProviderID = invalid
|
|
inputBadLocalProvider := buildReviewStageInput("immutable user task", paths, selector, localBadProvider)
|
|
if p, err := inputBadLocalProvider.prompt(hotPathPhaseReviewActive); err == nil || p != "" {
|
|
t.Fatalf("local ProviderID %q accepted in review stage: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Local Terminal in review stage
|
|
localBadTerminal := local
|
|
localBadTerminal.Terminal = invalid
|
|
inputBadLocalTerminal := buildReviewStageInput("immutable user task", paths, selector, localBadTerminal)
|
|
if p, err := inputBadLocalTerminal.prompt(hotPathPhaseReviewActive); err == nil || p != "" {
|
|
t.Fatalf("local Terminal %q accepted in review stage: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
}
|
|
|
|
// Test invalid IOP-owned ID field values fail closed.
|
|
invalidLogicalIDs := []string{
|
|
"",
|
|
"invalid:value",
|
|
"invalid,value",
|
|
"invalid.value",
|
|
"invalid\nvalue",
|
|
strings.Repeat("a", 257),
|
|
}
|
|
|
|
for _, invalid := range invalidLogicalIDs {
|
|
selBadStage := selector
|
|
selBadStage.StageID = invalid
|
|
if p, err := buildLocalStageInput("immutable user task", paths, selBadStage).prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector StageID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
selBadRun := selector
|
|
selBadRun.RunID = invalid
|
|
if p, err := buildLocalStageInput("immutable user task", paths, selBadRun).prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector RunID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
}
|
|
|
|
pinned := routeDispatch{
|
|
Managed: true, PrincipalRef: "principal", ModelGroupKey: "local-model", RouteID: "route-local",
|
|
CredentialSlotRef: "slot-local", ProfileID: "profile", UpstreamModel: "served-local",
|
|
ResourceSelector: "resource", RouteRevision: 7, CredentialRevision: 11, ProjectionGeneration: 13,
|
|
}
|
|
changed := pinned
|
|
changed.CredentialRevision++
|
|
if samePinnedHotPathRoute(pinned, changed) {
|
|
t.Fatal("credential revision drift was accepted")
|
|
}
|
|
changed = pinned
|
|
changed.RouteRevision++
|
|
if samePinnedHotPathRoute(pinned, changed) {
|
|
t.Fatal("route revision drift was accepted")
|
|
}
|
|
}
|
|
|
|
type scriptedLightPoolService struct {
|
|
providerFakeRunService
|
|
mu sync.Mutex
|
|
endpoint string
|
|
candidate edgeservice.ProviderPoolCandidate
|
|
responses []func(string) string
|
|
requests []edgeservice.ProviderPoolDispatchRequest
|
|
}
|
|
|
|
func (s *scriptedLightPoolService) SubmitProviderPool(ctx context.Context, req edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
s.mu.Lock()
|
|
index := len(s.requests)
|
|
s.requests = append(s.requests, req)
|
|
if index >= len(s.responses) {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("unexpected light stage dispatch %d", index+1)
|
|
}
|
|
response := s.responses[index]
|
|
candidate := s.candidate
|
|
endpoint := s.endpoint
|
|
s.mu.Unlock()
|
|
|
|
requestID := req.Run.Metadata["iop_logical_request_id"]
|
|
body := response(requestID)
|
|
dispatch := edgeservice.RunDispatch{
|
|
RunID: fmt.Sprintf("run-light-%d", index+1), NodeID: "node-light", ModelGroupKey: req.Run.ModelGroupKey,
|
|
ProviderID: candidate.ProviderID, ExecutionPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
ProfileID: candidate.ProfileID, ProfileDriver: candidate.ProfileDriver,
|
|
ProfileCapabilities: append([]string(nil), candidate.ProfileCapabilities...),
|
|
}
|
|
frames := staticProviderTunnelFrames(body)
|
|
if endpoint == "anthropic" {
|
|
frames = anthropicTunnelFrames(http.StatusOK, "application/json", []byte(body))
|
|
}
|
|
return &edgeservice.ProviderPoolDispatchResult{
|
|
Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &fakeTunnelHandle{dispatch: dispatch, frames: frames}, DispatchInfo: dispatch,
|
|
}, nil
|
|
}
|
|
|
|
func (s *scriptedLightPoolService) snapshots() []edgeservice.ProviderPoolDispatchRequest {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]edgeservice.ProviderPoolDispatchRequest(nil), s.requests...)
|
|
}
|
|
|
|
type scriptedLightFixture struct {
|
|
t *testing.T
|
|
endpoint string
|
|
server *Server
|
|
service *scriptedLightPoolService
|
|
tools []any
|
|
history []any
|
|
repair bool
|
|
}
|
|
|
|
func newScriptedLightFixture(t *testing.T, endpoint string, repair bool) *scriptedLightFixture {
|
|
t.Helper()
|
|
candidate := anthropicTestCandidate(t, map[string]string{"openai": "openai", "anthropic": "anthropic"}[endpoint])
|
|
service := &scriptedLightPoolService{endpoint: endpoint, candidate: candidate}
|
|
service.responses = []func(string) string{
|
|
func(requestID string) string { return scriptedArtifactPrepare(endpoint, requestID) },
|
|
func(requestID string) string { return scriptedArtifactPair(endpoint, requestID) },
|
|
func(requestID string) string { return scriptedArtifactLocalRead(endpoint, requestID) },
|
|
func(string) string { return scriptedLightCompletion(endpoint, "local-complete-visible") },
|
|
func(requestID string) string { return scriptedReviewWrite(endpoint, requestID) },
|
|
func(requestID string) string { return scriptedReviewRead(endpoint, requestID) },
|
|
}
|
|
if repair {
|
|
service.responses = append(service.responses,
|
|
func(string) string { return scriptedRepairTool(endpoint) },
|
|
func(string) string { return scriptedLightCompletion(endpoint, "repair-complete-visible") },
|
|
)
|
|
} else {
|
|
service.responses = append(service.responses, func(string) string {
|
|
return scriptedLightCompletion(endpoint, "review-resolution-visible PASS and DEFECT prose")
|
|
})
|
|
}
|
|
|
|
preset := hotPathSelectorPreset([]string{config.ModeDirect, config.ModeLight})
|
|
preset.WorkspaceTools = []config.ExecutionWorkspaceToolAlternative{scriptedLightWorkspaceAlternative()}
|
|
server := NewServer(config.EdgeOpenAIConf{}, service, nil)
|
|
server.SetEdgeID("edge-scripted-light")
|
|
server.SetExecutionPresets([]config.ExecutionPreset{preset})
|
|
server.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "virtual-model", ExecutionPreset: preset.ID},
|
|
{ID: "selector-model", Providers: map[string]string{candidate.ProviderID: "served-selector"}},
|
|
{ID: "local-model", Providers: map[string]string{candidate.ProviderID: "served-local"}},
|
|
{ID: "review-model", Providers: map[string]string{candidate.ProviderID: "served-review"}},
|
|
})
|
|
tools := scriptedLightTools(endpoint)
|
|
return &scriptedLightFixture{
|
|
t: t, endpoint: endpoint, server: server, service: service, tools: tools,
|
|
history: []any{map[string]any{"role": "user", "content": "immutable user task"}}, repair: repair,
|
|
}
|
|
}
|
|
|
|
func scriptedLightWorkspaceAlternative() config.ExecutionWorkspaceToolAlternative {
|
|
matcher := successMatcher()
|
|
return config.ExecutionWorkspaceToolAlternative{
|
|
Name: "scripted-light-tools",
|
|
Operations: map[string]config.ExecutionWorkspaceOperation{
|
|
"prepare": {ToolName: "mkdir_p", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path"}, ResultMatcher: matcher, CreatesParents: true},
|
|
"read": {ToolName: "read_file", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path"}, ResultMatcher: matcher},
|
|
"write": {ToolName: "write_file", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path", "content": "content"}, ResultMatcher: matcher, CreatesParents: false},
|
|
"delete": {ToolName: "delete_file", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path"}, ResultMatcher: matcher},
|
|
},
|
|
}
|
|
}
|
|
|
|
func scriptedLightTools(endpoint string) []any {
|
|
tools := scriptedArtifactTools(endpoint)
|
|
schema := map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string"}}, "required": []any{"command"}}
|
|
if endpoint == "anthropic" {
|
|
return append(tools, anthropicWorkspaceTool("run_command", schema))
|
|
}
|
|
return append(tools, openAIChatTool("run_command", schema))
|
|
}
|
|
|
|
func (f *scriptedLightFixture) run() *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
cleanup := f.runToCleanup()
|
|
f.consumeToolResponse(cleanup, []string{`{"written":true}`})
|
|
return f.request()
|
|
}
|
|
|
|
func (f *scriptedLightFixture) runToCleanup() *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
prepare := f.request()
|
|
f.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := f.request()
|
|
f.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
|
|
localRead := f.request()
|
|
f.consumeToolResponse(localRead, []string{`{"written":true}`})
|
|
reviewWrite := f.request()
|
|
f.consumeToolResponse(reviewWrite, []string{`{"written":true}`})
|
|
reviewRead := f.request()
|
|
f.consumeToolResponse(reviewRead, []string{`{"written":true}`})
|
|
resolution := f.request()
|
|
if !f.repair {
|
|
return resolution
|
|
}
|
|
f.consumeToolResponse(resolution, []string{`{"ok":true}`})
|
|
return f.request()
|
|
}
|
|
|
|
func (f *scriptedLightFixture) request() *httptest.ResponseRecorder {
|
|
return f.requestWithOptions(0, false)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) requestWithOptions(outputCap int, stream bool) *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
body := scriptedArtifactRequestBodyWithOptions(f.t, f.endpoint, f.tools, f.history, outputCap, stream)
|
|
return serveScriptedArtifactRequest(f.t, f.server, f.endpoint, body)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) requestWithContext(ctx context.Context, outputCap int) *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
body := scriptedArtifactRequestBodyWithOptions(f.t, f.endpoint, f.tools, f.history, outputCap, false)
|
|
return serveScriptedArtifactRequestContext(f.t, f.server, f.endpoint, body, ctx)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) consumeToolResponse(response *httptest.ResponseRecorder, results []string) {
|
|
f.t.Helper()
|
|
if response.Code != http.StatusOK {
|
|
f.t.Fatalf("tool response status=%d body=%s", response.Code, response.Body.String())
|
|
}
|
|
assistant, ids, err := artifactAssistantFromResponse(f.endpoint, response.Body.Bytes())
|
|
if err != nil || len(ids) != len(results) {
|
|
f.t.Fatalf("decode tool response: ids=%v results=%v err=%v body=%s", ids, results, err, response.Body.String())
|
|
}
|
|
f.history = append(f.history, assistant)
|
|
f.history = scriptedArtifactAppendResults(f.endpoint, f.history, ids, results)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) assertCleanupCommitted(wantCalls int) {
|
|
f.t.Helper()
|
|
requests := f.service.snapshots()
|
|
if len(requests) != wantCalls {
|
|
f.t.Fatalf("provider calls=%d, want %d", len(requests), wantCalls)
|
|
}
|
|
if requests[0].Run.ModelGroupKey != "selector-model" || requests[1].Run.ModelGroupKey != "selector-model" {
|
|
f.t.Fatalf("selector model groups changed: %q %q", requests[0].Run.ModelGroupKey, requests[0].Run.ModelGroupKey)
|
|
}
|
|
if requests[2].Run.ModelGroupKey != "local-model" || requests[3].Run.ModelGroupKey != "local-model" {
|
|
f.t.Fatalf("local model group changed: %q %q", requests[2].Run.ModelGroupKey, requests[3].Run.ModelGroupKey)
|
|
}
|
|
localStage := requests[2].Run.Metadata["iop_stage_id"]
|
|
if localStage == "" || requests[3].Run.Metadata["iop_stage_id"] != localStage {
|
|
f.t.Fatalf("local stage was not resumed: %#v %#v", requests[2].Run.Metadata, requests[3].Run.Metadata)
|
|
}
|
|
reviewStage := requests[4].Run.Metadata["iop_stage_id"]
|
|
if reviewStage == "" || reviewStage == localStage {
|
|
f.t.Fatalf("review stage identity is not fixed and distinct: local=%q review=%q", localStage, reviewStage)
|
|
}
|
|
for index := 4; index < len(requests); index++ {
|
|
if requests[index].Run.ModelGroupKey != "review-model" || requests[index].Run.Metadata["iop_stage_id"] != reviewStage {
|
|
f.t.Fatalf("review dispatch %d changed binding: group=%q metadata=%#v", index, requests[index].Run.ModelGroupKey, requests[index].Run.Metadata)
|
|
}
|
|
}
|
|
|
|
selectorStage := requests[1].Run.Metadata["iop_stage_id"]
|
|
selectorResponse := "chatcmpl-scripted-pair"
|
|
if f.endpoint == "anthropic" {
|
|
selectorResponse = "msg-scripted-pair"
|
|
}
|
|
|
|
localResponse := "chatcmpl-light-complete"
|
|
if f.endpoint == "anthropic" {
|
|
localResponse = "msg-light-complete"
|
|
}
|
|
|
|
// Regression: local stage must carry selector correlation and must NOT
|
|
// carry local correlation in both normalized Run.Input and tunnel body.
|
|
assertLocalCorrelationRegression(f.t, requests[2], f.service.candidate, selectorStage, selectorResponse)
|
|
assertLocalCorrelationRegression(f.t, requests[3], f.service.candidate, selectorStage, selectorResponse)
|
|
|
|
// Regression: review stage must carry both selector and local correlations
|
|
// in both normalized Run.Input and tunnel body.
|
|
assertReviewCorrelationRegression(f.t, requests[4], f.service.candidate, selectorStage, selectorResponse, localStage, localResponse)
|
|
assertReviewCorrelationRegression(f.t, requests[5], f.service.candidate, selectorStage, selectorResponse, localStage, localResponse)
|
|
|
|
// Regression: forbidden data must not appear in any provider-visible payload.
|
|
for index, req := range requests {
|
|
for _, forbidden := range []string{"PLAN_FILE_SECRET", "credential-secret", "previous internal prompt", "provider-target.internal"} {
|
|
if strings.Contains(req.Run.Prompt, forbidden) {
|
|
f.t.Fatalf("request %d Run.Prompt leaked %q", index, forbidden)
|
|
}
|
|
if body, ok := req.Run.Input["prompt"]; ok {
|
|
if strings.Contains(fmt.Sprint(body), forbidden) {
|
|
f.t.Fatalf("request %d Run.Input[\"prompt\"] leaked %q", index, forbidden)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
f.assertCleanupStoresRemoved()
|
|
}
|
|
|
|
func (f *scriptedLightFixture) assertCleanupStoresRemoved() {
|
|
f.t.Helper()
|
|
f.server.lightFlows.mu.Lock()
|
|
lightCount := len(f.server.lightFlows.records)
|
|
f.server.lightFlows.mu.Unlock()
|
|
if lightCount != 0 {
|
|
f.t.Fatalf("light records=%d, want 0 after cleanup commit", lightCount)
|
|
}
|
|
f.server.artifactFrontiers.mu.Lock()
|
|
artifactCount := len(f.server.artifactFrontiers.records)
|
|
f.server.artifactFrontiers.mu.Unlock()
|
|
if artifactCount != 0 {
|
|
f.t.Fatalf("artifact records=%d, want 0 after cleanup commit", artifactCount)
|
|
}
|
|
f.server.requestCoordinator.mu.Lock()
|
|
coordinatorCount := len(f.server.requestCoordinator.requests)
|
|
f.server.requestCoordinator.mu.Unlock()
|
|
if coordinatorCount != 0 {
|
|
f.t.Fatalf("coordinator records=%d, want 0 after cleanup commit", coordinatorCount)
|
|
}
|
|
}
|
|
|
|
// assertLocalCorrelationRegression verifies that a captured local-stage request
|
|
// carries the committed selector correlation in Run.Prompt, Run.Input["prompt"],
|
|
// and the decoded tunnel body, while omitting any local-stage correlation.
|
|
func assertLocalCorrelationRegression(t *testing.T, req edgeservice.ProviderPoolDispatchRequest, selected edgeservice.ProviderPoolCandidate, selectorStage, selectorResponse string) {
|
|
t.Helper()
|
|
prompt := req.Run.Prompt
|
|
if prompt == "" {
|
|
t.Fatalf("local request prompt is empty")
|
|
}
|
|
input, ok := req.Run.Input["prompt"]
|
|
if !ok || input == nil {
|
|
t.Fatalf("local Run.Input[\"prompt\"] is missing")
|
|
}
|
|
inputStr := fmt.Sprint(input)
|
|
|
|
if !strings.Contains(prompt, "Committed selector stage success:") {
|
|
t.Fatalf("local Run.Prompt missing selector correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, selectorStage) || !strings.Contains(prompt, selectorResponse) {
|
|
t.Fatalf("local Run.Prompt missing exact selector stage/response %q/%q: %s", selectorStage, selectorResponse, prompt)
|
|
}
|
|
|
|
if !strings.Contains(inputStr, "Committed selector stage success:") {
|
|
t.Fatalf("local Run.Input[\"prompt\"] missing selector correlation: %v", input)
|
|
}
|
|
if !strings.Contains(inputStr, selectorStage) || !strings.Contains(inputStr, selectorResponse) {
|
|
t.Fatalf("local Run.Input[\"prompt\"] missing exact selector stage/response %q/%q: %v", selectorStage, selectorResponse, input)
|
|
}
|
|
|
|
if strings.Contains(prompt, "Committed local stage success:") {
|
|
t.Fatalf("local Run.Prompt leaked local correlation: %s", prompt)
|
|
}
|
|
if strings.Contains(inputStr, "Committed local stage success:") {
|
|
t.Fatalf("local Run.Input[\"prompt\"] leaked local correlation: %v", input)
|
|
}
|
|
|
|
// Mandatory: decode and verify selected protocol tunnel prompt.
|
|
_, tunnelPrompt, err := decodeSelectedTunnelPrompt(req, selected)
|
|
if err != nil {
|
|
t.Fatalf("local tunnel decode error: %v", err)
|
|
}
|
|
if tunnelPrompt != prompt {
|
|
t.Fatalf("local decoded tunnel prompt mismatch: got %q want %q", tunnelPrompt, prompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, "Committed selector stage success:") {
|
|
t.Fatalf("local tunnel body missing selector correlation: %s", tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, selectorStage) || !strings.Contains(tunnelPrompt, selectorResponse) {
|
|
t.Fatalf("local tunnel body missing exact selector stage/response %q/%q: %s", selectorStage, selectorResponse, tunnelPrompt)
|
|
}
|
|
if strings.Contains(tunnelPrompt, "Committed local stage success:") {
|
|
t.Fatalf("local tunnel body leaked local correlation: %s", tunnelPrompt)
|
|
}
|
|
}
|
|
|
|
// assertReviewCorrelationRegression verifies that a captured review-stage request
|
|
// carries both committed selector and local correlations in Run.Prompt,
|
|
// Run.Input["prompt"], and the decoded tunnel body.
|
|
func assertReviewCorrelationRegression(t *testing.T, req edgeservice.ProviderPoolDispatchRequest, selected edgeservice.ProviderPoolCandidate, selectorStage, selectorResponse, localStage, localResponse string) {
|
|
t.Helper()
|
|
prompt := req.Run.Prompt
|
|
if prompt == "" {
|
|
t.Fatalf("review request prompt is empty")
|
|
}
|
|
input, ok := req.Run.Input["prompt"]
|
|
if !ok || input == nil {
|
|
t.Fatalf("review Run.Input[\"prompt\"] is missing")
|
|
}
|
|
inputStr := fmt.Sprint(input)
|
|
|
|
if !strings.Contains(prompt, "Committed selector stage success:") {
|
|
t.Fatalf("review Run.Prompt missing selector correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, "Committed local stage success:") {
|
|
t.Fatalf("review Run.Prompt missing local correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, selectorStage) || !strings.Contains(prompt, selectorResponse) {
|
|
t.Fatalf("review Run.Prompt missing exact selector stage/response %q/%q: %s", selectorStage, selectorResponse, prompt)
|
|
}
|
|
if !strings.Contains(prompt, localStage) || !strings.Contains(prompt, localResponse) {
|
|
t.Fatalf("review Run.Prompt missing exact local stage/response %q/%q: %s", localStage, localResponse, prompt)
|
|
}
|
|
|
|
if !strings.Contains(inputStr, "Committed selector stage success:") {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing selector correlation: %v", input)
|
|
}
|
|
if !strings.Contains(inputStr, "Committed local stage success:") {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing local correlation: %v", input)
|
|
}
|
|
if !strings.Contains(inputStr, selectorStage) || !strings.Contains(inputStr, selectorResponse) {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing exact selector stage/response %q/%q: %v", selectorStage, selectorResponse, input)
|
|
}
|
|
if !strings.Contains(inputStr, localStage) || !strings.Contains(inputStr, localResponse) {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing exact local stage/response %q/%q: %v", localStage, localResponse, input)
|
|
}
|
|
|
|
// Mandatory: decode and verify selected protocol tunnel prompt.
|
|
_, tunnelPrompt, err := decodeSelectedTunnelPrompt(req, selected)
|
|
if err != nil {
|
|
t.Fatalf("review tunnel decode error: %v", err)
|
|
}
|
|
if tunnelPrompt != prompt {
|
|
t.Fatalf("review decoded tunnel prompt mismatch: got %q want %q", tunnelPrompt, prompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, "Committed selector stage success:") {
|
|
t.Fatalf("review tunnel body missing selector correlation: %s", tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, "Committed local stage success:") {
|
|
t.Fatalf("review tunnel body missing local correlation: %s", tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, selectorStage) || !strings.Contains(tunnelPrompt, selectorResponse) {
|
|
t.Fatalf("review tunnel body missing exact selector stage/response %q/%q: %s", selectorStage, selectorResponse, tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, localStage) || !strings.Contains(tunnelPrompt, localResponse) {
|
|
t.Fatalf("review tunnel body missing exact local stage/response %q/%q: %s", localStage, localResponse, tunnelPrompt)
|
|
}
|
|
}
|
|
|
|
// decodeSelectedTunnelPrompt invokes PrepareProtocolTunnel unconditionally, builds the protocol
|
|
// body, checks expected path/op for OpenAI vs Anthropic, and extracts the first user message content string.
|
|
func decodeSelectedTunnelPrompt(req edgeservice.ProviderPoolDispatchRequest, selected edgeservice.ProviderPoolCandidate) (edgeservice.SubmitProviderTunnelRequest, string, error) {
|
|
if req.PrepareProtocolTunnel == nil {
|
|
return edgeservice.SubmitProviderTunnelRequest{}, "", fmt.Errorf("PrepareProtocolTunnel is not set")
|
|
}
|
|
prepared, err := req.PrepareProtocolTunnel(req.Tunnel, selected)
|
|
if err != nil {
|
|
return prepared, "", fmt.Errorf("PrepareProtocolTunnel error: %w", err)
|
|
}
|
|
if prepared.BuildBody == nil {
|
|
return prepared, "", fmt.Errorf("BuildBody is not set after PrepareProtocolTunnel")
|
|
}
|
|
bodyBytes, err := prepared.BuildBody("target-model")
|
|
if err != nil {
|
|
return prepared, "", fmt.Errorf("BuildBody error: %w", err)
|
|
}
|
|
|
|
if selected.ProfileDriver == string(config.ProtocolDriverAnthropicMessages) {
|
|
if prepared.Path != "/v1/messages" || prepared.Operation != string(config.OperationMessages) {
|
|
return prepared, "", fmt.Errorf("anthropic tunnel path/op mismatch: path=%q op=%q", prepared.Path, prepared.Operation)
|
|
}
|
|
var payload struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content any `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(bodyBytes, &payload); err != nil {
|
|
return prepared, "", fmt.Errorf("unmarshal anthropic payload: %w (body=%s)", err, string(bodyBytes))
|
|
}
|
|
if len(payload.Messages) == 0 || payload.Messages[0].Role != "user" {
|
|
return prepared, "", fmt.Errorf("anthropic body missing first user message: %s", string(bodyBytes))
|
|
}
|
|
return prepared, extractMessageContentString(payload.Messages[0].Content), nil
|
|
} else {
|
|
if prepared.Path != "/v1/chat/completions" || prepared.Operation != string(config.OperationChatCompletions) {
|
|
return prepared, "", fmt.Errorf("openai tunnel path/op mismatch: path=%q op=%q", prepared.Path, prepared.Operation)
|
|
}
|
|
var payload struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content any `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(bodyBytes, &payload); err != nil {
|
|
return prepared, "", fmt.Errorf("unmarshal openai payload: %w (body=%s)", err, string(bodyBytes))
|
|
}
|
|
if len(payload.Messages) == 0 || payload.Messages[0].Role != "user" {
|
|
return prepared, "", fmt.Errorf("openai body missing first user message: %s", string(bodyBytes))
|
|
}
|
|
return prepared, extractMessageContentString(payload.Messages[0].Content), nil
|
|
}
|
|
}
|
|
|
|
func extractMessageContentString(content any) string {
|
|
switch v := content.(type) {
|
|
case string:
|
|
return v
|
|
case []any:
|
|
var parts []string
|
|
for _, item := range v {
|
|
if m, ok := item.(map[string]any); ok {
|
|
if text, ok := m["text"].(string); ok {
|
|
parts = append(parts, text)
|
|
}
|
|
}
|
|
}
|
|
return strings.Join(parts, "")
|
|
default:
|
|
return fmt.Sprint(content)
|
|
}
|
|
}
|
|
|
|
func scriptedLightCompletion(endpoint, content string) string {
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-light-complete","type":"message","role":"assistant","content":[{"type":"text","text":%q}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`, content)
|
|
}
|
|
raw, _ := json.Marshal(content)
|
|
return fmt.Sprintf(`{"id":"chatcmpl-light-complete","created":9,"choices":[{"message":{"role":"assistant","content":%s},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, raw)
|
|
}
|
|
|
|
func scriptedLightCompletionWithUsage(endpoint, content, reasoning string, inputTokens, outputTokens int) string {
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-light-complete","type":"message","role":"assistant","content":[{"type":"thinking","thinking":%q,"signature":"sig-local"},{"type":"text","text":%q}],"stop_reason":"end_turn","usage":{"input_tokens":%d,"output_tokens":%d}}`, reasoning, content, inputTokens, outputTokens)
|
|
}
|
|
contentRaw, _ := json.Marshal(content)
|
|
reasoningRaw, _ := json.Marshal(reasoning)
|
|
return fmt.Sprintf(`{"id":"chatcmpl-light-complete","created":9,"choices":[{"message":{"role":"assistant","content":%s,"reasoning_content":%s},"finish_reason":"stop"}],"usage":{"prompt_tokens":%d,"completion_tokens":%d,"total_tokens":%d}}`, contentRaw, reasoningRaw, inputTokens, outputTokens, inputTokens+outputTokens)
|
|
}
|
|
|
|
func scriptedReviewWrite(endpoint, requestID string) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-write","type":"message","role":"assistant","content":[{"type":"text","text":"review-write-visible"},{"type":"tool_use","id":"provider-review-write","name":"write_file","input":{"path":%q,"content":"review body"}}],"stop_reason":"tool_use"}`, path)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path, "content": "review body"})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-write","created":5,"choices":[{"message":{"role":"assistant","content":"review-write-visible","tool_calls":[{"id":"provider-review-write","type":"function","function":{"name":"write_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(args))
|
|
}
|
|
|
|
func scriptedReviewWriteWithUsage(endpoint, requestID string, inputTokens, outputTokens int) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-write","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"review-reason","signature":"sig-review"},{"type":"text","text":"review-visible"},{"type":"tool_use","id":"provider-review-write","name":"write_file","input":{"path":%q,"content":"review body"}}],"stop_reason":"tool_use","usage":{"input_tokens":%d,"output_tokens":%d}}`, path, inputTokens, outputTokens)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path, "content": "review body"})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-write","created":5,"choices":[{"message":{"role":"assistant","content":"review-visible","reasoning_content":"review-reason","tool_calls":[{"id":"provider-review-write","type":"function","function":{"name":"write_file","arguments":%q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":%d,"completion_tokens":%d,"total_tokens":%d}}`, string(args), inputTokens, outputTokens, inputTokens+outputTokens)
|
|
}
|
|
|
|
func assertCapturedHotPathBudget(t *testing.T, req edgeservice.ProviderPoolDispatchRequest, candidate edgeservice.ProviderPoolCandidate, want int) {
|
|
t.Helper()
|
|
options, ok := req.Run.Input["options"].(map[string]any)
|
|
if !ok || options["max_tokens"] != want {
|
|
t.Fatalf("normalized remaining cap = %#v, want %d", req.Run.Input["options"], want)
|
|
}
|
|
prepared, _, err := decodeSelectedTunnelPrompt(req, candidate)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, err := prepared.BuildBody("served-stage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var tunnel map[string]any
|
|
if err := json.Unmarshal(body, &tunnel); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tunnel["max_tokens"] != float64(want) {
|
|
t.Fatalf("tunnel remaining cap = %#v, want %d; body=%s", tunnel["max_tokens"], want, body)
|
|
}
|
|
}
|
|
|
|
func scriptedReviewRead(endpoint, requestID string) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-read","type":"message","role":"assistant","content":[{"type":"text","text":"review-read-visible"},{"type":"tool_use","id":"provider-review-read","name":"read_file","input":{"path":%q}}],"stop_reason":"tool_use"}`, path)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-read","created":6,"choices":[{"message":{"role":"assistant","content":"review-read-visible","tool_calls":[{"id":"provider-review-read","type":"function","function":{"name":"read_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(args))
|
|
}
|
|
|
|
func scriptedRepairTool(endpoint string) string {
|
|
if endpoint == "anthropic" {
|
|
return `{"id":"msg-repair","type":"message","role":"assistant","content":[{"type":"text","text":"PASS prose but repair tool decides"},{"type":"tool_use","id":"provider-repair","name":"run_command","input":{"command":"go test ./..."}}],"stop_reason":"tool_use"}`
|
|
}
|
|
return `{"id":"chatcmpl-repair","created":7,"choices":[{"message":{"role":"assistant","content":"PASS prose but repair tool decides","tool_calls":[{"id":"provider-repair","type":"function","function":{"name":"run_command","arguments":"{\"command\":\"go test ./...\"}"}}]},"finish_reason":"tool_calls"}]}`
|
|
}
|