338 lines
10 KiB
Go
338 lines
10 KiB
Go
package taskloop
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"iop/packages/go/agentconfig"
|
|
"iop/packages/go/agenttask"
|
|
)
|
|
|
|
func TestEvidenceObservesCompletePlaceholderAndPiRepairIdentity(t *testing.T) {
|
|
fixture := newRuntimeFixture(t)
|
|
reviewRelative := "agent-task/m-m1/1_first/CODE_REVIEW-test.md"
|
|
reviewPath := filepath.Join(fixture.projectA, filepath.FromSlash(reviewRelative))
|
|
project := agenttask.ProjectRecord{
|
|
ProjectID: "project-a", WorkspaceID: WorkspaceIdentity(fixture.projectA),
|
|
}
|
|
target := agenttask.ExecutionTarget{ProviderID: "pi"}
|
|
session := agenttask.LocatorRecord{
|
|
Kind: agenttask.LocatorSession, Opaque: "native-session",
|
|
Revision: "session-r1", ProjectID: project.ProjectID,
|
|
WorkspaceID: project.WorkspaceID, WorkUnitID: "1", AttemptID: "attempt-1",
|
|
}
|
|
work := agenttask.WorkRecord{
|
|
Unit: agenttask.WorkUnit{
|
|
ID: "1", Metadata: map[string]string{"review_path": reviewRelative},
|
|
},
|
|
AttemptID: "attempt-1",
|
|
DispatchOrdinal: 7,
|
|
Target: &target,
|
|
Locators: map[agenttask.LocatorKind]agenttask.LocatorRecord{agenttask.LocatorSession: session},
|
|
}
|
|
submission := agenttask.Submission{
|
|
ProjectID: project.ProjectID, WorkUnitID: "1", AttemptID: work.AttemptID,
|
|
ArtifactID: artifactIdentity(work), Ready: true,
|
|
}
|
|
evidence := NewEvidence(fixture.snapshot, staticArtifactRoot(fixture.projectA), nil)
|
|
|
|
complete, err := evidence.Observe(context.Background(), agenttask.WorkflowEvidenceRequest{
|
|
Project: project, Work: work, Submission: submission,
|
|
})
|
|
if err != nil || complete.Completeness != agenttask.ArtifactComplete ||
|
|
complete.Identity.ArtifactID != submission.ArtifactID {
|
|
t.Fatalf("complete observation = %#v, err = %v", complete, err)
|
|
}
|
|
|
|
writeTaskFile(t, reviewPath, `# Code Review Reference
|
|
|
|
## Verification Results
|
|
|
|
_Paste actual stdout/stderr._
|
|
`)
|
|
placeholder, err := evidence.Observe(context.Background(), agenttask.WorkflowEvidenceRequest{
|
|
Project: project, Work: work, Submission: submission,
|
|
})
|
|
if err != nil || placeholder.Completeness != agenttask.ArtifactPlaceholder ||
|
|
placeholder.RepairIntent == nil ||
|
|
placeholder.RepairIntent.NativeLocator.Opaque != session.Opaque ||
|
|
placeholder.RepairIntent.DispatchOrdinal != 7 {
|
|
t.Fatalf("placeholder observation = %#v, err = %v", placeholder, err)
|
|
}
|
|
if err := evidence.Repair(context.Background(), agenttask.WorkflowEvidenceRepairRequest{
|
|
Project: project, Work: work, Submission: submission,
|
|
Intent: *placeholder.RepairIntent,
|
|
}); err == nil || !strings.Contains(err.Error(), "not configured") {
|
|
t.Fatalf("Repair error = %v, want configured-executor denial", err)
|
|
}
|
|
}
|
|
|
|
func TestEvidenceRejectsEscapingAndMissingArtifactLocators(t *testing.T) {
|
|
fixture := newRuntimeFixture(t)
|
|
evidence := NewEvidence(fixture.snapshot, staticArtifactRoot(fixture.projectA), nil)
|
|
project := agenttask.ProjectRecord{
|
|
ProjectID: "project-a", WorkspaceID: WorkspaceIdentity(fixture.projectA),
|
|
}
|
|
for _, reviewPath := range []string{"", "../outside.md"} {
|
|
work := agenttask.WorkRecord{
|
|
Unit: agenttask.WorkUnit{
|
|
ID: "1", Metadata: map[string]string{"review_path": reviewPath},
|
|
},
|
|
AttemptID: "attempt-1",
|
|
}
|
|
_, err := evidence.Observe(context.Background(), agenttask.WorkflowEvidenceRequest{
|
|
Project: project,
|
|
Work: work,
|
|
Submission: agenttask.Submission{
|
|
ProjectID: project.ProjectID, WorkUnitID: "1",
|
|
AttemptID: work.AttemptID, ArtifactID: artifactIdentity(work),
|
|
},
|
|
})
|
|
if err == nil {
|
|
t.Fatalf("review path %q was accepted", reviewPath)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPiEvidenceRepairResumesExactNativeSession(t *testing.T) {
|
|
providerCatalog := piTestCatalog()
|
|
fixture := newRetainedExecutionFixture(t, providerCatalog, "pi-headless")
|
|
resolved, ok := providerCatalog.ResolveProfile("pi-headless")
|
|
if !ok {
|
|
t.Fatal("Pi test profile is missing")
|
|
}
|
|
reference, err := newNativeSessionReference(
|
|
resolved,
|
|
*fixture.work.Target,
|
|
fixture.confinement.Binding(),
|
|
fixture.project,
|
|
fixture.work,
|
|
"worker",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("newNativeSessionReference: %v", err)
|
|
}
|
|
if err := os.MkdirAll(reference.SessionDirectory, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
nativeSession := filepath.Join(
|
|
reference.SessionDirectory,
|
|
"session_"+reference.SessionID+".jsonl",
|
|
)
|
|
writeTaskFile(t, nativeSession, `{"type":"session"}`+"\n")
|
|
opaque, err := json.Marshal(reference)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
locator := agenttask.LocatorRecord{
|
|
Kind: agenttask.LocatorSession,
|
|
Opaque: string(opaque),
|
|
Revision: digestStrings("native-session-locator", string(opaque)),
|
|
ProjectID: fixture.project.ProjectID,
|
|
WorkspaceID: fixture.project.WorkspaceID,
|
|
WorkUnitID: fixture.work.Unit.ID,
|
|
AttemptID: fixture.work.AttemptID,
|
|
}
|
|
fixture.work.Locators = map[agenttask.LocatorKind]agenttask.LocatorRecord{
|
|
agenttask.LocatorSession: locator,
|
|
}
|
|
reviewPath := filepath.Join(
|
|
fixture.descriptor.WorkingDir,
|
|
filepath.FromSlash(fixture.work.Unit.Metadata["review_path"]),
|
|
)
|
|
writeTaskFile(t, reviewPath, `# Code Review Reference
|
|
|
|
## Verification Results
|
|
|
|
_Paste actual stdout/stderr._
|
|
`)
|
|
fixture.confinement.launch = func(
|
|
ctx context.Context,
|
|
command agenttask.ConfinementCommand,
|
|
) (agenttask.StartedConfinement, error) {
|
|
return startFixtureCommand(
|
|
ctx,
|
|
"printf '# Code Review Reference\\n\\n## Implementation Notes\\n\\nEvidence complete.\\n' > \"$1\"",
|
|
reviewPath,
|
|
)
|
|
}
|
|
repairer, err := NewPiEvidenceRepairer(providerCatalog, fixture.backend)
|
|
if err != nil {
|
|
t.Fatalf("NewPiEvidenceRepairer: %v", err)
|
|
}
|
|
evidence := NewEvidence(
|
|
evidenceSnapshot(t, fixture.record.CanonicalRoot),
|
|
staticArtifactRoot(fixture.descriptor.WorkingDir),
|
|
repairer,
|
|
)
|
|
submission := agenttask.Submission{
|
|
ProjectID: fixture.project.ProjectID,
|
|
WorkUnitID: fixture.work.Unit.ID,
|
|
AttemptID: fixture.work.AttemptID,
|
|
ArtifactID: artifactIdentity(fixture.work),
|
|
Ready: true,
|
|
}
|
|
observed, err := evidence.Observe(
|
|
context.Background(),
|
|
agenttask.WorkflowEvidenceRequest{
|
|
Project: fixture.project, Work: fixture.work, Submission: submission,
|
|
},
|
|
)
|
|
if err != nil || observed.RepairIntent == nil {
|
|
t.Fatalf("placeholder observation = %#v, err = %v", observed, err)
|
|
}
|
|
if err := evidence.Repair(
|
|
context.Background(),
|
|
agentruntimeRepairRequest(
|
|
fixture.project,
|
|
fixture.work,
|
|
submission,
|
|
*observed.RepairIntent,
|
|
),
|
|
); err != nil {
|
|
t.Fatalf("Repair: %v", err)
|
|
}
|
|
if fixture.confinement.starts != 1 ||
|
|
!containsArgumentPair(fixture.confinement.command.Args, "--session", nativeSession) ||
|
|
!containsArgumentPair(
|
|
fixture.confinement.command.Args,
|
|
"--session-dir",
|
|
reference.SessionDirectory,
|
|
) {
|
|
t.Fatalf(
|
|
"Pi repair command/starts = %#v/%d",
|
|
fixture.confinement.command.Args,
|
|
fixture.confinement.starts,
|
|
)
|
|
}
|
|
fresh, err := evidence.Observe(
|
|
context.Background(),
|
|
agentruntimeEvidenceRequest(fixture.project, fixture.work, submission),
|
|
)
|
|
if err != nil || fresh.Completeness != agenttask.ArtifactComplete {
|
|
t.Fatalf("fresh observation = %#v, err = %v", fresh, err)
|
|
}
|
|
|
|
staleIntent := *observed.RepairIntent
|
|
staleIntent.NativeLocator.Revision = "stale"
|
|
if err := evidence.Repair(
|
|
context.Background(),
|
|
agentruntimeRepairRequest(
|
|
fixture.project,
|
|
fixture.work,
|
|
submission,
|
|
staleIntent,
|
|
),
|
|
); err == nil {
|
|
t.Fatal("stale native locator was accepted")
|
|
}
|
|
wrongProviderWork := fixture.work
|
|
wrongTarget := *fixture.work.Target
|
|
wrongTarget.ProviderID = "not-pi"
|
|
wrongProviderWork.Target = &wrongTarget
|
|
if err := evidence.Repair(
|
|
context.Background(),
|
|
agentruntimeRepairRequest(
|
|
fixture.project,
|
|
wrongProviderWork,
|
|
submission,
|
|
*observed.RepairIntent,
|
|
),
|
|
); err == nil {
|
|
t.Fatal("non-Pi repair was accepted")
|
|
}
|
|
if fixture.confinement.starts != 1 {
|
|
t.Fatalf("denied repairs launched a process; starts = %d", fixture.confinement.starts)
|
|
}
|
|
}
|
|
|
|
func agentruntimeRepairRequest(
|
|
project agenttask.ProjectRecord,
|
|
work agenttask.WorkRecord,
|
|
submission agenttask.Submission,
|
|
intent agenttask.EvidenceRepairIntent,
|
|
) agenttask.WorkflowEvidenceRepairRequest {
|
|
return agenttask.WorkflowEvidenceRepairRequest{
|
|
Project: project, Work: work, Submission: submission, Intent: intent,
|
|
}
|
|
}
|
|
|
|
func agentruntimeEvidenceRequest(
|
|
project agenttask.ProjectRecord,
|
|
work agenttask.WorkRecord,
|
|
submission agenttask.Submission,
|
|
) agenttask.WorkflowEvidenceRequest {
|
|
return agenttask.WorkflowEvidenceRequest{
|
|
Project: project, Work: work, Submission: submission,
|
|
}
|
|
}
|
|
|
|
func containsArgumentPair(arguments []string, key, value string) bool {
|
|
for index := 0; index+1 < len(arguments); index++ {
|
|
if arguments[index] == key && arguments[index+1] == value {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func piTestCatalog() agentconfig.Catalog {
|
|
capabilities := []string{
|
|
"approval_bypass", "run", "unattended", "writable_root_confinement",
|
|
}
|
|
return agentconfig.Catalog{
|
|
Version: agentconfig.SchemaVersion,
|
|
Providers: []agentconfig.Provider{{
|
|
ID: "pi", Command: "pi-provider-must-not-run-in-tests",
|
|
VersionProbe: agentconfig.CommandProbe{Args: []string{"--version"}},
|
|
Authentication: agentconfig.AuthenticationProbe{Args: []string{"auth", "status"}},
|
|
Capabilities: capabilities,
|
|
}},
|
|
Models: []agentconfig.Model{{
|
|
ID: "pi-model", Provider: "pi", Target: "pi-model",
|
|
}},
|
|
Profiles: []agentconfig.Profile{{
|
|
ID: "pi-headless", Provider: "pi", Model: "pi-model",
|
|
Args: []string{"--mode", "json", "--model", "{{model}}"},
|
|
ResumeArgs: []string{"--mode", "json", "--model", "{{model}}"},
|
|
Capabilities: capabilities,
|
|
}},
|
|
}
|
|
}
|
|
|
|
func evidenceSnapshot(t *testing.T, workspace string) agentconfig.RuntimeSnapshot {
|
|
t.Helper()
|
|
root := canonicalTempDir(t)
|
|
global := `version: "1"
|
|
defaults:
|
|
default_profile: p1
|
|
selection:
|
|
timezone: UTC
|
|
default:
|
|
provider: pi
|
|
model: pi-model
|
|
profile: p1
|
|
isolation:
|
|
default_mode: overlay
|
|
`
|
|
local := fmt.Sprintf(`version: "1"
|
|
device:
|
|
state_root: %s
|
|
overlay_root: %s
|
|
log_root: %s
|
|
projects:
|
|
project-a:
|
|
workspace: %s
|
|
enabled: true
|
|
`, filepath.Join(root, "state"), filepath.Join(root, "overlays"), filepath.Join(root, "logs"), workspace)
|
|
snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(global), []byte(local))
|
|
if err != nil {
|
|
t.Fatalf("LoadRuntimeConfigBytes: %v", err)
|
|
}
|
|
return snapshot
|
|
}
|