461 lines
14 KiB
Go
461 lines
14 KiB
Go
package taskloop
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"iop/packages/go/agentconfig"
|
|
"iop/packages/go/agentguard"
|
|
"iop/packages/go/agenttask"
|
|
"iop/packages/go/agentworkspace"
|
|
)
|
|
|
|
func TestReviewerRejectsPrematureAndIdentityMismatchedReview(t *testing.T) {
|
|
fixture := newRuntimeFixture(t)
|
|
reviewer, err := NewReviewer(
|
|
fixture.snapshot,
|
|
&agentworkspace.Backend{},
|
|
nil,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewReviewer: %v", err)
|
|
}
|
|
project := agenttask.ProjectRecord{
|
|
ProjectID: "project-a", WorkspaceID: WorkspaceIdentity(fixture.projectA),
|
|
}
|
|
submission := agenttask.Submission{
|
|
ProjectID: project.ProjectID, WorkUnitID: "1",
|
|
AttemptID: "attempt-1", ArtifactID: "artifact-1", Ready: true,
|
|
}
|
|
work := agenttask.WorkRecord{
|
|
Unit: agenttask.WorkUnit{
|
|
ID: "1",
|
|
Metadata: map[string]string{
|
|
"plan_path": "agent-task/m-m1/1_first/PLAN-test.md",
|
|
"review_path": "agent-task/m-m1/1_first/CODE_REVIEW-test.md",
|
|
},
|
|
},
|
|
AttemptID: "attempt-1",
|
|
}
|
|
if _, err := reviewer.Review(context.Background(), agenttask.ReviewRequest{
|
|
Project: project, Work: work, Submission: submission,
|
|
}); err == nil || !strings.Contains(err.Error(), "identity mismatch") {
|
|
t.Fatalf("missing durable submission error = %v", err)
|
|
}
|
|
|
|
mismatched := submission
|
|
mismatched.ArtifactID = "artifact-2"
|
|
work.Submission = &submission
|
|
if _, err := reviewer.Review(context.Background(), agenttask.ReviewRequest{
|
|
Project: project, Work: work, Submission: mismatched,
|
|
}); err == nil || !strings.Contains(err.Error(), "identity mismatch") {
|
|
t.Fatalf("mismatched submission error = %v", err)
|
|
}
|
|
|
|
if _, err := reviewer.Review(context.Background(), agenttask.ReviewRequest{
|
|
Project: project, Work: work, Submission: submission,
|
|
}); err == nil || !strings.Contains(err.Error(), "no retained isolation") {
|
|
t.Fatalf("premature review error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestReadReviewVerdictAcceptsCanonicalOverallVerdict(t *testing.T) {
|
|
fixture := newRuntimeFixture(t)
|
|
path := fixture.projectA + "/agent-task/m-m1/1_first/CODE_REVIEW-test.md"
|
|
writeTaskFile(t, path, `# Code Review Reference
|
|
|
|
## Code Review Result
|
|
|
|
Overall Verdict: PASS
|
|
`)
|
|
verdict, message, err := readReviewVerdict(path)
|
|
if err != nil || verdict != agenttask.ReviewVerdictPass ||
|
|
message != "official review passed" {
|
|
t.Fatalf("readReviewVerdict = %s/%q, err = %v", verdict, message, err)
|
|
}
|
|
}
|
|
|
|
func TestOfficialReviewPromptPreservesRetainedArtifacts(t *testing.T) {
|
|
plan := "agent-task/m-m/1/PLAN-cloud-G10.md"
|
|
review := "agent-task/m-m/1/CODE_REVIEW-cloud-G10.md"
|
|
prompt := officialReviewPrompt(plan, review)
|
|
for _, required := range []string{
|
|
plan,
|
|
review,
|
|
"bounded official-review step",
|
|
"not task finalization",
|
|
"Do not modify review-only checklists",
|
|
"archive, move, rename, or delete either active artifact",
|
|
"Append only one `## Code Review Result` section",
|
|
} {
|
|
if !strings.Contains(prompt, required) {
|
|
t.Fatalf("official review prompt does not contain %q: %q", required, prompt)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReadReviewVerdictRejectsLegacyDuplicateAndBareValues(t *testing.T) {
|
|
fixture := newRuntimeFixture(t)
|
|
path := fixture.projectA + "/agent-task/m-m1/1_first/CODE_REVIEW-test.md"
|
|
for name, result := range map[string]string{
|
|
"legacy": "Verdict: PASS",
|
|
"bare": "PASS",
|
|
"duplicate": "Overall Verdict: PASS\nOverall Verdict: FAIL",
|
|
"invalid": "Overall Verdict: pass",
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
writeTaskFile(t, path, "# Review\n\n## Code Review Result\n\n"+result+"\n")
|
|
if _, _, err := readReviewVerdict(path); err == nil {
|
|
t.Fatalf("result %q was accepted", result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCatalogReviewExecutorRequiresExactRetainedConfinement(t *testing.T) {
|
|
fixture := newRetainedExecutionFixture(t, testCatalog(), "p1")
|
|
reviewPath := filepath.Join(
|
|
fixture.descriptor.WorkingDir,
|
|
filepath.FromSlash(fixture.work.Unit.Metadata["review_path"]),
|
|
)
|
|
writeTaskFile(t, reviewPath, "# Code Review Reference\n")
|
|
fixture.confinement.launch = func(
|
|
ctx context.Context,
|
|
command agenttask.ConfinementCommand,
|
|
) (agenttask.StartedConfinement, error) {
|
|
if command.Name != "provider-must-not-run-in-tests" {
|
|
return nil, errors.New("unexpected provider command")
|
|
}
|
|
return startFixtureCommand(
|
|
ctx,
|
|
"printf '\\n## Code Review Result\\n\\nOverall Verdict: PASS\\n' >> \"$1\"",
|
|
reviewPath,
|
|
)
|
|
}
|
|
executor, err := NewCatalogReviewExecutor(testCatalog(), fixture.backend)
|
|
if err != nil {
|
|
t.Fatalf("NewCatalogReviewExecutor: %v", err)
|
|
}
|
|
request := agenttask.ReviewRequest{
|
|
Project: fixture.project,
|
|
Work: fixture.work,
|
|
Submission: agenttask.Submission{
|
|
ProjectID: fixture.project.ProjectID,
|
|
WorkUnitID: fixture.work.Unit.ID,
|
|
AttemptID: fixture.work.AttemptID,
|
|
ArtifactID: artifactIdentity(fixture.work),
|
|
Ready: true,
|
|
},
|
|
IdempotencyKey: "review-fixture",
|
|
}
|
|
if err := executor.ExecuteReview(
|
|
context.Background(),
|
|
request,
|
|
fixture.descriptor.WorkingDir,
|
|
fixture.work.Unit.Metadata["plan_path"],
|
|
fixture.work.Unit.Metadata["review_path"],
|
|
); err != nil {
|
|
t.Fatalf("ExecuteReview: %v", err)
|
|
}
|
|
if fixture.confinement.starts != 1 {
|
|
t.Fatalf("confined starts = %d, want 1", fixture.confinement.starts)
|
|
}
|
|
if verdict, _, err := readReviewVerdict(reviewPath); err != nil ||
|
|
verdict != agenttask.ReviewVerdictPass {
|
|
t.Fatalf("review result = %s, err = %v", verdict, err)
|
|
}
|
|
|
|
fixture.confinement.binding.ProfileRevision = "stale-profile"
|
|
if err := executor.ExecuteReview(
|
|
context.Background(),
|
|
request,
|
|
fixture.descriptor.WorkingDir,
|
|
fixture.work.Unit.Metadata["plan_path"],
|
|
fixture.work.Unit.Metadata["review_path"],
|
|
); err == nil {
|
|
t.Fatal("stale confinement was accepted")
|
|
}
|
|
if fixture.confinement.starts != 1 {
|
|
t.Fatalf("stale proof launched a process; starts = %d", fixture.confinement.starts)
|
|
}
|
|
}
|
|
|
|
type retainedExecutionFixture struct {
|
|
project agenttask.ProjectRecord
|
|
work agenttask.WorkRecord
|
|
record agentworkspace.OverlayRecord
|
|
descriptor agentguard.IsolationDescriptor
|
|
confinement *recordingConfinement
|
|
backend *fakeRetainedBackend
|
|
}
|
|
|
|
func newRetainedExecutionFixture(
|
|
t *testing.T,
|
|
providerCatalog agentconfig.Catalog,
|
|
profileID string,
|
|
) retainedExecutionFixture {
|
|
t.Helper()
|
|
resolved, ok := providerCatalog.ResolveProfile(profileID)
|
|
if !ok {
|
|
t.Fatalf("profile %q is missing", profileID)
|
|
}
|
|
root := canonicalTempDir(t)
|
|
taskRoot := filepath.Join(root, "retained")
|
|
viewRoot := filepath.Join(taskRoot, "view")
|
|
tempRoot := filepath.Join(taskRoot, "tmp")
|
|
cacheRoot := filepath.Join(taskRoot, "cache")
|
|
snapshotRoot := filepath.Join(root, "snapshot")
|
|
canonicalRoot := filepath.Join(root, "canonical")
|
|
for _, directory := range []string{
|
|
taskRoot, viewRoot, tempRoot, cacheRoot, snapshotRoot, canonicalRoot,
|
|
} {
|
|
if err := mkdirOwned(directory); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
workspaceID := WorkspaceIdentity(canonicalRoot)
|
|
project := agenttask.ProjectRecord{
|
|
ProjectID: "project-a",
|
|
WorkspaceID: workspaceID,
|
|
Intent: &agenttask.StartIntent{
|
|
ProjectID: "project-a",
|
|
WorkspaceID: workspaceID,
|
|
ConfigRevision: "config-r1",
|
|
GrantRevision: "grant-r1",
|
|
},
|
|
}
|
|
target := agenttask.ExecutionTarget{
|
|
ProviderID: resolved.Provider.ID,
|
|
ModelID: resolved.Model.ID,
|
|
ProfileID: resolved.Profile.ID,
|
|
ProfileRevision: guardProfile(resolved).Revision,
|
|
ConfigRevision: "config-r1",
|
|
Capacity: 1,
|
|
}
|
|
work := agenttask.WorkRecord{
|
|
Unit: agenttask.WorkUnit{
|
|
ID: "work-1",
|
|
IsolationMode: agentguard.IsolationModeOverlay,
|
|
Metadata: map[string]string{
|
|
"plan_path": "agent-task/m-m1/1_fixture/PLAN-test.md",
|
|
"review_path": "agent-task/m-m1/1_fixture/CODE_REVIEW-test.md",
|
|
},
|
|
},
|
|
AttemptID: "attempt-1",
|
|
DispatchOrdinal: 1,
|
|
Target: &target,
|
|
}
|
|
record := agentworkspace.OverlayRecord{
|
|
SchemaVersion: 2,
|
|
IsolationID: "overlay:fixture",
|
|
Revision: "overlay-r1",
|
|
IdempotencyKey: "dispatch/fixture/isolation",
|
|
ProjectID: string(project.ProjectID),
|
|
WorkspaceID: string(project.WorkspaceID),
|
|
WorkUnitID: string(work.Unit.ID),
|
|
AttemptID: string(work.AttemptID),
|
|
CanonicalRoot: canonicalRoot,
|
|
ConfigRevision: string(target.ConfigRevision),
|
|
GrantRevision: "grant-r1",
|
|
ProfileRevision: target.ProfileRevision,
|
|
SnapshotRevision: "snapshot-r1",
|
|
ConfinementRevision: "confinement-r1",
|
|
Mode: agentguard.IsolationModeOverlay,
|
|
Locator: agentworkspace.OverlayLocator{
|
|
SnapshotRoot: snapshotRoot,
|
|
ViewRoot: viewRoot,
|
|
TempRoot: tempRoot,
|
|
CacheRoot: cacheRoot,
|
|
OverlayRecord: filepath.Join(taskRoot, "overlay.json"),
|
|
},
|
|
Retention: agentworkspace.RetentionStateActive,
|
|
}
|
|
descriptor := agentguard.IsolationDescriptor{
|
|
ID: record.IsolationID,
|
|
Revision: record.Revision,
|
|
Mode: record.Mode,
|
|
BaseRoot: record.CanonicalRoot,
|
|
TaskRoot: taskRoot,
|
|
WorkingDir: viewRoot,
|
|
WritableRoots: []string{viewRoot, tempRoot, cacheRoot},
|
|
PinnedBaseRevision: record.SnapshotRevision,
|
|
ConfinementRevision: record.ConfinementRevision,
|
|
}
|
|
work.Isolation = &agenttask.IsolationIdentity{
|
|
ID: descriptor.ID,
|
|
Revision: descriptor.Revision,
|
|
Mode: descriptor.Mode,
|
|
PinnedBaseRevision: descriptor.PinnedBaseRevision,
|
|
TaskRoot: descriptor.TaskRoot,
|
|
}
|
|
binding := agenttask.ConfinementBinding{
|
|
Revision: record.ConfinementRevision,
|
|
IsolationID: record.IsolationID,
|
|
IsolationRevision: record.Revision,
|
|
PinnedBaseRevision: record.SnapshotRevision,
|
|
ConfigRevision: record.ConfigRevision,
|
|
GrantRevision: record.GrantRevision,
|
|
ProfileRevision: record.ProfileRevision,
|
|
BaseRoot: record.CanonicalRoot,
|
|
RuntimeRoot: root,
|
|
SnapshotRoot: record.Locator.SnapshotRoot,
|
|
TaskRoot: taskRoot,
|
|
WorkingDir: viewRoot,
|
|
WritableRoots: append([]string(nil), descriptor.WritableRoots...),
|
|
}
|
|
confinement := &recordingConfinement{binding: binding}
|
|
backend := &fakeRetainedBackend{
|
|
record: record,
|
|
prepared: agenttask.PreparedIsolation{
|
|
Descriptor: &descriptor,
|
|
Profile: guardProfile(resolved),
|
|
Confinement: confinement,
|
|
},
|
|
}
|
|
content, err := json.Marshal(record)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeTaskFile(t, record.Locator.OverlayRecord, string(content))
|
|
for _, relative := range []string{
|
|
work.Unit.Metadata["plan_path"],
|
|
work.Unit.Metadata["review_path"],
|
|
} {
|
|
path := filepath.Join(viewRoot, filepath.FromSlash(relative))
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeTaskFile(t, path, "# Fixture\n")
|
|
}
|
|
return retainedExecutionFixture{
|
|
project: project, work: work, record: record, descriptor: descriptor,
|
|
confinement: confinement, backend: backend,
|
|
}
|
|
}
|
|
|
|
func mkdirOwned(path string) error {
|
|
return os.MkdirAll(path, 0o700)
|
|
}
|
|
|
|
type fakeRetainedBackend struct {
|
|
record agentworkspace.OverlayRecord
|
|
prepared agenttask.PreparedIsolation
|
|
prepareErr error
|
|
prepares int
|
|
}
|
|
|
|
func (backend *fakeRetainedBackend) LoadRecord(
|
|
descriptor agentguard.IsolationDescriptor,
|
|
) (agentworkspace.OverlayRecord, error) {
|
|
if backend.prepared.Descriptor == nil ||
|
|
!reflect.DeepEqual(descriptor, *backend.prepared.Descriptor) {
|
|
return agentworkspace.OverlayRecord{}, errors.New("fixture descriptor mismatch")
|
|
}
|
|
return backend.record, nil
|
|
}
|
|
|
|
func (backend *fakeRetainedBackend) Prepare(
|
|
_ context.Context,
|
|
request agenttask.IsolationRequest,
|
|
) (agenttask.PreparedIsolation, error) {
|
|
backend.prepares++
|
|
if backend.prepareErr != nil {
|
|
return agenttask.PreparedIsolation{}, backend.prepareErr
|
|
}
|
|
if request.IdempotencyKey != backend.record.IdempotencyKey ||
|
|
string(request.Project.ProjectID) != backend.record.ProjectID ||
|
|
string(request.Work.Unit.ID) != backend.record.WorkUnitID ||
|
|
request.Target.ProfileRevision != backend.record.ProfileRevision {
|
|
return agenttask.PreparedIsolation{}, errors.New("fixture prepare identity mismatch")
|
|
}
|
|
return backend.prepared, nil
|
|
}
|
|
|
|
type recordingConfinement struct {
|
|
binding agenttask.ConfinementBinding
|
|
starts int
|
|
command agenttask.ConfinementCommand
|
|
launch func(
|
|
context.Context,
|
|
agenttask.ConfinementCommand,
|
|
) (agenttask.StartedConfinement, error)
|
|
}
|
|
|
|
func (confinement *recordingConfinement) Revision() string {
|
|
return confinement.binding.Revision
|
|
}
|
|
|
|
func (confinement *recordingConfinement) Binding() agenttask.ConfinementBinding {
|
|
binding := confinement.binding
|
|
binding.WritableRoots = append([]string(nil), binding.WritableRoots...)
|
|
return binding
|
|
}
|
|
|
|
func (confinement *recordingConfinement) Validate(
|
|
expected agenttask.ConfinementBinding,
|
|
) error {
|
|
if !reflect.DeepEqual(confinement.binding, expected) {
|
|
return errors.New("fixture confinement identity mismatch")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (confinement *recordingConfinement) Start(
|
|
ctx context.Context,
|
|
command agenttask.ConfinementCommand,
|
|
) (agenttask.StartedConfinement, error) {
|
|
confinement.starts++
|
|
confinement.command = command
|
|
if confinement.launch == nil {
|
|
return nil, errors.New("fixture launch is not configured")
|
|
}
|
|
return confinement.launch(ctx, command)
|
|
}
|
|
|
|
func startFixtureCommand(
|
|
ctx context.Context,
|
|
script string,
|
|
arguments ...string,
|
|
) (agenttask.StartedConfinement, error) {
|
|
args := append([]string{"-c", script, "fixture"}, arguments...)
|
|
child := exec.CommandContext(ctx, "/bin/sh", args...)
|
|
stdin, err := child.StdinPipe()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stdout, err := child.StdoutPipe()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stderr, err := child.StderrPipe()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := child.Start(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &fixtureStarted{
|
|
child: child, stdin: stdin, stdout: stdout, stderr: stderr,
|
|
}, nil
|
|
}
|
|
|
|
type fixtureStarted struct {
|
|
child *exec.Cmd
|
|
stdin io.WriteCloser
|
|
stdout io.ReadCloser
|
|
stderr io.ReadCloser
|
|
}
|
|
|
|
func (started *fixtureStarted) Child() *exec.Cmd { return started.child }
|
|
func (started *fixtureStarted) Stdin() io.WriteCloser { return started.stdin }
|
|
func (started *fixtureStarted) Stdout() io.ReadCloser { return started.stdout }
|
|
func (started *fixtureStarted) Stderr() io.ReadCloser { return started.stderr }
|
|
func (started *fixtureStarted) Abort() error { return started.child.Process.Kill() }
|