toki
dc9a9a8c59
승인된 execution preset을 Edge 조정 경계와 Node workspace/tool 실행 경계로 연결해 단일 요청 수명주기와 관측 계약을 일관되게 처리한다.
591 lines
30 KiB
Go
591 lines
30 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net"
|
|
"slices"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
toki "git.toki-labs.com/toki/proto-socket/go"
|
|
"git.toki-labs.com/toki/proto-socket/go/packets"
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
edgeevents "iop/apps/edge/internal/events"
|
|
edgenode "iop/apps/edge/internal/node"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
func TestWorkspaceWire(t *testing.T) {
|
|
edgeClient, nodeClient := workspaceWirePipe(t)
|
|
registry := edgenode.NewRegistry()
|
|
entry := &edgenode.NodeEntry{NodeID: "node-1", Client: edgeClient}
|
|
registry.Register(entry)
|
|
svc := New(registry, edgeevents.NewBus())
|
|
binding := workspaceWireBinding("workspace-1", entry.NodeID, entry.ConnectionGeneration, 1000)
|
|
binding.Limits.MaxWriteBytes = 999
|
|
openSeen := make(chan *iop.WorkspaceOpenRequest, 1)
|
|
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&nodeClient.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
openSeen <- proto.Clone(req).(*iop.WorkspaceOpenRequest)
|
|
return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
|
|
})
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&nodeClient.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
|
|
return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
|
|
})
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCancelRequest, *iop.WorkspaceCancelResponse](&nodeClient.Communicator, func(req *iop.WorkspaceCancelRequest) (*iop.WorkspaceCancelResponse, error) {
|
|
return &iop.WorkspaceCancelResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_CANCELLED, Error: "workspace command cancelled"}, nil
|
|
})
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&nodeClient.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
|
|
return &iop.WorkspaceCleanupResponse{RequestId: req.GetRequestId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS, CleanedArtifacts: 1}, nil
|
|
})
|
|
|
|
caller := &iop.WorkspaceOpenRequest{
|
|
RequestId: "request-1", WorkspaceRef: binding.Ref,
|
|
Operations: []iop.WorkspaceOperation{iop.WorkspaceOperation_WORKSPACE_OPERATION_DELETE},
|
|
CommandIds: []string{"caller-command"}, MaxReadBytes: 9999, MaxWriteBytes: 9999,
|
|
MaxOutputBytes: 9999, MaxCommandTimeoutMs: 9999,
|
|
}
|
|
if response, err := svc.workspaceOpen(context.Background(), binding, caller); err != nil || response.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS {
|
|
t.Fatalf("open = %+v, %v", response, err)
|
|
}
|
|
gotOpen := <-openSeen
|
|
if !proto.Equal(gotOpen, &iop.WorkspaceOpenRequest{
|
|
RequestId: "request-1", WorkspaceRef: "workspace-1",
|
|
Operations: []iop.WorkspaceOperation{iop.WorkspaceOperation_WORKSPACE_OPERATION_COMMAND, iop.WorkspaceOperation_WORKSPACE_OPERATION_READ},
|
|
CommandIds: []string{"test"}, MaxReadBytes: 64, MaxOutputBytes: 64, MaxCommandTimeoutMs: 1000,
|
|
}) {
|
|
t.Fatalf("open authority = %+v", gotOpen)
|
|
}
|
|
caller.Operations[0] = iop.WorkspaceOperation_WORKSPACE_OPERATION_WRITE
|
|
caller.CommandIds[0] = "mutated"
|
|
if !slices.Equal(gotOpen.GetOperations(), []iop.WorkspaceOperation{iop.WorkspaceOperation_WORKSPACE_OPERATION_COMMAND, iop.WorkspaceOperation_WORKSPACE_OPERATION_READ}) {
|
|
t.Fatalf("captured authority changed after caller mutation: %+v", gotOpen)
|
|
}
|
|
if response, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1", Operation: iop.WorkspaceOperation_WORKSPACE_OPERATION_READ}); err != nil || response.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS {
|
|
t.Fatalf("tool = %+v, %v", response, err)
|
|
}
|
|
if response, err := svc.workspaceCancel(context.Background(), binding, &iop.WorkspaceCancelRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"}); err != nil || response.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED {
|
|
t.Fatalf("cancel = %+v, %v", response, err)
|
|
}
|
|
if response, err := svc.workspaceCleanup(context.Background(), binding, &iop.WorkspaceCleanupRequest{RequestId: "request-1"}); err != nil || response.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS || response.GetCleanedArtifacts() != 1 {
|
|
t.Fatalf("cleanup = %+v, %v", response, err)
|
|
}
|
|
}
|
|
|
|
func TestWorkspaceWireStaleGenerationNeverReselects(t *testing.T) {
|
|
oldEdge, oldNode := workspaceWirePipe(t)
|
|
newEdge, newNode := workspaceWirePipe(t)
|
|
registry := edgenode.NewRegistry()
|
|
oldEntry := &edgenode.NodeEntry{NodeID: "node-1", Client: oldEdge}
|
|
registry.Register(oldEntry)
|
|
binding := workspaceWireBinding("workspace-1", oldEntry.NodeID, oldEntry.ConnectionGeneration, 1000)
|
|
registry.Register(&edgenode.NodeEntry{NodeID: "node-1", Client: newEdge})
|
|
var reachedNew atomic.Bool
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&newNode.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
reachedNew.Store(true)
|
|
return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
|
|
})
|
|
_ = oldNode
|
|
svc := New(registry, edgeevents.NewBus())
|
|
if _, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: binding.Ref}); err != errWorkspaceWireStale {
|
|
t.Fatalf("stale dispatch error = %v, want %v", err, errWorkspaceWireStale)
|
|
}
|
|
if reachedNew.Load() {
|
|
t.Fatal("stale binding must not reselect the reconnect client")
|
|
}
|
|
}
|
|
|
|
func workspaceWirePipe(t *testing.T) (*toki.TcpClient, *toki.TcpClient) {
|
|
t.Helper()
|
|
edgeConn, nodeConn := net.Pipe()
|
|
edge := toki.NewTcpClient(edgeConn, 0, 0, workspaceWireResponseParserMap())
|
|
node := toki.NewTcpClient(nodeConn, 0, 0, workspaceWireRequestParserMap())
|
|
t.Cleanup(func() { _ = edge.Close(); _ = node.Close() })
|
|
return edge, node
|
|
}
|
|
|
|
func workspaceWireRequestParserMap() toki.ParserMap {
|
|
return toki.ParserMap{
|
|
toki.TypeNameOf(&iop.WorkspaceOpenRequest{}): parseWorkspaceMessage[*iop.WorkspaceOpenRequest],
|
|
toki.TypeNameOf(&iop.WorkspaceToolRequest{}): parseWorkspaceMessage[*iop.WorkspaceToolRequest],
|
|
toki.TypeNameOf(&iop.WorkspaceCancelRequest{}): parseWorkspaceMessage[*iop.WorkspaceCancelRequest],
|
|
toki.TypeNameOf(&iop.WorkspaceCleanupRequest{}): parseWorkspaceMessage[*iop.WorkspaceCleanupRequest],
|
|
}
|
|
}
|
|
|
|
func workspaceWireResponseParserMap() toki.ParserMap {
|
|
return toki.ParserMap{
|
|
toki.TypeNameOf(&iop.WorkspaceOpenResponse{}): parseWorkspaceMessage[*iop.WorkspaceOpenResponse],
|
|
toki.TypeNameOf(&iop.WorkspaceToolResponse{}): parseWorkspaceMessage[*iop.WorkspaceToolResponse],
|
|
toki.TypeNameOf(&iop.WorkspaceCancelResponse{}): parseWorkspaceMessage[*iop.WorkspaceCancelResponse],
|
|
toki.TypeNameOf(&iop.WorkspaceCleanupResponse{}): parseWorkspaceMessage[*iop.WorkspaceCleanupResponse],
|
|
}
|
|
}
|
|
|
|
func parseWorkspaceMessage[T proto.Message](payload []byte) (proto.Message, error) {
|
|
var message T
|
|
message = newWorkspaceMessage[T]()
|
|
return message, proto.Unmarshal(payload, message)
|
|
}
|
|
|
|
func newWorkspaceMessage[T proto.Message]() T {
|
|
var zero T
|
|
switch any(zero).(type) {
|
|
case *iop.WorkspaceOpenRequest:
|
|
return any(&iop.WorkspaceOpenRequest{}).(T)
|
|
case *iop.WorkspaceToolRequest:
|
|
return any(&iop.WorkspaceToolRequest{}).(T)
|
|
case *iop.WorkspaceCancelRequest:
|
|
return any(&iop.WorkspaceCancelRequest{}).(T)
|
|
case *iop.WorkspaceCleanupRequest:
|
|
return any(&iop.WorkspaceCleanupRequest{}).(T)
|
|
case *iop.WorkspaceOpenResponse:
|
|
return any(&iop.WorkspaceOpenResponse{}).(T)
|
|
case *iop.WorkspaceToolResponse:
|
|
return any(&iop.WorkspaceToolResponse{}).(T)
|
|
case *iop.WorkspaceCancelResponse:
|
|
return any(&iop.WorkspaceCancelResponse{}).(T)
|
|
case *iop.WorkspaceCleanupResponse:
|
|
return any(&iop.WorkspaceCleanupResponse{}).(T)
|
|
default:
|
|
panic("unsupported workspace test message")
|
|
}
|
|
}
|
|
|
|
// newWorkspaceWireFixture wires an Edge Service to a single admitted Node over a
|
|
// net.Pipe and returns the Node communicator so a test can install responders.
|
|
func newWorkspaceWireFixture(t *testing.T) (*Service, *toki.TcpClient, *SingleRequestWorkspaceBinding) {
|
|
t.Helper()
|
|
edgeClient, nodeClient := workspaceWirePipe(t)
|
|
registry := edgenode.NewRegistry()
|
|
entry := &edgenode.NodeEntry{NodeID: "node-1", Client: edgeClient}
|
|
registry.Register(entry)
|
|
svc := New(registry, edgeevents.NewBus())
|
|
binding := workspaceWireBinding("workspace-1", entry.NodeID, entry.ConnectionGeneration, 2000)
|
|
return svc, nodeClient, binding
|
|
}
|
|
|
|
func workspaceWireBinding(ref, nodeID string, generation uint64, timeoutMS int) *SingleRequestWorkspaceBinding {
|
|
return &SingleRequestWorkspaceBinding{
|
|
Ref: ref, NodeID: nodeID, ConnectionGeneration: generation,
|
|
OperationIDs: []string{"command", "read"}, CommandIDs: []string{"test"},
|
|
Limits: SingleRequestWorkspaceLimits{MaxReadBytes: 64, MaxOutputBytes: 64, MaxCommandTimeoutMS: timeoutMS},
|
|
}
|
|
}
|
|
|
|
// serveWorkspaceConcurrent installs a Node-side workspace responder that runs off
|
|
// the communicator's single receive coordinator, mirroring the Session's
|
|
// concurrent dispatch. Without this, a blocked tool responder would stall the
|
|
// coordinator and no queued cancel could be observed while the tool is in flight.
|
|
func serveWorkspaceConcurrent[Req proto.Message, Res proto.Message](c *toki.Communicator, seq *atomic.Int32, respond func(Req) Res) {
|
|
c.AddRequestListener(toki.TypeNameOf(newWorkspaceMessage[Req]()), func(m proto.Message, requestNonce int32) {
|
|
req, ok := m.(Req)
|
|
if !ok {
|
|
return
|
|
}
|
|
go func() {
|
|
res := respond(req)
|
|
data, err := proto.Marshal(res)
|
|
if err != nil {
|
|
return
|
|
}
|
|
_ = c.QueuePacket(&packets.PacketBase{TypeName: toki.TypeNameOf(res), Nonce: seq.Add(1), ResponseNonce: requestNonce, Data: data})
|
|
}()
|
|
})
|
|
}
|
|
|
|
// TestWorkspaceWireCancelReachesBlockedTool proves Required R1: a caller context
|
|
// cancelled while the Node tool handler is genuinely in flight delivers exactly
|
|
// one typed cancel, carrying the immutable identities, to the admitted Node
|
|
// before the tool handler is released — without the cancel waiting behind the
|
|
// registry dispatch-owner mutex held by the in-flight tool request.
|
|
func TestWorkspaceWireCancelReachesBlockedTool(t *testing.T) {
|
|
svc, nodeClient, binding := newWorkspaceWireFixture(t)
|
|
var seq atomic.Int32
|
|
var cancelCount atomic.Int32
|
|
toolEntered := make(chan struct{})
|
|
cancelReached := make(chan *iop.WorkspaceCancelRequest, 1)
|
|
release := make(chan struct{})
|
|
serveWorkspaceConcurrent(&nodeClient.Communicator, &seq, func(req *iop.WorkspaceToolRequest) *iop.WorkspaceToolResponse {
|
|
close(toolEntered)
|
|
<-release
|
|
return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}
|
|
})
|
|
serveWorkspaceConcurrent(&nodeClient.Communicator, &seq, func(req *iop.WorkspaceCancelRequest) *iop.WorkspaceCancelResponse {
|
|
cancelCount.Add(1)
|
|
cancelReached <- req
|
|
return &iop.WorkspaceCancelResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_CANCELLED, Error: "workspace command cancelled"}
|
|
})
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
toolErr := make(chan error, 1)
|
|
go func() {
|
|
_, err := svc.workspaceTool(ctx, binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
toolErr <- err
|
|
}()
|
|
|
|
select {
|
|
case <-toolEntered:
|
|
case <-time.After(2 * time.Second):
|
|
close(release)
|
|
t.Fatal("node tool handler was never entered")
|
|
}
|
|
cancel()
|
|
|
|
select {
|
|
case got := <-cancelReached:
|
|
if got.GetRequestId() != "request-1" || got.GetStageId() != "work" || got.GetToolCallId() != "tool-1" {
|
|
close(release)
|
|
t.Fatalf("cancel identity mismatch: %+v", got)
|
|
}
|
|
case <-time.After(time.Second):
|
|
close(release)
|
|
t.Fatal("cancel did not reach the blocked tool before release")
|
|
}
|
|
close(release)
|
|
|
|
if err := <-toolErr; err == nil {
|
|
t.Fatal("cancelled workspace tool must return an error")
|
|
}
|
|
// The cancel handler is exercised at most once; give a late duplicate time to
|
|
// surface before asserting exactly-once delivery.
|
|
time.Sleep(50 * time.Millisecond)
|
|
if got := cancelCount.Load(); got != 1 {
|
|
t.Fatalf("cancel sent %d times, want exactly 1", got)
|
|
}
|
|
}
|
|
|
|
// TestWorkspaceWireRejectsBindingMismatchBeforeSend proves Required R2: an open
|
|
// request whose workspace reference is not the admitted reference fails closed
|
|
// before any transport dispatch and never reaches the Node.
|
|
func TestWorkspaceWireRejectsBindingMismatchBeforeSend(t *testing.T) {
|
|
edgeClient, nodeClient := workspaceWirePipe(t)
|
|
registry := edgenode.NewRegistry()
|
|
entry := &edgenode.NodeEntry{NodeID: "node-1", Client: edgeClient}
|
|
registry.Register(entry)
|
|
svc := New(registry, edgeevents.NewBus())
|
|
binding := workspaceWireBinding("approved", entry.NodeID, entry.ConnectionGeneration, 1000)
|
|
var reached atomic.Bool
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&nodeClient.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
reached.Store(true)
|
|
return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
|
|
})
|
|
if _, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: "not-approved"}); !errors.Is(err, errWorkspaceWireReference) {
|
|
t.Fatalf("mismatch error = %v, want %v", err, errWorkspaceWireReference)
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
if reached.Load() {
|
|
t.Fatal("mismatched workspace reference must not reach the node")
|
|
}
|
|
}
|
|
|
|
// TestWorkspaceWireRejectsInvalidResponse proves Required R3: every response
|
|
// family rejects a disallowed terminal status or a mismatched identity echo with
|
|
// a stable internal error, and never leaks the raw Node Error string.
|
|
func TestWorkspaceWireRejectsInvalidResponse(t *testing.T) {
|
|
const rawSentinel = "RAW-NODE-ERROR-DO-NOT-LEAK-4711"
|
|
cases := []struct {
|
|
name string
|
|
run func(t *testing.T) error
|
|
}{
|
|
{"open status failure", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_INTERNAL, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: binding.Ref})
|
|
return err
|
|
}},
|
|
{"open identity mismatch", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
return &iop.WorkspaceOpenResponse{RequestId: "other-request", WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: binding.Ref})
|
|
return err
|
|
}},
|
|
{"tool status failure", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
|
|
return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_TIMEOUT, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_TIMEOUT, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
return err
|
|
}},
|
|
{"tool identity mismatch", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
|
|
return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: "other-tool", Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
return err
|
|
}},
|
|
{"cancel wrong terminal", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCancelRequest, *iop.WorkspaceCancelResponse](&node.Communicator, func(req *iop.WorkspaceCancelRequest) (*iop.WorkspaceCancelResponse, error) {
|
|
return &iop.WorkspaceCancelResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceCancel(context.Background(), binding, &iop.WorkspaceCancelRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
return err
|
|
}},
|
|
{"cleanup status failure", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
|
|
return &iop.WorkspaceCleanupResponse{RequestId: req.GetRequestId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_UNSUPPORTED, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_NOT_READY, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceCleanup(context.Background(), binding, &iop.WorkspaceCleanupRequest{RequestId: "request-1"})
|
|
return err
|
|
}},
|
|
{"cleanup identity mismatch", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
|
|
return &iop.WorkspaceCleanupResponse{RequestId: "other-request", Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS, Error: rawSentinel}, nil
|
|
})
|
|
_, err := svc.workspaceCleanup(context.Background(), binding, &iop.WorkspaceCleanupRequest{RequestId: "request-1"})
|
|
return err
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := tc.run(t)
|
|
if err == nil {
|
|
t.Fatal("invalid response must fail")
|
|
}
|
|
if !errors.Is(err, errWorkspaceWireResponse) {
|
|
t.Fatalf("error = %v, want errWorkspaceWireResponse", err)
|
|
}
|
|
if strings.Contains(err.Error(), rawSentinel) {
|
|
t.Fatalf("raw node error leaked in %q", err.Error())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestWorkspaceWireBoundsRequestTimeout proves the request waiter is bounded: a
|
|
// Node that never responds fails the call well within the admitted command
|
|
// timeout rather than blocking indefinitely.
|
|
func TestWorkspaceWireBoundsRequestTimeout(t *testing.T) {
|
|
edgeClient, nodeClient := workspaceWirePipe(t)
|
|
registry := edgenode.NewRegistry()
|
|
entry := &edgenode.NodeEntry{NodeID: "node-1", Client: edgeClient}
|
|
registry.Register(entry)
|
|
svc := New(registry, edgeevents.NewBus())
|
|
binding := workspaceWireBinding("workspace-1", entry.NodeID, entry.ConnectionGeneration, 50)
|
|
_ = nodeClient // no responder registered: the request must time out at the bound
|
|
start := time.Now()
|
|
if _, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: binding.Ref}); err == nil {
|
|
t.Fatal("unanswered workspace open must fail")
|
|
}
|
|
if elapsed := time.Since(start); elapsed > time.Second {
|
|
t.Fatalf("request wait exceeded bound: %v", elapsed)
|
|
}
|
|
}
|
|
|
|
// TestWorkspaceWireRejectsContradictoryTerminalOutcome proves Required R3:
|
|
// an allowed terminal status paired with a failure error code or non-empty raw error
|
|
// returns nil response, returns errWorkspaceWireResponse, and does not leak raw error text.
|
|
func TestWorkspaceWireRejectsContradictoryTerminalOutcome(t *testing.T) {
|
|
const rawSentinel = "RAW-CONTRADICTORY-ERROR-4711"
|
|
cases := []struct {
|
|
name string
|
|
run func(t *testing.T) error
|
|
}{
|
|
{"open success with error code", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
return &iop.WorkspaceOpenResponse{
|
|
RequestId: req.GetRequestId(),
|
|
WorkspaceRef: req.GetWorkspaceRef(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_INTERNAL,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: binding.Ref})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"open success with raw error text", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
|
|
return &iop.WorkspaceOpenResponse{
|
|
RequestId: req.GetRequestId(),
|
|
WorkspaceRef: req.GetWorkspaceRef(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_UNSPECIFIED,
|
|
Error: rawSentinel,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceOpen(context.Background(), binding, &iop.WorkspaceOpenRequest{RequestId: "request-1", WorkspaceRef: binding.Ref})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"tool success with error code", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
|
|
return &iop.WorkspaceToolResponse{
|
|
RequestId: req.GetRequestId(),
|
|
StageId: req.GetStageId(),
|
|
ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_INTERNAL,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"tool success with raw error text", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
|
|
return &iop.WorkspaceToolResponse{
|
|
RequestId: req.GetRequestId(),
|
|
StageId: req.GetStageId(),
|
|
ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_UNSPECIFIED,
|
|
Error: rawSentinel,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"cancel terminal with error code", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCancelRequest, *iop.WorkspaceCancelResponse](&node.Communicator, func(req *iop.WorkspaceCancelRequest) (*iop.WorkspaceCancelResponse, error) {
|
|
return &iop.WorkspaceCancelResponse{
|
|
RequestId: req.GetRequestId(),
|
|
StageId: req.GetStageId(),
|
|
ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_INTERNAL,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceCancel(context.Background(), binding, &iop.WorkspaceCancelRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"cancel terminal with raw error text", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCancelRequest, *iop.WorkspaceCancelResponse](&node.Communicator, func(req *iop.WorkspaceCancelRequest) (*iop.WorkspaceCancelResponse, error) {
|
|
return &iop.WorkspaceCancelResponse{
|
|
RequestId: req.GetRequestId(),
|
|
StageId: req.GetStageId(),
|
|
ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_UNSPECIFIED,
|
|
Error: rawSentinel,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceCancel(context.Background(), binding, &iop.WorkspaceCancelRequest{RequestId: "request-1", StageId: "work", ToolCallId: "tool-1"})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"cleanup success with error code", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
|
|
return &iop.WorkspaceCleanupResponse{
|
|
RequestId: req.GetRequestId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_INTERNAL,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceCleanup(context.Background(), binding, &iop.WorkspaceCleanupRequest{RequestId: "request-1"})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
{"cleanup success with raw error text", func(t *testing.T) error {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
|
|
return &iop.WorkspaceCleanupResponse{
|
|
RequestId: req.GetRequestId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
|
|
ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_UNSPECIFIED,
|
|
Error: rawSentinel,
|
|
}, nil
|
|
})
|
|
resp, err := svc.workspaceCleanup(context.Background(), binding, &iop.WorkspaceCleanupRequest{RequestId: "request-1"})
|
|
if resp != nil {
|
|
t.Fatalf("expected nil response, got %+v", resp)
|
|
}
|
|
return err
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := tc.run(t)
|
|
if err == nil {
|
|
t.Fatal("contradictory terminal outcome must fail")
|
|
}
|
|
if !errors.Is(err, errWorkspaceWireResponse) {
|
|
t.Fatalf("error = %v, want errWorkspaceWireResponse", err)
|
|
}
|
|
if strings.Contains(err.Error(), rawSentinel) {
|
|
t.Fatalf("raw node error leaked in %q", err.Error())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestWorkspaceWireAcceptsCanonicalNonSuccessResponses proves that Edge accepts
|
|
// canonical non-success tool and cancel responses and retains bounded fields.
|
|
func TestWorkspaceWireAcceptsCanonicalNonSuccessResponses(t *testing.T) {
|
|
svc, node, binding := newWorkspaceWireFixture(t)
|
|
toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
|
|
switch req.GetToolCallId() {
|
|
case "nonzero":
|
|
return &iop.WorkspaceToolResponse{
|
|
RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_INTERNAL,
|
|
Error: "workspace operation failed", Stdout: []byte("out"), Stderr: []byte("err"), ExitCode: 7, DurationMs: 50,
|
|
}, nil
|
|
case "timeout":
|
|
return &iop.WorkspaceToolResponse{
|
|
RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_TIMEOUT, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_TIMEOUT,
|
|
Error: "workspace command timed out", ExitCode: -1, DurationMs: 2000,
|
|
}, nil
|
|
case "cancelled":
|
|
return &iop.WorkspaceToolResponse{
|
|
RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_CANCELLED,
|
|
Error: "workspace command cancelled", ExitCode: -1,
|
|
}, nil
|
|
default:
|
|
return nil, errors.New("unknown tool call")
|
|
}
|
|
})
|
|
|
|
nonzero, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "nonzero"})
|
|
if err != nil || nonzero.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR || nonzero.GetExitCode() != 7 || string(nonzero.GetStdout()) != "out" || string(nonzero.GetStderr()) != "err" {
|
|
t.Fatalf("nonzero response = %+v, %v", nonzero, err)
|
|
}
|
|
|
|
timeout, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "timeout"})
|
|
if err != nil || timeout.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_TIMEOUT || timeout.GetErrorCode() != iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_TIMEOUT || timeout.GetExitCode() != -1 {
|
|
t.Fatalf("timeout response = %+v, %v", timeout, err)
|
|
}
|
|
|
|
cancelled, err := svc.workspaceTool(context.Background(), binding, &iop.WorkspaceToolRequest{RequestId: "request-1", StageId: "work", ToolCallId: "cancelled"})
|
|
if err != nil || cancelled.GetStatus() != iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED || cancelled.GetErrorCode() != iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_CANCELLED {
|
|
t.Fatalf("cancelled response = %+v, %v", cancelled, err)
|
|
}
|
|
}
|