iop/apps/edge/internal/service/single_request.go
toki d7a150c7fe feat(agent): 단일 요청 실행 경로를 완성한다
Claude의 단일 Anthropic 요청 안에서 IOP가 Plan, Work, Review와 workspace 도구 실행을 끝내고 실제 dev smoke로 계약을 검증할 수 있어야 한다.\n\n완료 task evidence와 마일스톤 검토 상태도 같은 변경에 고정한다.
2026-08-08 23:35:13 +09:00

1119 lines
39 KiB
Go

package service
import (
"context"
"errors"
"fmt"
"sync"
"time"
)
var (
ErrSingleRequestExecutorUnavailable = errors.New("single-request executor is unavailable")
ErrSingleRequestInvalidRequest = errors.New("single-request: invalid request")
ErrSingleRequestInvalidBinding = errors.New("single-request: invalid binding")
ErrSingleRequestIdentityMismatch = errors.New("single-request: identity mismatch")
ErrSingleRequestInvalidSequence = errors.New("single-request: invalid envelope sequence")
ErrSingleRequestInvalidState = errors.New("single-request: invalid state transition")
ErrSingleRequestAlreadyAcknowledged = errors.New("single-request: already acknowledged")
ErrSingleRequestCancelled = errors.New("single-request: cancelled")
ErrSingleRequestFailed = errors.New("single-request: failed")
ErrSingleRequestTerminal = errors.New("single-request: execution is terminal")
ErrSingleRequestInvalidTerminal = errors.New("single-request: invalid terminal disposition")
ErrSingleRequestWorkspaceCleanup = errors.New("single-request: workspace cleanup failed")
)
type SingleRequestState string
const (
SingleRequestStateAccepted SingleRequestState = "accepted"
SingleRequestStatePlanning SingleRequestState = "planning"
SingleRequestStateWorking SingleRequestState = "working"
SingleRequestStateReviewing SingleRequestState = "reviewing"
SingleRequestStateRepairing SingleRequestState = "repairing"
SingleRequestStateInternalTool SingleRequestState = "internal_tool"
SingleRequestStateFinalizing SingleRequestState = "finalizing"
SingleRequestStateCompleted SingleRequestState = "completed"
SingleRequestStateFailed SingleRequestState = "failed"
SingleRequestStateCancelled SingleRequestState = "cancelled"
)
type SingleRequestRequest struct {
RequestID string
Binding *SingleRequestBinding
Prompt string
}
// SingleRequestTerminalKind is the closed public terminal vocabulary carried
// from the coordinator to endpoint projectors. It never contains provider,
// workspace, request, or raw error data.
type SingleRequestTerminalKind string
const (
SingleRequestTerminalEndTurn SingleRequestTerminalKind = "end_turn"
SingleRequestTerminalLength SingleRequestTerminalKind = "length"
SingleRequestTerminalError SingleRequestTerminalKind = "error"
SingleRequestTerminalCancelled SingleRequestTerminalKind = "cancelled"
)
// SingleRequestTerminalErrorClass is the closed caller-safe failure class.
// Endpoint adapters may map these values to their native status/error shapes,
// but must never replace them with raw internal errors.
type SingleRequestTerminalErrorClass string
const (
SingleRequestTerminalErrorProvider SingleRequestTerminalErrorClass = "provider"
SingleRequestTerminalErrorValidation SingleRequestTerminalErrorClass = "validation"
SingleRequestTerminalErrorTimeout SingleRequestTerminalErrorClass = "timeout"
SingleRequestTerminalErrorBudget SingleRequestTerminalErrorClass = "budget"
SingleRequestTerminalErrorRepetition SingleRequestTerminalErrorClass = "repetition"
SingleRequestTerminalErrorMalformed SingleRequestTerminalErrorClass = "malformed"
SingleRequestTerminalErrorContext SingleRequestTerminalErrorClass = "context"
SingleRequestTerminalErrorInternalTool SingleRequestTerminalErrorClass = "internal_tool"
SingleRequestTerminalErrorWorkspaceCleanup SingleRequestTerminalErrorClass = "workspace_cleanup"
)
// SingleRequestTerminalDisposition is a copy-safe terminal candidate. The
// zero value is accepted only on legacy SingleRequestResult values, where the
// coordinator normalizes it to end_turn during envelope validation.
type SingleRequestTerminalDisposition struct {
Kind SingleRequestTerminalKind
ErrorClass SingleRequestTerminalErrorClass
}
// Validate rejects every non-canonical kind/class combination.
func (d SingleRequestTerminalDisposition) Validate() error {
switch d.Kind {
case SingleRequestTerminalEndTurn, SingleRequestTerminalLength, SingleRequestTerminalCancelled:
if d.ErrorClass != "" {
return ErrSingleRequestInvalidTerminal
}
return nil
case SingleRequestTerminalError:
switch d.ErrorClass {
case SingleRequestTerminalErrorProvider, SingleRequestTerminalErrorValidation,
SingleRequestTerminalErrorTimeout, SingleRequestTerminalErrorBudget,
SingleRequestTerminalErrorRepetition, SingleRequestTerminalErrorMalformed,
SingleRequestTerminalErrorContext, SingleRequestTerminalErrorInternalTool,
SingleRequestTerminalErrorWorkspaceCleanup:
return nil
default:
return ErrSingleRequestInvalidTerminal
}
default:
return ErrSingleRequestInvalidTerminal
}
}
type SingleRequestResult struct {
Output string
Terminal SingleRequestTerminalDisposition
}
type SingleRequestProgress struct {
RequestID string
Stage SingleRequestState
Message string
Result *SingleRequestResult
Terminal *SingleRequestTerminalDisposition
Err error
}
type SingleRequestEnvelope struct {
RequestID string
Sequence uint64
Stage SingleRequestState
SavedStage SingleRequestState
ToolCall *InternalWorkspaceToolCall
Message string
Result *SingleRequestResult
Terminal *SingleRequestTerminalDisposition
Err error
}
type SingleRequestController interface {
RequestID() string
Binding() *SingleRequestBinding
Context() context.Context
State() SingleRequestState
ReadInternalArtifact(context.Context, SingleRequestArtifactKind) ([]byte, error)
WriteInternalArtifact(context.Context, SingleRequestArtifactKind, []byte) error
SubmitEnvelope(env SingleRequestEnvelope) error
}
type SingleRequestExecutor interface {
ExecuteSingleRequest(ctx context.Context, req SingleRequestRequest, ctrl SingleRequestController) error
}
type SingleRequestExecution interface {
RequestID() string
Binding() *SingleRequestBinding
State() SingleRequestState
Progress() <-chan SingleRequestProgress
AcknowledgeTerminal(success bool) error
SubmitEnvelope(env SingleRequestEnvelope) error
Wait() (SingleRequestResult, error)
Cancel()
}
type singleRequestHandle struct {
mu sync.Mutex
req SingleRequestRequest
binding *SingleRequestBinding
state SingleRequestState
savedStage SingleRequestState
lastSequence uint64
result *SingleRequestResult
terminal *SingleRequestTerminalDisposition
terminalFrozen bool
err error
acknowledged bool
progressCh chan SingleRequestProgress
progressClosed bool
doneCh chan struct{}
callerCtx context.Context
execCtx context.Context
cancelExec context.CancelFunc
execWg sync.WaitGroup
toolWg sync.WaitGroup
toolWork int
toolLoop singleRequestToolLoopState
requestDeadline time.Time
cleanupOnce sync.Once
cleanupComplete bool
// terminalErrorClass is captured when the primary failure or cancellation
// wins, before cleanup can append a secondary error.
terminalErrorClass singleRequestErrorClass
// timing is the service-owned closed observation timing accumulator.
// It is nil when the service has not injected an observer; hooks become
// no-ops in that case.
timing *singleRequestTimingAccumulator
}
func startSingleRequest(
ctx context.Context,
executor SingleRequestExecutor,
req SingleRequestRequest,
) (SingleRequestExecution, error) {
return startSingleRequestWithToolLoopObserved(ctx, executor, nil, nil, req, nil, nil)
}
func startSingleRequestWithToolLoop(
ctx context.Context,
executor SingleRequestExecutor,
continuation SingleRequestToolContinuation,
runtime singleRequestWorkspaceToolRuntime,
req SingleRequestRequest,
) (SingleRequestExecution, error) {
return startSingleRequestWithToolLoopObserved(ctx, executor, continuation, runtime, req, nil, nil)
}
// startSingleRequestWithToolLoopObserved constructs the request-owned timing
// accumulator before the accepted event or executor can run. The unobserved
// wrapper remains for focused coordinator tests; it still receives a noop
// accumulator so lifecycle hooks never need a nil-specific branch.
func startSingleRequestWithToolLoopObserved(
ctx context.Context,
executor SingleRequestExecutor,
continuation SingleRequestToolContinuation,
runtime singleRequestWorkspaceToolRuntime,
req SingleRequestRequest,
observer singleRequestObserver,
clock singleRequestClock,
) (SingleRequestExecution, error) {
if executor == nil {
return nil, ErrSingleRequestExecutorUnavailable
}
if req.RequestID == "" {
return nil, fmt.Errorf("%w: missing request_id", ErrSingleRequestInvalidRequest)
}
if req.Binding == nil {
return nil, fmt.Errorf("%w: missing binding", ErrSingleRequestInvalidBinding)
}
// Reconstruct the admission value so callers that bypassed the constructor
// cannot hand the coordinator a partially valid binding. The coordinator and
// executor then receive independent copies; neither party retains the
// caller's mutable binding.
bindingCopy, err := cloneValidatedSingleRequestBinding(req.Binding)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrSingleRequestInvalidBinding, err)
}
executorReq := SingleRequestRequest{
RequestID: req.RequestID,
Binding: bindingCopy.Clone(),
Prompt: req.Prompt,
}
execCtx, cancelExec := context.WithTimeout(ctx, time.Duration(bindingCopy.Limits.WallClockMS)*time.Millisecond)
requestDeadline, _ := execCtx.Deadline()
h := &singleRequestHandle{
req: SingleRequestRequest{
RequestID: req.RequestID,
Binding: bindingCopy.Clone(),
Prompt: req.Prompt,
},
binding: bindingCopy,
state: SingleRequestStateAccepted,
progressCh: make(chan SingleRequestProgress, 64),
doneCh: make(chan struct{}),
callerCtx: ctx,
execCtx: execCtx,
cancelExec: cancelExec,
requestDeadline: requestDeadline,
toolLoop: singleRequestToolLoopState{
continuation: continuation,
runtime: runtime,
lifecycle: workspaceLifecycle(runtime),
seenCallIDs: make(map[string]struct{}),
usage: make(map[string]singleRequestToolUsage),
},
timing: newSingleRequestTimingAccumulator(clock, observer),
}
// Send initial progress for accepted state.
h.emitProgressLocked(SingleRequestStateAccepted, false)
// The accumulator exists before accepted is published and before the
// executor goroutine launches, so every admitted request has one owner.
h.timing.onRequest()
// Monitor caller cancellation and the immutable request wall-clock budget.
go func() {
select {
case <-ctx.Done():
h.Cancel()
case <-execCtx.Done():
h.mu.Lock()
if !isTerminalState(h.state) && h.state != SingleRequestStateFinalizing {
if ctx.Err() != nil {
h.cancelLocked()
} else if errors.Is(execCtx.Err(), context.DeadlineExceeded) {
h.failLockedWithErrorClass(ErrSingleRequestInternalToolBudget, singleRequestErrorClassInternalToolBudget)
} else {
h.cancelLocked()
}
}
h.mu.Unlock()
case <-h.doneCh:
}
}()
// Launch background executor
h.execWg.Add(1)
go func() {
defer h.execWg.Done()
err := executor.ExecuteSingleRequest(execCtx, executorReq, h)
h.finalizeExecutorReturn(err)
}()
return h, nil
}
func (h *singleRequestHandle) RequestID() string {
return h.req.RequestID
}
func (h *singleRequestHandle) Binding() *SingleRequestBinding {
return h.binding.Clone()
}
func (h *singleRequestHandle) Context() context.Context {
return h.execCtx
}
func (h *singleRequestHandle) State() SingleRequestState {
h.mu.Lock()
defer h.mu.Unlock()
return h.state
}
func (h *singleRequestHandle) Progress() <-chan SingleRequestProgress {
return h.progressCh
}
func (h *singleRequestHandle) SubmitEnvelope(env SingleRequestEnvelope) error {
h.mu.Lock()
var pending *singleRequestPendingTool
defer func() {
h.mu.Unlock()
if pending != nil {
go func() {
defer func() {
h.mu.Lock()
h.toolWork--
h.mu.Unlock()
h.toolWg.Done()
}()
h.executeInternalWorkspaceTool(pending)
}()
}
}()
if env.RequestID != h.req.RequestID {
h.failLocked(ErrSingleRequestIdentityMismatch)
return ErrSingleRequestIdentityMismatch
}
if isTerminalState(h.state) {
return ErrSingleRequestTerminal
}
if env.Sequence == 0 || env.Sequence <= h.lastSequence {
h.failLocked(fmt.Errorf("%w: got %d after %d", ErrSingleRequestInvalidSequence, env.Sequence, h.lastSequence))
return ErrSingleRequestInvalidSequence
}
h.lastSequence = env.Sequence
candidate, terminal, err := h.validateEnvelopeTerminalLocked(env)
if err != nil {
h.failLocked(err)
return ErrSingleRequestInvalidState
}
if env.Stage == SingleRequestStateFailed || env.Err != nil {
err := env.Err
if err == nil {
err = ErrSingleRequestFailed
}
h.failLockedWithTerminal(err, terminal)
return nil
}
if env.Stage == SingleRequestStateCancelled {
h.cancelLockedWithTerminal(terminal)
return nil
}
if !h.validSavedStageLocked(env) {
err := fmt.Errorf("%w: invalid saved stage %s for %s -> %s", ErrSingleRequestInvalidState, env.SavedStage, h.state, env.Stage)
h.failLocked(err)
return ErrSingleRequestInvalidState
}
validTransition := isValidTransition(h.state, env.Stage, h.savedStage)
if !validTransition && candidate != nil && candidate.Terminal.Kind == SingleRequestTerminalLength && env.Stage == SingleRequestStateFinalizing {
validTransition = h.state == SingleRequestStatePlanning || h.state == SingleRequestStateWorking
}
if !validTransition {
err := fmt.Errorf("%w: invalid transition from %s to %s", ErrSingleRequestInvalidState, h.state, env.Stage)
h.failLocked(err)
return ErrSingleRequestInvalidState
}
if h.state == SingleRequestStateInternalTool && env.Stage == h.savedStage && !h.toolLoop.pendingResultReady {
err := fmt.Errorf("%w: saved stage resumed before its tool result", ErrSingleRequestInvalidState)
h.failLocked(err)
return ErrSingleRequestInvalidState
}
if env.Stage == SingleRequestStateInternalTool {
var err error
var errorClass singleRequestErrorClass
pending, err, errorClass = h.prepareInternalWorkspaceToolLocked(env.ToolCall)
if err != nil {
if errorClass == singleRequestErrorClassCancel {
h.cancelLocked()
return ErrSingleRequestCancelled
}
if errors.Is(err, ErrSingleRequestInternalToolInvalidCall) {
disposition := SingleRequestTerminalDisposition{Kind: SingleRequestTerminalError, ErrorClass: SingleRequestTerminalErrorMalformed}
h.failLockedWithTerminalAndObservation(err, &disposition, errorClass)
return err
}
h.failLockedWithErrorClass(err, errorClass)
return err
}
h.toolWg.Add(1)
h.toolWork++
}
previousStageID := h.activeStageIDLocked()
previousState := h.state
if env.Stage == SingleRequestStateInternalTool {
h.savedStage = previousState
} else if previousState == SingleRequestStateInternalTool {
h.savedStage = ""
h.toolLoop.pendingCallID = ""
h.toolLoop.pendingResultReady = false
}
h.observeTransitionLocked(previousState, previousStageID, env.Stage)
h.state = env.Stage
h.updateStageBudgetLocked(previousStageID)
// When a stage's nominal deadline reaches or exceeds the immutable request
// deadline, the request monitor is the only terminal owner. The stage
// context still inherits execCtx, but its timer must not race the request
// wall-clock budget with a timeout disposition.
if !h.requestDeadline.IsZero() && !h.toolLoop.stageDeadline.IsZero() && !h.toolLoop.stageDeadline.Before(h.requestDeadline) {
h.stopStageBudgetLocked()
}
if candidate != nil {
h.result = candidate
h.terminal = cloneSingleRequestTerminal(&candidate.Terminal)
}
if h.state == SingleRequestStateFinalizing {
h.requestTerminalCleanupLocked()
} else {
h.emitProgressLocked(h.state, false)
}
return nil
}
// observeTransitionLocked keeps timing keyed to the semantic provider role,
// not the transient envelope state. In particular reviewing -> repairing is
// one review stage, and internal_tool pauses and later resumes its saved stage.
// Caller must hold h.mu.
func (h *singleRequestHandle) observeTransitionLocked(previousState SingleRequestState, previousStageID string, next SingleRequestState) {
previousStage := singleRequestNormalizeStage(previousStageID)
nextStage := singleRequestNormalizeStage(canonicalSingleRequestStageID(next))
switch {
case next == SingleRequestStateInternalTool:
h.timing.onToolEnter()
case previousState == SingleRequestStateInternalTool && nextStage == previousStage:
// executeInternalWorkspaceTool emitted the actual tool completion and
// resumed this stage before its continuation submitted the envelope.
case nextStage != previousStage:
if previousStage != "" {
h.timing.onStageExit(previousStage, singleRequestOutcomeSuccess, "")
}
if nextStage != "" {
h.timing.onStageEnter(nextStage)
}
}
}
// closeObservationStageLocked closes the active semantic stage at a terminal
// candidate. It also handles a stage paused for an in-flight tool; the
// accumulator retains the pre-tool segment while the tool itself records its
// own outcome when the Node call settles.
func (h *singleRequestHandle) closeObservationStageLocked(outcome singleRequestOutcome, errorClass singleRequestErrorClass) {
if stage := singleRequestNormalizeStage(h.activeStageIDLocked()); stage != "" {
h.timing.onStageExit(stage, outcome, errorClass)
}
}
func (h *singleRequestHandle) AcknowledgeTerminal(success bool) error {
h.mu.Lock()
defer h.mu.Unlock()
if h.state != SingleRequestStateFinalizing {
return fmt.Errorf("%w: cannot acknowledge terminal in state %s", ErrSingleRequestInvalidState, h.state)
}
if h.acknowledged {
return ErrSingleRequestAlreadyAcknowledged
}
if success && h.result == nil {
err := fmt.Errorf("%w: successful acknowledgement requires a finalizing candidate", ErrSingleRequestInvalidState)
h.failLocked(err)
return ErrSingleRequestInvalidState
}
if !h.cleanupComplete {
return fmt.Errorf("%w: workspace cleanup is pending", ErrSingleRequestInvalidState)
}
if success {
h.acknowledged = true
h.state = SingleRequestStateCompleted
h.emitProgressLocked(h.state, true)
h.finishLocked(nil)
} else {
h.acknowledged = true
err := fmt.Errorf("%w: endpoint write failed", ErrSingleRequestFailed)
h.failLocked(err)
}
return nil
}
func (h *singleRequestHandle) Wait() (SingleRequestResult, error) {
<-h.doneCh
h.execWg.Wait()
h.toolWg.Wait()
h.mu.Lock()
defer h.mu.Unlock()
var res SingleRequestResult
if h.result != nil {
res = *h.result
}
return res, h.err
}
func (h *singleRequestHandle) Cancel() {
h.mu.Lock()
defer h.mu.Unlock()
h.cancelLocked()
}
func (h *singleRequestHandle) cancelLocked() {
disposition := SingleRequestTerminalDisposition{Kind: SingleRequestTerminalCancelled}
h.cancelLockedWithTerminal(&disposition)
}
func (h *singleRequestHandle) cancelLockedWithTerminal(terminal *SingleRequestTerminalDisposition) {
if isTerminalState(h.state) {
return
}
if terminal == nil || terminal.Validate() != nil || terminal.Kind != SingleRequestTerminalCancelled {
fallback := SingleRequestTerminalDisposition{Kind: SingleRequestTerminalCancelled}
terminal = &fallback
}
if h.err == nil {
h.err = ErrSingleRequestCancelled
h.terminalErrorClass = singleRequestErrorClassCancel
}
if !h.terminalFrozen {
h.result = nil
h.terminal = cloneSingleRequestTerminal(terminal)
}
h.closeObservationStageLocked(singleRequestOutcomeCancel, singleRequestErrorClassCancel)
h.state = SingleRequestStateCancelled
if h.terminalFrozen {
// The endpoint already owns the frozen finalizing candidate. Cancellation
// may change the internal completion outcome, but cannot publish another
// terminal candidate on the coordinator progress channel.
h.finishLocked(h.err)
return
}
if h.cleanupComplete {
h.emitProgressLocked(h.state, true)
h.finishLocked(h.err)
return
}
h.requestTerminalCleanupLocked()
}
func (h *singleRequestHandle) failLocked(err error) {
h.failLockedWithErrorClass(err, "")
}
// failLockedWithErrorClass preserves the caller-visible failure sentinel while
// allowing a lifecycle owner to record its more specific terminal observation
// class. The first primary failure remains authoritative across cleanup joins.
func (h *singleRequestHandle) failLockedWithErrorClass(err error, errorClass singleRequestErrorClass) {
disposition := singleRequestTerminalDispositionFromError(err, errorClass)
h.failLockedWithTerminalAndObservation(err, &disposition, errorClass)
}
func (h *singleRequestHandle) failLockedWithTerminal(err error, terminal *SingleRequestTerminalDisposition) {
h.failLockedWithTerminalAndObservation(err, terminal, "")
}
func (h *singleRequestHandle) failLockedWithTerminalAndObservation(err error, terminal *SingleRequestTerminalDisposition, errorClass singleRequestErrorClass) {
if isTerminalState(h.state) {
return
}
if terminal == nil || terminal.Validate() != nil || terminal.Kind != SingleRequestTerminalError {
fallback := singleRequestTerminalDispositionFromError(err, errorClass)
terminal = &fallback
}
if h.err == nil && err != nil {
h.err = err
if errorClass == "" {
errorClass = singleRequestObservationErrorClass(*terminal, err)
}
h.terminalErrorClass = errorClass
}
if !h.terminalFrozen {
h.result = nil
h.terminal = cloneSingleRequestTerminal(terminal)
}
terminalErrorClass := h.terminalErrorClass
if terminalErrorClass == "" {
terminalErrorClass = singleRequestObservationErrorClass(*terminal, h.err)
}
h.closeObservationStageLocked(singleRequestOutcomeError, terminalErrorClass)
h.state = SingleRequestStateFailed
if h.terminalFrozen {
// A finalizing disposition already crossed the public progress boundary.
// A write acknowledgement failure is internal-only and never emits a
// conflicting failed disposition.
h.finishLocked(h.err)
return
}
if h.cleanupComplete {
h.emitProgressLocked(h.state, true)
h.finishLocked(h.err)
return
}
h.requestTerminalCleanupLocked()
}
func workspaceLifecycle(runtime singleRequestWorkspaceToolRuntime) SingleRequestWorkspaceLifecycle {
if runtime == nil {
return nil
}
lifecycle, _ := runtime.(SingleRequestWorkspaceLifecycle)
return lifecycle
}
// requestTerminalCleanupLocked starts the one coordinator-owned cleanup gate.
// It is called with h.mu held for successful, failed, and cancelled candidates.
func (h *singleRequestHandle) requestTerminalCleanupLocked() {
h.cancelExec()
h.stopStageBudgetLocked()
h.cleanupOnce.Do(func() {
h.timing.onCleanupEnter()
if !h.toolLoop.opened && h.toolWork == 0 {
h.completeTerminalCleanupLocked(nil)
return
}
go h.runTerminalCleanup()
})
}
func (h *singleRequestHandle) runTerminalCleanup() {
// An open or tool request that raced the terminal candidate must settle
// before deciding whether a Node workspace lifecycle exists.
h.toolWg.Wait()
h.mu.Lock()
opened := h.toolLoop.opened
lifecycle := h.toolLoop.lifecycle
binding := h.binding.Workspace.Clone()
requestID := h.req.RequestID
deadline := h.requestDeadline
h.mu.Unlock()
var cleanupErr error
if opened {
if lifecycle == nil || binding == nil {
cleanupErr = ErrSingleRequestWorkspaceCleanup
} else {
var (
cleanupCtx context.Context
cancel context.CancelFunc
)
if deadline.IsZero() {
cleanupCtx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
} else {
cleanupCtx, cancel = context.WithDeadline(context.Background(), deadline)
}
cleanupErr = lifecycle.CleanupWorkspace(cleanupCtx, binding, requestID)
cancel()
}
}
h.mu.Lock()
h.completeTerminalCleanupLocked(cleanupErr)
h.mu.Unlock()
}
func (h *singleRequestHandle) completeTerminalCleanupLocked(cleanupErr error) {
if h.cleanupComplete {
return
}
h.cleanupComplete = true
cleanupOutcome := singleRequestOutcomeSuccess
cleanupClass := singleRequestErrorClass("")
if cleanupErr != nil {
cleanupOutcome = singleRequestOutcomeError
cleanupClass = singleRequestErrorClassWorkspaceCleanup
}
h.timing.onCleanupExit(cleanupOutcome, cleanupClass)
if cleanupErr != nil {
cleanupConvertedSuccess := h.err == nil && h.state == SingleRequestStateFinalizing
if h.err == nil {
h.err = ErrSingleRequestWorkspaceCleanup
} else if !errors.Is(h.err, ErrSingleRequestWorkspaceCleanup) {
h.err = errors.Join(h.err, ErrSingleRequestWorkspaceCleanup)
}
if h.state == SingleRequestStateFinalizing {
h.state = SingleRequestStateFailed
}
if cleanupConvertedSuccess {
h.terminalErrorClass = singleRequestErrorClassWorkspaceCleanup
h.result = nil
h.terminal = &SingleRequestTerminalDisposition{
Kind: SingleRequestTerminalError,
ErrorClass: SingleRequestTerminalErrorWorkspaceCleanup,
}
}
}
switch h.state {
case SingleRequestStateFinalizing:
h.emitProgressLocked(h.state, true)
case SingleRequestStateFailed, SingleRequestStateCancelled:
h.emitProgressLocked(h.state, true)
h.finishLocked(h.err)
}
}
func (h *singleRequestHandle) finishLocked(err error) {
if h.err == nil && err != nil {
h.err = err
}
h.cancelExec()
h.stopStageBudgetLocked()
// Emit terminal and total observation events exactly once.
// The terminal winner owns exactly one terminal event and one request-total event.
outcome, errorClass := h.terminalOutcomeAndErrorClass()
h.closeObservationStageLocked(outcome, errorClass)
h.timing.onTerminal(outcome, errorClass, h.terminalHasResultLocked())
select {
case <-h.doneCh:
default:
close(h.doneCh)
}
if !h.progressClosed {
h.progressClosed = true
close(h.progressCh)
}
}
func (h *singleRequestHandle) finalizeExecutorReturn(err error) {
h.mu.Lock()
defer h.mu.Unlock()
if isTerminalState(h.state) || h.state == SingleRequestStateFinalizing {
return
}
if err != nil {
if h.callerCtx.Err() != nil {
h.cancelLocked()
return
}
if errors.Is(h.execCtx.Err(), context.DeadlineExceeded) ||
!h.requestDeadline.IsZero() && !time.Now().Before(h.requestDeadline) {
h.failLockedWithErrorClass(ErrSingleRequestInternalToolBudget, singleRequestErrorClassInternalToolBudget)
return
}
h.failLocked(err)
return
}
// A finalizing request legitimately waits for the surface to commit the
// prepared terminal. Every other normal executor return is premature.
if h.state != SingleRequestStateFinalizing {
h.failLocked(fmt.Errorf("%w: executor returned in state %s", ErrSingleRequestFailed, h.state))
}
}
// classifyChildOperationContext applies the request-owned cancellation and
// deadline order to every derived tool or artifact context. The immutable
// request budget is authoritative over an inherited child deadline; a child
// timeout is reported only while the caller and request contexts remain live.
func (h *singleRequestHandle) classifyChildOperationContext(ctx context.Context, fallback singleRequestErrorClass) (singleRequestOutcome, singleRequestErrorClass) {
now := time.Now()
switch {
case h.callerCtx != nil && h.callerCtx.Err() != nil:
return singleRequestOutcomeCancel, singleRequestErrorClassCancel
case h.execCtx != nil && errors.Is(h.execCtx.Err(), context.DeadlineExceeded),
!h.requestDeadline.IsZero() && !now.Before(h.requestDeadline):
return singleRequestOutcomeError, singleRequestErrorClassInternalToolBudget
case ctx != nil && errors.Is(ctx.Err(), context.DeadlineExceeded), childOperationDeadlineReached(ctx, now):
return singleRequestOutcomeError, singleRequestErrorClassTimeout
case ctx != nil && errors.Is(ctx.Err(), context.Canceled):
return singleRequestOutcomeCancel, singleRequestErrorClassCancel
default:
if fallback == "" {
fallback = singleRequestErrorClassInternalToolFailed
}
return singleRequestOutcomeError, fallback
}
}
func childOperationDeadlineReached(ctx context.Context, now time.Time) bool {
if ctx == nil {
return false
}
deadline, ok := ctx.Deadline()
return ok && !now.Before(deadline)
}
func (h *singleRequestHandle) validSavedStageLocked(env SingleRequestEnvelope) bool {
if h.state == SingleRequestStateInternalTool {
return env.Stage == h.savedStage && env.SavedStage == h.savedStage
}
if env.Stage == SingleRequestStateInternalTool {
return env.SavedStage == h.state
}
return env.SavedStage == ""
}
func (h *singleRequestHandle) validateEnvelopeTerminalLocked(env SingleRequestEnvelope) (*SingleRequestResult, *SingleRequestTerminalDisposition, error) {
if env.Stage == SingleRequestStateInternalTool {
if env.ToolCall == nil {
return nil, nil, fmt.Errorf("%w: internal tool stage requires one call", ErrSingleRequestInvalidState)
}
} else if env.ToolCall != nil {
return nil, nil, fmt.Errorf("%w: tool call is only valid for internal tool stage", ErrSingleRequestInvalidState)
}
isFailure := env.Stage == SingleRequestStateFailed || env.Err != nil
switch {
case isFailure:
if env.Result != nil {
return nil, nil, fmt.Errorf("%w: failed terminal cannot carry a result", ErrSingleRequestInvalidState)
}
terminal := cloneSingleRequestTerminal(env.Terminal)
if terminal == nil {
fallback := singleRequestTerminalDispositionFromError(env.Err, "")
terminal = &fallback
}
if terminal.Validate() != nil || terminal.Kind != SingleRequestTerminalError {
return nil, nil, ErrSingleRequestInvalidTerminal
}
return nil, terminal, nil
case env.Stage == SingleRequestStateCancelled:
if env.Result != nil {
return nil, nil, fmt.Errorf("%w: cancelled terminal cannot carry a result", ErrSingleRequestInvalidState)
}
terminal := cloneSingleRequestTerminal(env.Terminal)
if terminal == nil {
terminal = &SingleRequestTerminalDisposition{Kind: SingleRequestTerminalCancelled}
}
if terminal.Validate() != nil || terminal.Kind != SingleRequestTerminalCancelled {
return nil, nil, ErrSingleRequestInvalidTerminal
}
return nil, terminal, nil
case env.Stage == SingleRequestStateFinalizing:
if env.Terminal != nil {
return nil, nil, fmt.Errorf("%w: finalizing disposition belongs to its result", ErrSingleRequestInvalidState)
}
if env.Result == nil {
return nil, nil, fmt.Errorf("%w: finalizing requires a result", ErrSingleRequestInvalidState)
}
candidate := cloneSingleRequestResult(env.Result)
if candidate.Terminal.Kind == "" && candidate.Terminal.ErrorClass == "" {
candidate.Terminal.Kind = SingleRequestTerminalEndTurn
}
if candidate.Terminal.Validate() != nil ||
(candidate.Terminal.Kind != SingleRequestTerminalEndTurn && candidate.Terminal.Kind != SingleRequestTerminalLength) {
return nil, nil, ErrSingleRequestInvalidTerminal
}
return candidate, cloneSingleRequestTerminal(&candidate.Terminal), nil
default:
if env.Result != nil {
return nil, nil, fmt.Errorf("%w: result is only valid for finalizing", ErrSingleRequestInvalidState)
}
if env.Terminal != nil {
return nil, nil, fmt.Errorf("%w: terminal is only valid for a terminal candidate", ErrSingleRequestInvalidState)
}
return nil, nil, nil
}
}
func cloneSingleRequestResult(result *SingleRequestResult) *SingleRequestResult {
if result == nil {
return nil
}
return &SingleRequestResult{Output: result.Output, Terminal: result.Terminal}
}
func cloneSingleRequestTerminal(terminal *SingleRequestTerminalDisposition) *SingleRequestTerminalDisposition {
if terminal == nil {
return nil
}
copy := *terminal
return &copy
}
func (h *singleRequestHandle) emitProgressLocked(stage SingleRequestState, critical bool) {
progress := SingleRequestProgress{
RequestID: h.req.RequestID,
Stage: stage,
Message: safeSingleRequestProgressMessage(stage),
}
if stage == SingleRequestStateFinalizing {
progress.Result = cloneSingleRequestResult(h.result)
}
if stage == SingleRequestStateFinalizing || stage == SingleRequestStateFailed || stage == SingleRequestStateCancelled {
progress.Terminal = cloneSingleRequestTerminal(h.terminal)
h.terminalFrozen = progress.Terminal != nil
}
h.notifyProgressLocked(progress, critical)
}
func safeSingleRequestProgressMessage(stage SingleRequestState) string {
switch stage {
case SingleRequestStateAccepted:
return "request accepted"
case SingleRequestStatePlanning:
return "planning started"
case SingleRequestStateWorking:
return "work started"
case SingleRequestStateReviewing:
return "review started"
case SingleRequestStateRepairing:
return "repair started"
case SingleRequestStateInternalTool:
return "internal work in progress"
case SingleRequestStateFinalizing:
return "final response ready"
case SingleRequestStateCompleted:
return "execution completed"
case SingleRequestStateFailed:
return "execution failed"
case SingleRequestStateCancelled:
return "execution cancelled"
default:
return "execution update"
}
}
func (h *singleRequestHandle) notifyProgressLocked(prog SingleRequestProgress, critical bool) {
if h.progressClosed {
return
}
// Keep two slots available for the finalizing candidate and the terminal
// outcome. Regular updates are intentionally lossy, but those two lifecycle
// boundaries cannot be displaced by a saturated executor progress stream.
if !critical && len(h.progressCh) >= cap(h.progressCh)-2 {
return
}
select {
case h.progressCh <- prog:
default:
if critical {
select {
case <-h.progressCh:
default:
}
select {
case h.progressCh <- prog:
default:
}
}
}
}
func isTerminalState(s SingleRequestState) bool {
return s == SingleRequestStateCompleted || s == SingleRequestStateFailed || s == SingleRequestStateCancelled
}
// terminalOutcomeAndErrorClass derives the closed outcome and error class
// for the terminal observation event from the current handle state. It is
// called exactly once by finishLocked under h.mu.
func (h *singleRequestHandle) terminalOutcomeAndErrorClass() (singleRequestOutcome, singleRequestErrorClass) {
switch h.state {
case SingleRequestStateCompleted:
return singleRequestOutcomeSuccess, ""
case SingleRequestStateCancelled:
return singleRequestOutcomeCancel, singleRequestErrorClassCancel
case SingleRequestStateFailed:
if h.terminalErrorClass != "" {
return singleRequestOutcomeError, h.terminalErrorClass
}
return singleRequestOutcomeError, singleRequestErrorClassFromErr(h.err)
default:
return singleRequestOutcomeError, singleRequestErrorClassProvider
}
}
// singleRequestErrorClassFromErr derives a closed error class with typed
// sentinel matching. Unknown errors normalize to provider.
func singleRequestErrorClassFromErr(err error) singleRequestErrorClass {
if err == nil {
return singleRequestErrorClassProvider
}
switch {
case errors.Is(err, ErrSingleRequestInternalToolBudget):
return singleRequestErrorClassInternalToolBudget
case errors.Is(err, ErrSingleRequestInternalToolFailed), errors.Is(err, ErrSingleRequestInternalToolUnavailable):
return singleRequestErrorClassInternalToolFailed
case errors.Is(err, ErrSingleRequestWorkspaceCleanup):
return singleRequestErrorClassWorkspaceCleanup
case errors.Is(err, context.DeadlineExceeded):
return singleRequestErrorClassTimeout
case errors.Is(err, context.Canceled), errors.Is(err, ErrSingleRequestCancelled):
return singleRequestErrorClassCancel
case errors.Is(err, ErrSingleRequestInvalidRequest), errors.Is(err, ErrSingleRequestInvalidBinding),
errors.Is(err, ErrSingleRequestIdentityMismatch), errors.Is(err, ErrSingleRequestInvalidSequence),
errors.Is(err, ErrSingleRequestInvalidState), errors.Is(err, ErrSingleRequestInternalToolInvalidCall),
errors.Is(err, ErrSingleRequestInternalToolDenied):
return singleRequestErrorClassValidation
default:
return singleRequestErrorClassProvider
}
}
func singleRequestTerminalDispositionFromError(err error, observed singleRequestErrorClass) SingleRequestTerminalDisposition {
errorClass := SingleRequestTerminalErrorProvider
switch {
case observed == singleRequestErrorClassValidation:
errorClass = SingleRequestTerminalErrorValidation
case observed == singleRequestErrorClassTimeout:
errorClass = SingleRequestTerminalErrorTimeout
case observed == singleRequestErrorClassInternalToolBudget:
errorClass = SingleRequestTerminalErrorBudget
case observed == singleRequestErrorClassInternalToolFailed:
errorClass = SingleRequestTerminalErrorInternalTool
case observed == singleRequestErrorClassWorkspaceCleanup:
errorClass = SingleRequestTerminalErrorWorkspaceCleanup
case errors.Is(err, ErrSingleRequestInternalToolBudget):
errorClass = SingleRequestTerminalErrorBudget
case errors.Is(err, ErrSingleRequestInternalToolFailed), errors.Is(err, ErrSingleRequestInternalToolUnavailable):
errorClass = SingleRequestTerminalErrorInternalTool
case errors.Is(err, ErrSingleRequestWorkspaceCleanup):
errorClass = SingleRequestTerminalErrorWorkspaceCleanup
case errors.Is(err, context.DeadlineExceeded):
errorClass = SingleRequestTerminalErrorTimeout
case errors.Is(err, ErrSingleRequestInvalidRequest), errors.Is(err, ErrSingleRequestInvalidBinding),
errors.Is(err, ErrSingleRequestIdentityMismatch), errors.Is(err, ErrSingleRequestInvalidSequence),
errors.Is(err, ErrSingleRequestInvalidState), errors.Is(err, ErrSingleRequestInvalidTerminal),
errors.Is(err, ErrSingleRequestInternalToolInvalidCall), errors.Is(err, ErrSingleRequestInternalToolDenied):
errorClass = SingleRequestTerminalErrorValidation
}
return SingleRequestTerminalDisposition{Kind: SingleRequestTerminalError, ErrorClass: errorClass}
}
// singleRequestObservationErrorClass projects the richer endpoint disposition
// into the pre-existing closed metric vocabulary. This task intentionally adds
// no metric labels or cardinality.
func singleRequestObservationErrorClass(terminal SingleRequestTerminalDisposition, err error) singleRequestErrorClass {
switch terminal.ErrorClass {
case SingleRequestTerminalErrorValidation, SingleRequestTerminalErrorContext, SingleRequestTerminalErrorMalformed:
return singleRequestErrorClassValidation
case SingleRequestTerminalErrorTimeout:
return singleRequestErrorClassTimeout
case SingleRequestTerminalErrorBudget, SingleRequestTerminalErrorRepetition:
return singleRequestErrorClassInternalToolBudget
case SingleRequestTerminalErrorInternalTool:
return singleRequestErrorClassInternalToolFailed
case SingleRequestTerminalErrorWorkspaceCleanup:
return singleRequestErrorClassWorkspaceCleanup
case SingleRequestTerminalErrorProvider:
return singleRequestErrorClassProvider
default:
return singleRequestErrorClassFromErr(err)
}
}
// terminalHasResultLocked reports whether a finalizing candidate was prepared.
// Caller must hold h.mu.
func (h *singleRequestHandle) terminalHasResultLocked() bool {
return h.result != nil
}
func isValidTransition(from, to, savedStage SingleRequestState) bool {
if isTerminalState(from) {
return false
}
switch from {
case SingleRequestStateAccepted:
return to == SingleRequestStatePlanning || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
case SingleRequestStatePlanning:
return to == SingleRequestStateInternalTool || to == SingleRequestStateWorking || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
case SingleRequestStateWorking:
return to == SingleRequestStateInternalTool || to == SingleRequestStateReviewing || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
case SingleRequestStateReviewing:
return to == SingleRequestStateInternalTool || to == SingleRequestStateRepairing || to == SingleRequestStateFinalizing || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
case SingleRequestStateRepairing:
return to == SingleRequestStateInternalTool || to == SingleRequestStateFinalizing || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
case SingleRequestStateInternalTool:
if to == SingleRequestStateFailed || to == SingleRequestStateCancelled {
return true
}
return to == savedStage
case SingleRequestStateFinalizing:
return to == SingleRequestStateFailed || to == SingleRequestStateCancelled
default:
return false
}
}