승인된 execution preset을 Edge 조정 경계와 Node workspace/tool 실행 경계로 연결해 단일 요청 수명주기와 관측 계약을 일관되게 처리한다.
840 lines
26 KiB
Go
840 lines
26 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
ErrSingleRequestExecutorUnavailable = errors.New("single-request executor is unavailable")
|
|
ErrSingleRequestInvalidRequest = errors.New("single-request: invalid request")
|
|
ErrSingleRequestInvalidBinding = errors.New("single-request: invalid binding")
|
|
ErrSingleRequestIdentityMismatch = errors.New("single-request: identity mismatch")
|
|
ErrSingleRequestInvalidSequence = errors.New("single-request: invalid envelope sequence")
|
|
ErrSingleRequestInvalidState = errors.New("single-request: invalid state transition")
|
|
ErrSingleRequestAlreadyAcknowledged = errors.New("single-request: already acknowledged")
|
|
ErrSingleRequestCancelled = errors.New("single-request: cancelled")
|
|
ErrSingleRequestFailed = errors.New("single-request: failed")
|
|
ErrSingleRequestTerminal = errors.New("single-request: execution is terminal")
|
|
ErrSingleRequestWorkspaceCleanup = errors.New("single-request: workspace cleanup failed")
|
|
)
|
|
|
|
type SingleRequestState string
|
|
|
|
const (
|
|
SingleRequestStateAccepted SingleRequestState = "accepted"
|
|
SingleRequestStatePlanning SingleRequestState = "planning"
|
|
SingleRequestStateWorking SingleRequestState = "working"
|
|
SingleRequestStateReviewing SingleRequestState = "reviewing"
|
|
SingleRequestStateRepairing SingleRequestState = "repairing"
|
|
SingleRequestStateInternalTool SingleRequestState = "internal_tool"
|
|
SingleRequestStateFinalizing SingleRequestState = "finalizing"
|
|
SingleRequestStateCompleted SingleRequestState = "completed"
|
|
SingleRequestStateFailed SingleRequestState = "failed"
|
|
SingleRequestStateCancelled SingleRequestState = "cancelled"
|
|
)
|
|
|
|
type SingleRequestRequest struct {
|
|
RequestID string
|
|
Binding *SingleRequestBinding
|
|
Prompt string
|
|
}
|
|
|
|
type SingleRequestResult struct {
|
|
Output string
|
|
}
|
|
|
|
type SingleRequestProgress struct {
|
|
RequestID string
|
|
Stage SingleRequestState
|
|
Message string
|
|
Result *SingleRequestResult
|
|
Err error
|
|
}
|
|
|
|
type SingleRequestEnvelope struct {
|
|
RequestID string
|
|
Sequence uint64
|
|
Stage SingleRequestState
|
|
SavedStage SingleRequestState
|
|
ToolCall *InternalWorkspaceToolCall
|
|
Message string
|
|
Result *SingleRequestResult
|
|
Err error
|
|
}
|
|
|
|
type SingleRequestController interface {
|
|
RequestID() string
|
|
Binding() *SingleRequestBinding
|
|
Context() context.Context
|
|
State() SingleRequestState
|
|
SubmitEnvelope(env SingleRequestEnvelope) error
|
|
}
|
|
|
|
type SingleRequestExecutor interface {
|
|
ExecuteSingleRequest(ctx context.Context, req SingleRequestRequest, ctrl SingleRequestController) error
|
|
}
|
|
|
|
type SingleRequestExecution interface {
|
|
RequestID() string
|
|
Binding() *SingleRequestBinding
|
|
State() SingleRequestState
|
|
Progress() <-chan SingleRequestProgress
|
|
AcknowledgeTerminal(success bool) error
|
|
SubmitEnvelope(env SingleRequestEnvelope) error
|
|
Wait() (SingleRequestResult, error)
|
|
Cancel()
|
|
}
|
|
|
|
type singleRequestHandle struct {
|
|
mu sync.Mutex
|
|
req SingleRequestRequest
|
|
binding *SingleRequestBinding
|
|
state SingleRequestState
|
|
savedStage SingleRequestState
|
|
lastSequence uint64
|
|
result *SingleRequestResult
|
|
err error
|
|
acknowledged bool
|
|
progressCh chan SingleRequestProgress
|
|
progressClosed bool
|
|
doneCh chan struct{}
|
|
callerCtx context.Context
|
|
execCtx context.Context
|
|
cancelExec context.CancelFunc
|
|
execWg sync.WaitGroup
|
|
toolWg sync.WaitGroup
|
|
toolWork int
|
|
toolLoop singleRequestToolLoopState
|
|
requestDeadline time.Time
|
|
cleanupOnce sync.Once
|
|
cleanupComplete bool
|
|
// terminalErrorClass is captured when the primary failure or cancellation
|
|
// wins, before cleanup can append a secondary error.
|
|
terminalErrorClass singleRequestErrorClass
|
|
// timing is the service-owned closed observation timing accumulator.
|
|
// It is nil when the service has not injected an observer; hooks become
|
|
// no-ops in that case.
|
|
timing *singleRequestTimingAccumulator
|
|
}
|
|
|
|
func startSingleRequest(
|
|
ctx context.Context,
|
|
executor SingleRequestExecutor,
|
|
req SingleRequestRequest,
|
|
) (SingleRequestExecution, error) {
|
|
return startSingleRequestWithToolLoopObserved(ctx, executor, nil, nil, req, nil, nil)
|
|
}
|
|
|
|
func startSingleRequestWithToolLoop(
|
|
ctx context.Context,
|
|
executor SingleRequestExecutor,
|
|
continuation SingleRequestToolContinuation,
|
|
runtime singleRequestWorkspaceToolRuntime,
|
|
req SingleRequestRequest,
|
|
) (SingleRequestExecution, error) {
|
|
return startSingleRequestWithToolLoopObserved(ctx, executor, continuation, runtime, req, nil, nil)
|
|
}
|
|
|
|
// startSingleRequestWithToolLoopObserved constructs the request-owned timing
|
|
// accumulator before the accepted event or executor can run. The unobserved
|
|
// wrapper remains for focused coordinator tests; it still receives a noop
|
|
// accumulator so lifecycle hooks never need a nil-specific branch.
|
|
func startSingleRequestWithToolLoopObserved(
|
|
ctx context.Context,
|
|
executor SingleRequestExecutor,
|
|
continuation SingleRequestToolContinuation,
|
|
runtime singleRequestWorkspaceToolRuntime,
|
|
req SingleRequestRequest,
|
|
observer singleRequestObserver,
|
|
clock singleRequestClock,
|
|
) (SingleRequestExecution, error) {
|
|
if executor == nil {
|
|
return nil, ErrSingleRequestExecutorUnavailable
|
|
}
|
|
if req.RequestID == "" {
|
|
return nil, fmt.Errorf("%w: missing request_id", ErrSingleRequestInvalidRequest)
|
|
}
|
|
if req.Binding == nil {
|
|
return nil, fmt.Errorf("%w: missing binding", ErrSingleRequestInvalidBinding)
|
|
}
|
|
|
|
// Reconstruct the admission value so callers that bypassed the constructor
|
|
// cannot hand the coordinator a partially valid binding. The coordinator and
|
|
// executor then receive independent copies; neither party retains the
|
|
// caller's mutable binding.
|
|
bindingCopy, err := cloneValidatedSingleRequestBinding(req.Binding)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: %v", ErrSingleRequestInvalidBinding, err)
|
|
}
|
|
executorReq := SingleRequestRequest{
|
|
RequestID: req.RequestID,
|
|
Binding: bindingCopy.Clone(),
|
|
Prompt: req.Prompt,
|
|
}
|
|
|
|
execCtx, cancelExec := context.WithTimeout(ctx, time.Duration(bindingCopy.Limits.WallClockMS)*time.Millisecond)
|
|
requestDeadline, _ := execCtx.Deadline()
|
|
|
|
h := &singleRequestHandle{
|
|
req: SingleRequestRequest{
|
|
RequestID: req.RequestID,
|
|
Binding: bindingCopy.Clone(),
|
|
Prompt: req.Prompt,
|
|
},
|
|
binding: bindingCopy,
|
|
state: SingleRequestStateAccepted,
|
|
progressCh: make(chan SingleRequestProgress, 64),
|
|
doneCh: make(chan struct{}),
|
|
callerCtx: ctx,
|
|
execCtx: execCtx,
|
|
cancelExec: cancelExec,
|
|
requestDeadline: requestDeadline,
|
|
toolLoop: singleRequestToolLoopState{
|
|
continuation: continuation,
|
|
runtime: runtime,
|
|
lifecycle: workspaceLifecycle(runtime),
|
|
seenCallIDs: make(map[string]struct{}),
|
|
usage: make(map[string]singleRequestToolUsage),
|
|
},
|
|
timing: newSingleRequestTimingAccumulator(clock, observer),
|
|
}
|
|
|
|
// Send initial progress for accepted state.
|
|
h.emitProgressLocked(SingleRequestStateAccepted, false)
|
|
|
|
// The accumulator exists before accepted is published and before the
|
|
// executor goroutine launches, so every admitted request has one owner.
|
|
h.timing.onRequest()
|
|
|
|
// Monitor caller cancellation and the immutable request wall-clock budget.
|
|
go func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
h.Cancel()
|
|
case <-execCtx.Done():
|
|
h.mu.Lock()
|
|
if !isTerminalState(h.state) && h.state != SingleRequestStateFinalizing {
|
|
if ctx.Err() != nil {
|
|
h.cancelLocked()
|
|
} else if errors.Is(execCtx.Err(), context.DeadlineExceeded) {
|
|
h.failLockedWithErrorClass(ErrSingleRequestInternalToolBudget, singleRequestErrorClassTimeout)
|
|
} else {
|
|
h.cancelLocked()
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
case <-h.doneCh:
|
|
}
|
|
}()
|
|
|
|
// Launch background executor
|
|
h.execWg.Add(1)
|
|
go func() {
|
|
defer h.execWg.Done()
|
|
err := executor.ExecuteSingleRequest(execCtx, executorReq, h)
|
|
h.finalizeExecutorReturn(err)
|
|
}()
|
|
|
|
return h, nil
|
|
}
|
|
|
|
func (h *singleRequestHandle) RequestID() string {
|
|
return h.req.RequestID
|
|
}
|
|
|
|
func (h *singleRequestHandle) Binding() *SingleRequestBinding {
|
|
return h.binding.Clone()
|
|
}
|
|
|
|
func (h *singleRequestHandle) Context() context.Context {
|
|
return h.execCtx
|
|
}
|
|
|
|
func (h *singleRequestHandle) State() SingleRequestState {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
return h.state
|
|
}
|
|
|
|
func (h *singleRequestHandle) Progress() <-chan SingleRequestProgress {
|
|
return h.progressCh
|
|
}
|
|
|
|
func (h *singleRequestHandle) SubmitEnvelope(env SingleRequestEnvelope) error {
|
|
h.mu.Lock()
|
|
var pending *singleRequestPendingTool
|
|
defer func() {
|
|
h.mu.Unlock()
|
|
if pending != nil {
|
|
go func() {
|
|
defer func() {
|
|
h.mu.Lock()
|
|
h.toolWork--
|
|
h.mu.Unlock()
|
|
h.toolWg.Done()
|
|
}()
|
|
h.executeInternalWorkspaceTool(pending)
|
|
}()
|
|
}
|
|
}()
|
|
|
|
if env.RequestID != h.req.RequestID {
|
|
h.failLocked(ErrSingleRequestIdentityMismatch)
|
|
return ErrSingleRequestIdentityMismatch
|
|
}
|
|
|
|
if isTerminalState(h.state) {
|
|
return ErrSingleRequestTerminal
|
|
}
|
|
if env.Sequence == 0 || env.Sequence <= h.lastSequence {
|
|
h.failLocked(fmt.Errorf("%w: got %d after %d", ErrSingleRequestInvalidSequence, env.Sequence, h.lastSequence))
|
|
return ErrSingleRequestInvalidSequence
|
|
}
|
|
h.lastSequence = env.Sequence
|
|
candidate, err := h.validateEnvelopeResultLocked(env)
|
|
if err != nil {
|
|
h.failLocked(err)
|
|
return ErrSingleRequestInvalidState
|
|
}
|
|
|
|
if env.Stage == SingleRequestStateFailed || env.Err != nil {
|
|
err := env.Err
|
|
if err == nil {
|
|
err = ErrSingleRequestFailed
|
|
}
|
|
h.failLocked(err)
|
|
return nil
|
|
}
|
|
|
|
if env.Stage == SingleRequestStateCancelled {
|
|
h.cancelLocked()
|
|
return nil
|
|
}
|
|
|
|
if !h.validSavedStageLocked(env) {
|
|
err := fmt.Errorf("%w: invalid saved stage %s for %s -> %s", ErrSingleRequestInvalidState, env.SavedStage, h.state, env.Stage)
|
|
h.failLocked(err)
|
|
return ErrSingleRequestInvalidState
|
|
}
|
|
|
|
if !isValidTransition(h.state, env.Stage, h.savedStage) {
|
|
err := fmt.Errorf("%w: invalid transition from %s to %s", ErrSingleRequestInvalidState, h.state, env.Stage)
|
|
h.failLocked(err)
|
|
return ErrSingleRequestInvalidState
|
|
}
|
|
if h.state == SingleRequestStateInternalTool && env.Stage == h.savedStage && !h.toolLoop.pendingResultReady {
|
|
err := fmt.Errorf("%w: saved stage resumed before its tool result", ErrSingleRequestInvalidState)
|
|
h.failLocked(err)
|
|
return ErrSingleRequestInvalidState
|
|
}
|
|
if env.Stage == SingleRequestStateInternalTool {
|
|
var err error
|
|
var errorClass singleRequestErrorClass
|
|
pending, err, errorClass = h.prepareInternalWorkspaceToolLocked(env.ToolCall)
|
|
if err != nil {
|
|
h.failLockedWithErrorClass(err, errorClass)
|
|
return err
|
|
}
|
|
h.toolWg.Add(1)
|
|
h.toolWork++
|
|
}
|
|
|
|
previousStageID := h.activeStageIDLocked()
|
|
previousState := h.state
|
|
if env.Stage == SingleRequestStateInternalTool {
|
|
h.savedStage = previousState
|
|
} else if previousState == SingleRequestStateInternalTool {
|
|
h.savedStage = ""
|
|
h.toolLoop.pendingCallID = ""
|
|
h.toolLoop.pendingResultReady = false
|
|
}
|
|
|
|
h.observeTransitionLocked(previousState, previousStageID, env.Stage)
|
|
|
|
h.state = env.Stage
|
|
h.updateStageBudgetLocked(previousStageID)
|
|
if candidate != nil {
|
|
h.result = candidate
|
|
}
|
|
if h.state == SingleRequestStateFinalizing {
|
|
h.requestTerminalCleanupLocked()
|
|
} else {
|
|
h.emitProgressLocked(h.state, false)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// observeTransitionLocked keeps timing keyed to the semantic provider role,
|
|
// not the transient envelope state. In particular reviewing -> repairing is
|
|
// one review stage, and internal_tool pauses and later resumes its saved stage.
|
|
// Caller must hold h.mu.
|
|
func (h *singleRequestHandle) observeTransitionLocked(previousState SingleRequestState, previousStageID string, next SingleRequestState) {
|
|
previousStage := singleRequestNormalizeStage(previousStageID)
|
|
nextStage := singleRequestNormalizeStage(canonicalSingleRequestStageID(next))
|
|
|
|
switch {
|
|
case next == SingleRequestStateInternalTool:
|
|
h.timing.onToolEnter()
|
|
case previousState == SingleRequestStateInternalTool && nextStage == previousStage:
|
|
// executeInternalWorkspaceTool emitted the actual tool completion and
|
|
// resumed this stage before its continuation submitted the envelope.
|
|
case nextStage != previousStage:
|
|
if previousStage != "" {
|
|
h.timing.onStageExit(previousStage, singleRequestOutcomeSuccess, "")
|
|
}
|
|
if nextStage != "" {
|
|
h.timing.onStageEnter(nextStage)
|
|
}
|
|
}
|
|
}
|
|
|
|
// closeObservationStageLocked closes the active semantic stage at a terminal
|
|
// candidate. It also handles a stage paused for an in-flight tool; the
|
|
// accumulator retains the pre-tool segment while the tool itself records its
|
|
// own outcome when the Node call settles.
|
|
func (h *singleRequestHandle) closeObservationStageLocked(outcome singleRequestOutcome, errorClass singleRequestErrorClass) {
|
|
if stage := singleRequestNormalizeStage(h.activeStageIDLocked()); stage != "" {
|
|
h.timing.onStageExit(stage, outcome, errorClass)
|
|
}
|
|
}
|
|
|
|
func (h *singleRequestHandle) AcknowledgeTerminal(success bool) error {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
|
|
if h.state != SingleRequestStateFinalizing {
|
|
return fmt.Errorf("%w: cannot acknowledge terminal in state %s", ErrSingleRequestInvalidState, h.state)
|
|
}
|
|
|
|
if h.acknowledged {
|
|
return ErrSingleRequestAlreadyAcknowledged
|
|
}
|
|
|
|
if success && h.result == nil {
|
|
err := fmt.Errorf("%w: successful acknowledgement requires a finalizing candidate", ErrSingleRequestInvalidState)
|
|
h.failLocked(err)
|
|
return ErrSingleRequestInvalidState
|
|
}
|
|
if !h.cleanupComplete {
|
|
return fmt.Errorf("%w: workspace cleanup is pending", ErrSingleRequestInvalidState)
|
|
}
|
|
|
|
if success {
|
|
h.acknowledged = true
|
|
h.state = SingleRequestStateCompleted
|
|
h.emitProgressLocked(h.state, true)
|
|
h.finishLocked(nil)
|
|
} else {
|
|
h.acknowledged = true
|
|
err := fmt.Errorf("%w: endpoint write failed", ErrSingleRequestFailed)
|
|
h.failLocked(err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (h *singleRequestHandle) Wait() (SingleRequestResult, error) {
|
|
<-h.doneCh
|
|
h.execWg.Wait()
|
|
h.toolWg.Wait()
|
|
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
|
|
var res SingleRequestResult
|
|
if h.result != nil {
|
|
res = *h.result
|
|
}
|
|
return res, h.err
|
|
}
|
|
|
|
func (h *singleRequestHandle) Cancel() {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.cancelLocked()
|
|
}
|
|
|
|
func (h *singleRequestHandle) cancelLocked() {
|
|
if isTerminalState(h.state) {
|
|
return
|
|
}
|
|
if h.err == nil {
|
|
h.err = ErrSingleRequestCancelled
|
|
h.terminalErrorClass = singleRequestErrorClassCancel
|
|
}
|
|
h.closeObservationStageLocked(singleRequestOutcomeCancel, singleRequestErrorClassCancel)
|
|
h.state = SingleRequestStateCancelled
|
|
if h.cleanupComplete {
|
|
h.emitProgressLocked(h.state, true)
|
|
h.finishLocked(h.err)
|
|
return
|
|
}
|
|
h.requestTerminalCleanupLocked()
|
|
}
|
|
|
|
func (h *singleRequestHandle) failLocked(err error) {
|
|
h.failLockedWithErrorClass(err, "")
|
|
}
|
|
|
|
// failLockedWithErrorClass preserves the caller-visible failure sentinel while
|
|
// allowing a lifecycle owner to record its more specific terminal observation
|
|
// class. The first primary failure remains authoritative across cleanup joins.
|
|
func (h *singleRequestHandle) failLockedWithErrorClass(err error, errorClass singleRequestErrorClass) {
|
|
if isTerminalState(h.state) {
|
|
return
|
|
}
|
|
if h.err == nil && err != nil {
|
|
h.err = err
|
|
if errorClass == "" {
|
|
errorClass = singleRequestErrorClassFromErr(err)
|
|
}
|
|
h.terminalErrorClass = errorClass
|
|
}
|
|
terminalErrorClass := h.terminalErrorClass
|
|
if terminalErrorClass == "" {
|
|
terminalErrorClass = singleRequestErrorClassFromErr(h.err)
|
|
}
|
|
h.closeObservationStageLocked(singleRequestOutcomeError, terminalErrorClass)
|
|
h.state = SingleRequestStateFailed
|
|
if h.cleanupComplete {
|
|
h.emitProgressLocked(h.state, true)
|
|
h.finishLocked(h.err)
|
|
return
|
|
}
|
|
h.requestTerminalCleanupLocked()
|
|
}
|
|
|
|
func workspaceLifecycle(runtime singleRequestWorkspaceToolRuntime) SingleRequestWorkspaceLifecycle {
|
|
if runtime == nil {
|
|
return nil
|
|
}
|
|
lifecycle, _ := runtime.(SingleRequestWorkspaceLifecycle)
|
|
return lifecycle
|
|
}
|
|
|
|
// requestTerminalCleanupLocked starts the one coordinator-owned cleanup gate.
|
|
// It is called with h.mu held for successful, failed, and cancelled candidates.
|
|
func (h *singleRequestHandle) requestTerminalCleanupLocked() {
|
|
h.cancelExec()
|
|
h.stopStageBudgetLocked()
|
|
h.cleanupOnce.Do(func() {
|
|
h.timing.onCleanupEnter()
|
|
if !h.toolLoop.opened && h.toolWork == 0 {
|
|
h.completeTerminalCleanupLocked(nil)
|
|
return
|
|
}
|
|
go h.runTerminalCleanup()
|
|
})
|
|
}
|
|
|
|
func (h *singleRequestHandle) runTerminalCleanup() {
|
|
// An open or tool request that raced the terminal candidate must settle
|
|
// before deciding whether a Node workspace lifecycle exists.
|
|
h.toolWg.Wait()
|
|
|
|
h.mu.Lock()
|
|
opened := h.toolLoop.opened
|
|
lifecycle := h.toolLoop.lifecycle
|
|
binding := h.binding.Workspace.Clone()
|
|
requestID := h.req.RequestID
|
|
deadline := h.requestDeadline
|
|
h.mu.Unlock()
|
|
|
|
var cleanupErr error
|
|
if opened {
|
|
if lifecycle == nil || binding == nil {
|
|
cleanupErr = ErrSingleRequestWorkspaceCleanup
|
|
} else {
|
|
var (
|
|
cleanupCtx context.Context
|
|
cancel context.CancelFunc
|
|
)
|
|
if deadline.IsZero() {
|
|
cleanupCtx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
|
|
} else {
|
|
cleanupCtx, cancel = context.WithDeadline(context.Background(), deadline)
|
|
}
|
|
cleanupErr = lifecycle.CleanupWorkspace(cleanupCtx, binding, requestID)
|
|
cancel()
|
|
}
|
|
}
|
|
|
|
h.mu.Lock()
|
|
h.completeTerminalCleanupLocked(cleanupErr)
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
func (h *singleRequestHandle) completeTerminalCleanupLocked(cleanupErr error) {
|
|
if h.cleanupComplete {
|
|
return
|
|
}
|
|
h.cleanupComplete = true
|
|
cleanupOutcome := singleRequestOutcomeSuccess
|
|
cleanupClass := singleRequestErrorClass("")
|
|
if cleanupErr != nil {
|
|
cleanupOutcome = singleRequestOutcomeError
|
|
cleanupClass = singleRequestErrorClassWorkspaceCleanup
|
|
}
|
|
h.timing.onCleanupExit(cleanupOutcome, cleanupClass)
|
|
if cleanupErr != nil {
|
|
cleanupConvertedSuccess := h.err == nil && h.state == SingleRequestStateFinalizing
|
|
if h.err == nil {
|
|
h.err = ErrSingleRequestWorkspaceCleanup
|
|
} else if !errors.Is(h.err, ErrSingleRequestWorkspaceCleanup) {
|
|
h.err = errors.Join(h.err, ErrSingleRequestWorkspaceCleanup)
|
|
}
|
|
if h.state == SingleRequestStateFinalizing {
|
|
h.state = SingleRequestStateFailed
|
|
}
|
|
if cleanupConvertedSuccess {
|
|
h.terminalErrorClass = singleRequestErrorClassWorkspaceCleanup
|
|
}
|
|
}
|
|
switch h.state {
|
|
case SingleRequestStateFinalizing:
|
|
h.emitProgressLocked(h.state, true)
|
|
case SingleRequestStateFailed, SingleRequestStateCancelled:
|
|
h.emitProgressLocked(h.state, true)
|
|
h.finishLocked(h.err)
|
|
}
|
|
}
|
|
|
|
func (h *singleRequestHandle) finishLocked(err error) {
|
|
if h.err == nil && err != nil {
|
|
h.err = err
|
|
}
|
|
h.cancelExec()
|
|
h.stopStageBudgetLocked()
|
|
|
|
// Emit terminal and total observation events exactly once.
|
|
// The terminal winner owns exactly one terminal event and one request-total event.
|
|
outcome, errorClass := h.terminalOutcomeAndErrorClass()
|
|
h.closeObservationStageLocked(outcome, errorClass)
|
|
h.timing.onTerminal(outcome, errorClass, h.terminalHasResultLocked())
|
|
|
|
select {
|
|
case <-h.doneCh:
|
|
default:
|
|
close(h.doneCh)
|
|
}
|
|
if !h.progressClosed {
|
|
h.progressClosed = true
|
|
close(h.progressCh)
|
|
}
|
|
}
|
|
|
|
func (h *singleRequestHandle) finalizeExecutorReturn(err error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
|
|
if isTerminalState(h.state) || h.state == SingleRequestStateFinalizing {
|
|
return
|
|
}
|
|
if err != nil {
|
|
if h.callerCtx.Err() != nil || errors.Is(err, context.Canceled) && !errors.Is(h.execCtx.Err(), context.DeadlineExceeded) {
|
|
h.cancelLocked()
|
|
return
|
|
}
|
|
if errors.Is(err, context.DeadlineExceeded) || errors.Is(h.execCtx.Err(), context.DeadlineExceeded) {
|
|
h.failLockedWithErrorClass(ErrSingleRequestInternalToolBudget, singleRequestErrorClassTimeout)
|
|
return
|
|
}
|
|
h.failLocked(err)
|
|
return
|
|
}
|
|
// A finalizing request legitimately waits for the surface to commit the
|
|
// prepared terminal. Every other normal executor return is premature.
|
|
if h.state != SingleRequestStateFinalizing {
|
|
h.failLocked(fmt.Errorf("%w: executor returned in state %s", ErrSingleRequestFailed, h.state))
|
|
}
|
|
}
|
|
|
|
func (h *singleRequestHandle) validSavedStageLocked(env SingleRequestEnvelope) bool {
|
|
if h.state == SingleRequestStateInternalTool {
|
|
return env.Stage == h.savedStage && env.SavedStage == h.savedStage
|
|
}
|
|
if env.Stage == SingleRequestStateInternalTool {
|
|
return env.SavedStage == h.state
|
|
}
|
|
return env.SavedStage == ""
|
|
}
|
|
|
|
func (h *singleRequestHandle) validateEnvelopeResultLocked(env SingleRequestEnvelope) (*SingleRequestResult, error) {
|
|
if env.Stage == SingleRequestStateInternalTool {
|
|
if env.ToolCall == nil {
|
|
return nil, fmt.Errorf("%w: internal tool stage requires one call", ErrSingleRequestInvalidState)
|
|
}
|
|
} else if env.ToolCall != nil {
|
|
return nil, fmt.Errorf("%w: tool call is only valid for internal tool stage", ErrSingleRequestInvalidState)
|
|
}
|
|
if env.Stage != SingleRequestStateFinalizing {
|
|
if env.Result != nil {
|
|
return nil, fmt.Errorf("%w: result is only valid for finalizing", ErrSingleRequestInvalidState)
|
|
}
|
|
return nil, nil
|
|
}
|
|
if env.Result == nil {
|
|
return nil, fmt.Errorf("%w: finalizing requires a result", ErrSingleRequestInvalidState)
|
|
}
|
|
return cloneSingleRequestResult(env.Result), nil
|
|
}
|
|
|
|
func cloneSingleRequestResult(result *SingleRequestResult) *SingleRequestResult {
|
|
if result == nil {
|
|
return nil
|
|
}
|
|
return &SingleRequestResult{Output: result.Output}
|
|
}
|
|
|
|
func (h *singleRequestHandle) emitProgressLocked(stage SingleRequestState, critical bool) {
|
|
progress := SingleRequestProgress{
|
|
RequestID: h.req.RequestID,
|
|
Stage: stage,
|
|
Message: safeSingleRequestProgressMessage(stage),
|
|
}
|
|
if stage == SingleRequestStateFinalizing {
|
|
progress.Result = cloneSingleRequestResult(h.result)
|
|
}
|
|
h.notifyProgressLocked(progress, critical)
|
|
}
|
|
|
|
func safeSingleRequestProgressMessage(stage SingleRequestState) string {
|
|
switch stage {
|
|
case SingleRequestStateAccepted:
|
|
return "request accepted"
|
|
case SingleRequestStatePlanning:
|
|
return "planning started"
|
|
case SingleRequestStateWorking:
|
|
return "work started"
|
|
case SingleRequestStateReviewing:
|
|
return "review started"
|
|
case SingleRequestStateRepairing:
|
|
return "repair started"
|
|
case SingleRequestStateInternalTool:
|
|
return "internal work in progress"
|
|
case SingleRequestStateFinalizing:
|
|
return "final response ready"
|
|
case SingleRequestStateCompleted:
|
|
return "execution completed"
|
|
case SingleRequestStateFailed:
|
|
return "execution failed"
|
|
case SingleRequestStateCancelled:
|
|
return "execution cancelled"
|
|
default:
|
|
return "execution update"
|
|
}
|
|
}
|
|
|
|
func (h *singleRequestHandle) notifyProgressLocked(prog SingleRequestProgress, critical bool) {
|
|
if h.progressClosed {
|
|
return
|
|
}
|
|
// Keep two slots available for the finalizing candidate and the terminal
|
|
// outcome. Regular updates are intentionally lossy, but those two lifecycle
|
|
// boundaries cannot be displaced by a saturated executor progress stream.
|
|
if !critical && len(h.progressCh) >= cap(h.progressCh)-2 {
|
|
return
|
|
}
|
|
select {
|
|
case h.progressCh <- prog:
|
|
default:
|
|
if critical {
|
|
select {
|
|
case <-h.progressCh:
|
|
default:
|
|
}
|
|
select {
|
|
case h.progressCh <- prog:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func isTerminalState(s SingleRequestState) bool {
|
|
return s == SingleRequestStateCompleted || s == SingleRequestStateFailed || s == SingleRequestStateCancelled
|
|
}
|
|
|
|
// terminalOutcomeAndErrorClass derives the closed outcome and error class
|
|
// for the terminal observation event from the current handle state. It is
|
|
// called exactly once by finishLocked under h.mu.
|
|
func (h *singleRequestHandle) terminalOutcomeAndErrorClass() (singleRequestOutcome, singleRequestErrorClass) {
|
|
switch h.state {
|
|
case SingleRequestStateCompleted:
|
|
return singleRequestOutcomeSuccess, ""
|
|
case SingleRequestStateCancelled:
|
|
return singleRequestOutcomeCancel, singleRequestErrorClassCancel
|
|
case SingleRequestStateFailed:
|
|
if h.terminalErrorClass != "" {
|
|
return singleRequestOutcomeError, h.terminalErrorClass
|
|
}
|
|
return singleRequestOutcomeError, singleRequestErrorClassFromErr(h.err)
|
|
default:
|
|
return singleRequestOutcomeError, singleRequestErrorClassProvider
|
|
}
|
|
}
|
|
|
|
// singleRequestErrorClassFromErr derives a closed error class with typed
|
|
// sentinel matching. Unknown errors normalize to provider.
|
|
func singleRequestErrorClassFromErr(err error) singleRequestErrorClass {
|
|
if err == nil {
|
|
return singleRequestErrorClassProvider
|
|
}
|
|
switch {
|
|
case errors.Is(err, ErrSingleRequestInternalToolBudget):
|
|
return singleRequestErrorClassInternalToolBudget
|
|
case errors.Is(err, ErrSingleRequestInternalToolFailed), errors.Is(err, ErrSingleRequestInternalToolUnavailable):
|
|
return singleRequestErrorClassInternalToolFailed
|
|
case errors.Is(err, ErrSingleRequestWorkspaceCleanup):
|
|
return singleRequestErrorClassWorkspaceCleanup
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|
return singleRequestErrorClassTimeout
|
|
case errors.Is(err, context.Canceled), errors.Is(err, ErrSingleRequestCancelled):
|
|
return singleRequestErrorClassCancel
|
|
case errors.Is(err, ErrSingleRequestInvalidRequest), errors.Is(err, ErrSingleRequestInvalidBinding),
|
|
errors.Is(err, ErrSingleRequestIdentityMismatch), errors.Is(err, ErrSingleRequestInvalidSequence),
|
|
errors.Is(err, ErrSingleRequestInvalidState), errors.Is(err, ErrSingleRequestInternalToolInvalidCall),
|
|
errors.Is(err, ErrSingleRequestInternalToolDenied):
|
|
return singleRequestErrorClassValidation
|
|
default:
|
|
return singleRequestErrorClassProvider
|
|
}
|
|
}
|
|
|
|
// terminalHasResultLocked reports whether a finalizing candidate was prepared.
|
|
// Caller must hold h.mu.
|
|
func (h *singleRequestHandle) terminalHasResultLocked() bool {
|
|
return h.result != nil
|
|
}
|
|
|
|
func isValidTransition(from, to, savedStage SingleRequestState) bool {
|
|
if isTerminalState(from) {
|
|
return false
|
|
}
|
|
switch from {
|
|
case SingleRequestStateAccepted:
|
|
return to == SingleRequestStatePlanning || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
|
|
case SingleRequestStatePlanning:
|
|
return to == SingleRequestStateInternalTool || to == SingleRequestStateWorking || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
|
|
case SingleRequestStateWorking:
|
|
return to == SingleRequestStateInternalTool || to == SingleRequestStateReviewing || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
|
|
case SingleRequestStateReviewing:
|
|
return to == SingleRequestStateInternalTool || to == SingleRequestStateRepairing || to == SingleRequestStateFinalizing || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
|
|
case SingleRequestStateRepairing:
|
|
return to == SingleRequestStateInternalTool || to == SingleRequestStateFinalizing || to == SingleRequestStateFailed || to == SingleRequestStateCancelled
|
|
case SingleRequestStateInternalTool:
|
|
if to == SingleRequestStateFailed || to == SingleRequestStateCancelled {
|
|
return true
|
|
}
|
|
return to == savedStage
|
|
case SingleRequestStateFinalizing:
|
|
return to == SingleRequestStateFailed || to == SingleRequestStateCancelled
|
|
default:
|
|
return false
|
|
}
|
|
}
|