toki
e5869a53ae
caller별 예외 대신 요청 의미와 protocol profile capability로 operation을 선택해 tools와 effort 조합을 보존한다. 지원하지 않는 effort는 가장 가까운 하위 등급으로만 내리고 상향 매핑은 거부한다.
832 lines
33 KiB
Go
832 lines
33 KiB
Go
package openai
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
type anthropicClientError struct {
|
|
errorType string
|
|
message string
|
|
}
|
|
|
|
const anthropicPreIngressRejectionLogMessage = "edge_anthropic_pre_ingress_rejection"
|
|
|
|
const anthropicSingleRequestTerminalRejectionLogMessage = "edge_single_request_terminal_rejection"
|
|
|
|
type anthropicPreIngressRejectionClass string
|
|
|
|
const (
|
|
anthropicPreIngressMethod anthropicPreIngressRejectionClass = "method"
|
|
anthropicPreIngressInvalidHeader anthropicPreIngressRejectionClass = "invalid_header"
|
|
anthropicPreIngressUnsupportedBeta anthropicPreIngressRejectionClass = "unsupported_beta"
|
|
anthropicPreIngressBodyRead anthropicPreIngressRejectionClass = "body_read"
|
|
anthropicPreIngressBodyLimit anthropicPreIngressRejectionClass = "body_limit"
|
|
anthropicPreIngressInvalidEnvelope anthropicPreIngressRejectionClass = "invalid_envelope"
|
|
anthropicPreIngressInvalidMaxTokens anthropicPreIngressRejectionClass = "invalid_max_tokens"
|
|
anthropicPreIngressRoute anthropicPreIngressRejectionClass = "route"
|
|
anthropicPreIngressUnknownField anthropicPreIngressRejectionClass = "unknown_field"
|
|
anthropicPreIngressInvalidThinking anthropicPreIngressRejectionClass = "invalid_thinking"
|
|
anthropicPreIngressInvalidOutput anthropicPreIngressRejectionClass = "invalid_output_config"
|
|
anthropicPreIngressInvalidRequest anthropicPreIngressRejectionClass = "invalid_request"
|
|
anthropicPreIngressRuntimeUnavailable anthropicPreIngressRejectionClass = "runtime_unavailable"
|
|
)
|
|
|
|
func classifyAnthropicPreIngressRejection(err error) anthropicPreIngressRejectionClass {
|
|
if err == nil {
|
|
return anthropicPreIngressInvalidRequest
|
|
}
|
|
message := err.Error()
|
|
switch {
|
|
case strings.Contains(message, "unsupported anthropic-beta"):
|
|
return anthropicPreIngressUnsupportedBeta
|
|
case strings.Contains(message, "json: unknown field"):
|
|
return anthropicPreIngressUnknownField
|
|
case strings.Contains(message, "thinking.display"),
|
|
strings.Contains(message, "adaptive thinking"),
|
|
strings.Contains(message, "thinking must be enabled"):
|
|
return anthropicPreIngressInvalidThinking
|
|
case strings.Contains(message, "output_config.effort"),
|
|
strings.Contains(message, "output_config.format"):
|
|
return anthropicPreIngressInvalidOutput
|
|
default:
|
|
return anthropicPreIngressInvalidRequest
|
|
}
|
|
}
|
|
|
|
func (s *Server) observeAnthropicPreIngressRejection(class anthropicPreIngressRejectionClass, status int) {
|
|
s.logger.Info(
|
|
anthropicPreIngressRejectionLogMessage,
|
|
zap.String("surface", "messages"),
|
|
zap.String("rejection_class", string(class)),
|
|
zap.Int("http_status", status),
|
|
)
|
|
}
|
|
|
|
func (s *Server) writeAnthropicPreIngressError(
|
|
w http.ResponseWriter,
|
|
status int,
|
|
errorType string,
|
|
message string,
|
|
class anthropicPreIngressRejectionClass,
|
|
) {
|
|
s.observeAnthropicPreIngressRejection(class, status)
|
|
writeAnthropicError(w, status, errorType, message)
|
|
}
|
|
|
|
// anthropicHotPathDispositionPolicy is the caller-native projection of the
|
|
// protocol-neutral Hot Path terminal vocabulary. The codec decides whether the
|
|
// response is still uncommitted (JSON status/error) or already streaming (one
|
|
// error event); this table owns only the stable Anthropic semantic mapping.
|
|
type anthropicHotPathDispositionPolicy struct {
|
|
status int
|
|
errorType string
|
|
stopReason string
|
|
silent bool
|
|
errorTerminal bool
|
|
}
|
|
|
|
func anthropicHotPathPolicy(disposition hotPathTerminalDisposition) anthropicHotPathDispositionPolicy {
|
|
switch disposition.Kind {
|
|
case hotPathDispositionSuccess:
|
|
return anthropicHotPathDispositionPolicy{status: http.StatusOK, stopReason: "end_turn"}
|
|
case hotPathDispositionToolTurn:
|
|
return anthropicHotPathDispositionPolicy{status: http.StatusOK, stopReason: "tool_use"}
|
|
case hotPathDispositionLength:
|
|
return anthropicHotPathDispositionPolicy{status: http.StatusOK, stopReason: "max_tokens"}
|
|
case hotPathDispositionValidationError:
|
|
return anthropicHotPathDispositionPolicy{
|
|
status: http.StatusBadRequest, errorType: "invalid_request_error", errorTerminal: true,
|
|
}
|
|
case hotPathDispositionProviderError, hotPathDispositionTimeout:
|
|
return anthropicHotPathDispositionPolicy{
|
|
status: http.StatusBadGateway, errorType: "api_error", errorTerminal: true,
|
|
}
|
|
case hotPathDispositionCallerCancel:
|
|
return anthropicHotPathDispositionPolicy{silent: true}
|
|
default:
|
|
return anthropicHotPathDispositionPolicy{
|
|
status: http.StatusBadGateway, errorType: "api_error", errorTerminal: true,
|
|
}
|
|
}
|
|
}
|
|
|
|
// singleRequestAnthropicTerminalPolicy is the one buffered/SSE projection of
|
|
// the service-owned terminal disposition. Messages and statuses are closed and
|
|
// never contain provider, tool, workspace, or raw error data.
|
|
type singleRequestAnthropicTerminalPolicy struct {
|
|
status int
|
|
errorType string
|
|
message string
|
|
stopReason string
|
|
silent bool
|
|
errorTerminal bool
|
|
}
|
|
|
|
func singleRequestAnthropicPolicy(disposition edgeservice.SingleRequestTerminalDisposition) singleRequestAnthropicTerminalPolicy {
|
|
if disposition.Kind == "" && disposition.ErrorClass == "" {
|
|
disposition.Kind = edgeservice.SingleRequestTerminalEndTurn
|
|
}
|
|
if disposition.Validate() != nil {
|
|
disposition = edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}
|
|
}
|
|
switch disposition.Kind {
|
|
case edgeservice.SingleRequestTerminalEndTurn:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusOK, stopReason: "end_turn"}
|
|
case edgeservice.SingleRequestTerminalLength:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusOK, stopReason: "max_tokens"}
|
|
case edgeservice.SingleRequestTerminalCancelled:
|
|
return singleRequestAnthropicTerminalPolicy{silent: true}
|
|
case edgeservice.SingleRequestTerminalError:
|
|
switch disposition.ErrorClass {
|
|
case edgeservice.SingleRequestTerminalErrorValidation:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusBadRequest, errorType: "invalid_request_error", message: "single-request execution was rejected", errorTerminal: true}
|
|
case edgeservice.SingleRequestTerminalErrorContext:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusBadRequest, errorType: "invalid_request_error", message: "single-request context limit exceeded", errorTerminal: true}
|
|
case edgeservice.SingleRequestTerminalErrorTimeout:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusBadGateway, errorType: "api_error", message: "single-request execution timed out", errorTerminal: true}
|
|
default:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusBadGateway, errorType: "api_error", message: "single-request execution failed", errorTerminal: true}
|
|
}
|
|
default:
|
|
return singleRequestAnthropicTerminalPolicy{status: http.StatusBadGateway, errorType: "api_error", message: "single-request execution failed", errorTerminal: true}
|
|
}
|
|
}
|
|
|
|
func (e *anthropicClientError) Error() string { return e.message }
|
|
|
|
func newAnthropicClientError(errorType string, err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
return &anthropicClientError{errorType: errorType, message: err.Error()}
|
|
}
|
|
|
|
func (s *Server) handleAnthropicMessages(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
s.writeAnthropicPreIngressError(w, http.StatusMethodNotAllowed, "invalid_request_error", "method not allowed", anthropicPreIngressMethod)
|
|
return
|
|
}
|
|
defer r.Body.Close()
|
|
if err := validateAnthropicHeaders(r); err != nil {
|
|
class := anthropicPreIngressInvalidHeader
|
|
if classifyAnthropicPreIngressRejection(err) == anthropicPreIngressUnsupportedBeta {
|
|
class = anthropicPreIngressUnsupportedBeta
|
|
}
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", err.Error(), class)
|
|
return
|
|
}
|
|
body, err := readOpenAIIngressBody(w, r, s.maxIngressSnapshotBytes())
|
|
if err != nil {
|
|
class := anthropicPreIngressBodyRead
|
|
if errors.Is(err, errOpenAIIngressTooLarge) {
|
|
class = anthropicPreIngressBodyLimit
|
|
}
|
|
s.observeAnthropicPreIngressRejection(class, anthropicIngressErrorStatus(err))
|
|
writeAnthropicIngressError(w, err)
|
|
return
|
|
}
|
|
envelope, err := decodeAnthropicEnvelope(body)
|
|
if err != nil {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", err.Error(), anthropicPreIngressInvalidEnvelope)
|
|
return
|
|
}
|
|
var tokenLimit struct {
|
|
MaxTokens *int `json:"max_tokens"`
|
|
}
|
|
if err := json.Unmarshal(body, &tokenLimit); err != nil {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", "decode Messages request", anthropicPreIngressInvalidMaxTokens)
|
|
return
|
|
}
|
|
if tokenLimit.MaxTokens == nil {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", "max_tokens is required", anthropicPreIngressInvalidMaxTokens)
|
|
return
|
|
}
|
|
if *tokenLimit.MaxTokens <= 0 {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", "max_tokens must be positive", anthropicPreIngressInvalidMaxTokens)
|
|
return
|
|
}
|
|
if err := validateAnthropicOutputEffort(body); err != nil {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", err.Error(), anthropicPreIngressInvalidOutput)
|
|
return
|
|
}
|
|
dispatch, err := s.resolveRouteDispatchForPrincipal(r.Context(), envelope.Model)
|
|
if err != nil || !dispatch.ProviderPool {
|
|
s.observeAnthropicPreIngressRejection(anthropicPreIngressRoute, http.StatusBadRequest)
|
|
s.writeAnthropicRouteError(w, err)
|
|
return
|
|
}
|
|
if dispatch.SingleRequest != nil {
|
|
request, err := decodeAnthropicMessageRequest(body, true)
|
|
if err != nil {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", err.Error(), classifyAnthropicPreIngressRejection(err))
|
|
return
|
|
}
|
|
capability, ok := s.service.(singleRequestService)
|
|
if !ok {
|
|
s.observeAnthropicPreIngressRejection(anthropicPreIngressRuntimeUnavailable, http.StatusServiceUnavailable)
|
|
writeAnthropicSingleRequestUnavailable(w)
|
|
return
|
|
}
|
|
recordSingleRequestIngress()
|
|
if request.Stream {
|
|
s.handleAnthropicSingleRequestStream(w, r, capability, dispatch, body)
|
|
} else {
|
|
s.handleAnthropicSingleRequest(w, r, capability, dispatch, body)
|
|
}
|
|
return
|
|
}
|
|
|
|
requirements, err := decodeAnthropicProviderRequirements(body)
|
|
if err != nil {
|
|
s.writeAnthropicPreIngressError(w, http.StatusBadRequest, "invalid_request_error", err.Error(), anthropicPreIngressInvalidEnvelope)
|
|
return
|
|
}
|
|
poolReq, presetIngress, err := s.anthropicPoolRequest(r, dispatch, envelope, body, config.OperationMessages, requirements)
|
|
if err != nil {
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
if dispatch.IsPreset {
|
|
applyHotPathOutputTokenCap(poolReq.Run.Metadata, tokenLimit.MaxTokens)
|
|
presetCodec := newAnthropicHotPathCodec(
|
|
w, dispatch.ExternalModelID, envelope.Stream,
|
|
poolReq.Run.Metadata["iop_logical_request_id"], hotPathOutputTokenCap(poolReq.Run.Metadata),
|
|
)
|
|
r = withHotPathAnthropicCodec(r, presetCodec)
|
|
}
|
|
if presetIngress.localStageEligible() {
|
|
_ = s.runHotPathLocalEligible(w, r, dispatch, "anthropic", envelope.Stream, poolReq.Run.Metadata)
|
|
return
|
|
}
|
|
if presetIngress.lightStageContinuation() {
|
|
_ = s.runHotPathLightContinuation(w, r, dispatch, "anthropic", envelope.Stream, poolReq.Run.Metadata)
|
|
return
|
|
}
|
|
if presetIngress.cleanupIssued() {
|
|
_ = s.writeHotPathStageResponse(w, r, dispatch, "anthropic", envelope.Stream, presetIngress.Cleanup.RequestID, presetIngress.Cleanup.Output)
|
|
return
|
|
}
|
|
if presetIngress.terminalReady() {
|
|
_ = s.writeHotPathTerminal(w, r, dispatch, "anthropic", envelope.Stream, poolReq.Run.Metadata["iop_logical_request_id"], *presetIngress.Terminal)
|
|
return
|
|
}
|
|
result, err := s.service.SubmitProviderPool(r.Context(), poolReq)
|
|
if err != nil {
|
|
s.writeAnthropicDispatchError(w, err)
|
|
return
|
|
}
|
|
if presetHotPathEnabled(dispatch) {
|
|
presetCodec := hotPathAnthropicCodecFromRequest(r)
|
|
if presetCodec == nil {
|
|
s.terminalPresetRequest(poolReq.Run.Metadata["iop_logical_request_id"], s.edgeIDValue())
|
|
writeAnthropicError(w, http.StatusInternalServerError, "api_error", "Anthropic outer codec is unavailable")
|
|
return
|
|
}
|
|
_, collected, turnErr := presetCodec.runInitialPresetTurn(s, w, r, dispatch, poolReq.Run.Metadata, result)
|
|
if !collected {
|
|
s.terminalPresetRequest(poolReq.Run.Metadata["iop_logical_request_id"], s.edgeIDValue())
|
|
disposition, ok := hotPathDispositionFromError(turnErr)
|
|
if !ok {
|
|
disposition = hotPathTerminalDisposition{
|
|
Kind: hotPathDispositionForError(turnErr), Cause: turnErr.Error(), Source: "selector_collection",
|
|
}
|
|
}
|
|
_ = presetCodec.writeDisposition(
|
|
disposition, httpStatusForRunError(turnErr), "api_error", turnErr.Error(),
|
|
)
|
|
return
|
|
}
|
|
return
|
|
}
|
|
if result == nil || result.Tunnel == nil || result.Path != edgeservice.ProviderPoolPathTunnel {
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "selected provider did not return a tunnel")
|
|
return
|
|
}
|
|
defer result.Tunnel.Close()
|
|
|
|
profileOperation := result.DispatchInfo.ProfileOperation
|
|
if profileOperation == "" {
|
|
switch result.DispatchInfo.ProfileDriver {
|
|
case string(config.ProtocolDriverAnthropicMessages):
|
|
profileOperation = string(config.OperationMessages)
|
|
case string(config.ProtocolDriverOpenAIChat):
|
|
profileOperation = string(config.OperationChatCompletions)
|
|
case string(config.ProtocolDriverOpenAIResponses):
|
|
profileOperation = string(config.OperationResponses)
|
|
}
|
|
}
|
|
switch profileOperation {
|
|
case string(config.OperationMessages):
|
|
publicModelID := ""
|
|
if dispatch.IsPreset {
|
|
publicModelID = dispatch.ExternalModelID
|
|
}
|
|
s.writeAnthropicNativeTunnelResponse(w, r, result.Tunnel, publicModelID)
|
|
case string(config.OperationChatCompletions):
|
|
s.writeAnthropicChatBridgeResponse(w, r, result.Tunnel, envelope)
|
|
case string(config.OperationResponses):
|
|
s.writeAnthropicResponsesBridgeResponse(w, r, result.Tunnel, envelope)
|
|
default:
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "selected provider returned an unsupported protocol driver")
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleAnthropicSingleRequestStream(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
capability singleRequestService,
|
|
dispatch routeDispatch,
|
|
body []byte,
|
|
) {
|
|
requestID, err := newLogicalRequestRandomID()
|
|
if err != nil {
|
|
writeAnthropicError(w, http.StatusServiceUnavailable, "api_error", "single-request execution is unavailable")
|
|
return
|
|
}
|
|
requestID = "req_" + requestID
|
|
stream, err := newSingleRequestAnthropicStream(w, requestID, dispatch.SingleRequest.PublicModel)
|
|
if err != nil {
|
|
writeAnthropicError(w, http.StatusInternalServerError, "api_error", "single-request streaming is unavailable")
|
|
return
|
|
}
|
|
stream.setTerminalRejectionObserver(s.observeAnthropicSingleRequestTerminalRejection)
|
|
execution, err := capability.StartSingleRequest(r.Context(), edgeservice.SingleRequestRequest{
|
|
RequestID: requestID,
|
|
Binding: dispatch.SingleRequest.Clone(),
|
|
Prompt: string(append([]byte(nil), body...)),
|
|
})
|
|
if err != nil || execution == nil {
|
|
if errors.Is(err, edgeservice.ErrSingleRequestExecutorUnavailable) {
|
|
writeAnthropicSingleRequestUnavailable(w)
|
|
return
|
|
}
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "single-request execution could not be started")
|
|
return
|
|
}
|
|
defer execution.Cancel()
|
|
_ = pumpSingleRequestAnthropicStream(r.Context(), execution, stream, newWallClockSingleRequestAnthropicTicker)
|
|
}
|
|
|
|
// handleAnthropicSingleRequest keeps the HTTP adapter thin: the service owns
|
|
// the state machine and supplies only a final, caller-safe result. The adapter
|
|
// commits one buffered Anthropic terminal, then acknowledges whether that
|
|
// terminal write succeeded. Internal progress and executor errors are never
|
|
// projected into the caller response.
|
|
func (s *Server) handleAnthropicSingleRequest(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
capability singleRequestService,
|
|
dispatch routeDispatch,
|
|
body []byte,
|
|
) {
|
|
requestID, err := newLogicalRequestRandomID()
|
|
if err != nil {
|
|
writeAnthropicError(w, http.StatusServiceUnavailable, "api_error", "single-request execution is unavailable")
|
|
return
|
|
}
|
|
requestID = "req_" + requestID
|
|
execution, err := capability.StartSingleRequest(r.Context(), edgeservice.SingleRequestRequest{
|
|
RequestID: requestID,
|
|
Binding: dispatch.SingleRequest.Clone(),
|
|
Prompt: string(append([]byte(nil), body...)),
|
|
})
|
|
if err != nil || execution == nil {
|
|
if errors.Is(err, edgeservice.ErrSingleRequestExecutorUnavailable) {
|
|
writeAnthropicSingleRequestUnavailable(w)
|
|
return
|
|
}
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "single-request execution could not be started")
|
|
return
|
|
}
|
|
defer execution.Cancel()
|
|
|
|
for {
|
|
select {
|
|
case <-r.Context().Done():
|
|
execution.Cancel()
|
|
return
|
|
case progress, ok := <-execution.Progress():
|
|
if !ok {
|
|
if r.Context().Err() != nil || execution.State() == edgeservice.SingleRequestStateCancelled {
|
|
return
|
|
}
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "single-request execution failed")
|
|
return
|
|
}
|
|
switch progress.Stage {
|
|
case edgeservice.SingleRequestStateFinalizing:
|
|
if progress.Result == nil {
|
|
_ = execution.AcknowledgeTerminal(false)
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "single-request execution failed")
|
|
return
|
|
}
|
|
writeErr := writeAnthropicSingleRequestTerminal(w, requestID, dispatch.SingleRequest.PublicModel, *progress.Result)
|
|
_ = execution.AcknowledgeTerminal(writeErr == nil)
|
|
return
|
|
case edgeservice.SingleRequestStateFailed:
|
|
disposition := singleRequestProgressTerminal(progress, edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider})
|
|
s.observeAnthropicSingleRequestTerminalRejection(disposition)
|
|
writeAnthropicSingleRequestError(w, disposition)
|
|
return
|
|
case edgeservice.SingleRequestStateCancelled:
|
|
// Cancelled is reserved for caller cancellation/disconnect and is
|
|
// therefore silent even when the HTTP context races state delivery.
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Server) observeAnthropicSingleRequestTerminalRejection(disposition edgeservice.SingleRequestTerminalDisposition) {
|
|
policy := singleRequestAnthropicPolicy(disposition)
|
|
if s == nil || s.logger == nil || policy.silent || !policy.errorTerminal {
|
|
return
|
|
}
|
|
s.logger.Info(
|
|
anthropicSingleRequestTerminalRejectionLogMessage,
|
|
zap.String("surface", "messages"),
|
|
zap.String("terminal_kind", string(disposition.Kind)),
|
|
zap.String("terminal_error_class", string(disposition.ErrorClass)),
|
|
zap.Int("http_status", policy.status),
|
|
)
|
|
}
|
|
|
|
func writeAnthropicSingleRequestUnavailable(w http.ResponseWriter) {
|
|
writeAnthropicError(w, http.StatusServiceUnavailable, "api_error", "single-request execution is unavailable")
|
|
}
|
|
|
|
func singleRequestProgressTerminal(progress edgeservice.SingleRequestProgress, fallback edgeservice.SingleRequestTerminalDisposition) edgeservice.SingleRequestTerminalDisposition {
|
|
if progress.Terminal != nil && progress.Terminal.Validate() == nil {
|
|
return *progress.Terminal
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func writeAnthropicSingleRequestError(w http.ResponseWriter, disposition edgeservice.SingleRequestTerminalDisposition) {
|
|
policy := singleRequestAnthropicPolicy(disposition)
|
|
if policy.silent {
|
|
return
|
|
}
|
|
if !policy.errorTerminal {
|
|
policy = singleRequestAnthropicPolicy(edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider})
|
|
}
|
|
writeAnthropicError(w, policy.status, policy.errorType, policy.message)
|
|
}
|
|
|
|
// writeAnthropicSingleRequestTerminal encodes before committing headers and
|
|
// reports short/failed writes so the service never records successful terminal
|
|
// acknowledgement merely because response construction succeeded.
|
|
func writeAnthropicSingleRequestTerminal(w http.ResponseWriter, requestID, publicModel string, result edgeservice.SingleRequestResult) error {
|
|
policy := singleRequestAnthropicPolicy(result.Terminal)
|
|
if policy.silent || policy.errorTerminal || policy.stopReason == "" {
|
|
return errors.New("single-request result has no Anthropic message terminal")
|
|
}
|
|
content := []map[string]any{{"type": "text", "text": result.Output}}
|
|
if result.Terminal.Kind == edgeservice.SingleRequestTerminalLength {
|
|
// A stage/output limit never exposes a private partial stage payload.
|
|
content = []map[string]any{}
|
|
}
|
|
response := anthropicMessageResponse{
|
|
ID: "msg_iop_" + strings.TrimPrefix(requestID, "req_"),
|
|
Type: "message",
|
|
Role: "assistant",
|
|
Model: publicModel,
|
|
Content: content,
|
|
StopReason: &policy.stopReason,
|
|
Usage: anthropicUsage{},
|
|
}
|
|
encoded, err := json.Marshal(response)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
encoded = append(encoded, '\n')
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(policy.status)
|
|
n, err := w.Write(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != len(encoded) {
|
|
return io.ErrShortWrite
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) handleAnthropicCountTokens(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
writeAnthropicError(w, http.StatusMethodNotAllowed, "invalid_request_error", "method not allowed")
|
|
return
|
|
}
|
|
defer r.Body.Close()
|
|
if err := validateAnthropicHeaders(r); err != nil {
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
body, err := readOpenAIIngressBody(w, r, s.maxIngressSnapshotBytes())
|
|
if err != nil {
|
|
writeAnthropicIngressError(w, err)
|
|
return
|
|
}
|
|
envelope, err := decodeAnthropicEnvelope(body)
|
|
if err != nil {
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
dispatch, err := s.resolveRouteDispatchForPrincipal(r.Context(), envelope.Model)
|
|
if err != nil || !dispatch.ProviderPool {
|
|
s.writeAnthropicRouteError(w, err)
|
|
return
|
|
}
|
|
|
|
if entry := s.findProviderPoolEntry(dispatch.effectiveModelGroupKey(envelope.Model)); entry != nil && entry.TokenCounter != nil {
|
|
req, decodeErr := decodeAnthropicMessageRequest(body, false)
|
|
if decodeErr != nil {
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", decodeErr.Error())
|
|
return
|
|
}
|
|
count, countErr := countAnthropicInputTokens(req, *entry.TokenCounter)
|
|
if countErr != nil {
|
|
writeAnthropicError(w, http.StatusBadRequest, "not_supported_error", countErr.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, anthropicCountTokensResponse{InputTokens: count})
|
|
return
|
|
}
|
|
|
|
poolReq, _, err := s.anthropicPoolRequest(r, dispatch, envelope, body, config.OperationCountTokens, providerRequestRequirements{})
|
|
if err != nil {
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
result, err := s.service.SubmitProviderPool(r.Context(), poolReq)
|
|
if err != nil {
|
|
s.writeAnthropicDispatchError(w, err)
|
|
return
|
|
}
|
|
if result == nil || result.Tunnel == nil || result.Path != edgeservice.ProviderPoolPathTunnel ||
|
|
result.DispatchInfo.ProfileDriver != string(config.ProtocolDriverAnthropicMessages) {
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "selected provider did not return a native count-tokens tunnel")
|
|
return
|
|
}
|
|
defer result.Tunnel.Close()
|
|
s.writeAnthropicNativeTunnelResponse(w, r, result.Tunnel, "")
|
|
}
|
|
|
|
func (s *Server) anthropicPoolRequest(
|
|
r *http.Request,
|
|
dispatch routeDispatch,
|
|
envelope anthropicRequestEnvelope,
|
|
body []byte,
|
|
operation config.ProtocolOperation,
|
|
requirements providerRequestRequirements,
|
|
) (edgeservice.ProviderPoolDispatchRequest, presetIngressResult, error) {
|
|
metadata := principalMetadata(r.Context())
|
|
if metadata == nil {
|
|
metadata = make(map[string]string)
|
|
}
|
|
metadata["anthropic_model"] = envelope.Model
|
|
metadata["anthropic_stream"] = fmt.Sprintf("%t", envelope.Stream)
|
|
applyTrustedManagedBindingMetadata(metadata, dispatch)
|
|
if dispatch.IsPreset && operation == config.OperationMessages {
|
|
presetIngress, err := s.joinPresetAnthropicIngress(r, dispatch, body, metadata)
|
|
if err != nil {
|
|
return edgeservice.ProviderPoolDispatchRequest{}, presetIngressResult{}, err
|
|
}
|
|
if presetIngress.localStageEligible() || presetIngress.lightStageContinuation() || presetIngress.cleanupIssued() || presetIngress.terminalReady() {
|
|
return edgeservice.ProviderPoolDispatchRequest{
|
|
Run: edgeservice.SubmitRunRequest{Metadata: metadata},
|
|
}, presetIngress, nil
|
|
}
|
|
// Resume-selector and ordinary continuations both construct the same
|
|
// trusted selector request; only local eligibility bypasses the pool.
|
|
return s.buildAnthropicPoolRequest(r, dispatch, envelope, body, operation, requirements, metadata, presetIngress)
|
|
}
|
|
return s.buildAnthropicPoolRequest(r, dispatch, envelope, body, operation, requirements, metadata, presetIngressResult{})
|
|
}
|
|
|
|
func (s *Server) buildAnthropicPoolRequest(
|
|
r *http.Request,
|
|
dispatch routeDispatch,
|
|
envelope anthropicRequestEnvelope,
|
|
body []byte,
|
|
operation config.ProtocolOperation,
|
|
requirements providerRequestRequirements,
|
|
metadata map[string]string,
|
|
presetIngress presetIngressResult,
|
|
) (edgeservice.ProviderPoolDispatchRequest, presetIngressResult, error) {
|
|
estimate := estimateInputTokensBytes(body, metadata, nil, nil)
|
|
contextClass := classifyContext(estimate, s.longContextThreshold())
|
|
modelGroupKey := dispatch.effectiveModelGroupKey(envelope.Model)
|
|
if dispatch.IsPreset && operation == config.OperationMessages {
|
|
modelGroupKey = presetSelectorModelGroupKey(dispatch, envelope.Model)
|
|
}
|
|
|
|
poolReq := edgeservice.ProviderPoolDispatchRequest{
|
|
Run: edgeservice.SubmitRunRequest{
|
|
NodeRef: dispatch.NodeRef, ModelGroupKey: modelGroupKey,
|
|
ProviderID: dispatch.ProviderID, UsageAttribution: dispatch.UsageAttribution,
|
|
SessionID: dispatch.SessionID, TimeoutSec: dispatch.TimeoutSec,
|
|
MaxQueue: dispatch.MaxQueue, QueueTimeoutMS: dispatch.QueueTimeoutMS,
|
|
Metadata: metadata, EstimatedInputTokens: estimate, ContextClass: contextClass,
|
|
ProviderPool: true,
|
|
},
|
|
Tunnel: edgeservice.SubmitProviderTunnelRequest{
|
|
CredentialBinding: dispatch.credentialBinding(),
|
|
ModelGroupKey: modelGroupKey, ProviderID: dispatch.ProviderID,
|
|
UsageAttribution: dispatch.UsageAttribution, SessionID: dispatch.SessionID,
|
|
Method: http.MethodPost, Path: r.URL.Path, Stream: envelope.Stream,
|
|
TimeoutSec: dispatch.TimeoutSec, MaxQueue: dispatch.MaxQueue,
|
|
QueueTimeoutMS: dispatch.QueueTimeoutMS, Metadata: metadata,
|
|
EstimatedInputTokens: estimate, ContextClass: contextClass, ProviderPool: true,
|
|
},
|
|
}
|
|
poolReq.AcceptCandidate = anthropicCandidatePredicate(operation, requirements)
|
|
if dispatch.Managed {
|
|
poolReq.AcceptCandidate = composeCandidatePredicates(poolReq.AcceptCandidate, dispatch.CandidatePredicate())
|
|
}
|
|
poolReq.PrepareProtocolTunnel = func(tunnelReq edgeservice.SubmitProviderTunnelRequest, selected edgeservice.ProviderPoolCandidate) (edgeservice.SubmitProviderTunnelRequest, error) {
|
|
if selected.ProtocolProfile == nil {
|
|
return tunnelReq, newAnthropicClientError("not_supported_error", fmt.Errorf("selected provider has no concrete protocol profile"))
|
|
}
|
|
profile := selected.ProtocolProfile.Clone()
|
|
plan, err := selectProviderOperation(profile, operation, requirements)
|
|
if err != nil {
|
|
return tunnelReq, newAnthropicClientError("not_supported_error", err)
|
|
}
|
|
headers, err := s.anthropicUpstreamHeaders(r, profile, profile.Driver == config.ProtocolDriverAnthropicMessages)
|
|
if err != nil {
|
|
return tunnelReq, newAnthropicClientError("invalid_request_error", err)
|
|
}
|
|
tunnelReq.Headers = headers
|
|
switch plan.Operation {
|
|
case config.OperationMessages, config.OperationCountTokens:
|
|
tunnelReq.Operation = string(plan.Operation)
|
|
tunnelReq.BuildBody = func(target string) ([]byte, error) {
|
|
return rewriteResponsesModel(body, target)
|
|
}
|
|
case config.OperationChatCompletions:
|
|
if operation != config.OperationMessages {
|
|
return tunnelReq, newAnthropicClientError("not_supported_error", fmt.Errorf("selected Chat profile has no native count-tokens operation"))
|
|
}
|
|
if err := validateAnthropicHeaders(r); err != nil {
|
|
return tunnelReq, newAnthropicClientError("invalid_request_error", err)
|
|
}
|
|
bridged, _, err := prepareAnthropicChatBridge(body, selected.ActualModel, profile)
|
|
if err != nil {
|
|
return tunnelReq, newAnthropicClientError("invalid_request_error", err)
|
|
}
|
|
tunnelReq.Operation = string(config.OperationChatCompletions)
|
|
tunnelReq.Body = bridged
|
|
tunnelReq.BuildBody = nil
|
|
case config.OperationResponses:
|
|
if operation != config.OperationMessages {
|
|
return tunnelReq, newAnthropicClientError("not_supported_error", fmt.Errorf("selected Responses profile has no native count-tokens operation"))
|
|
}
|
|
if err := validateAnthropicHeaders(r); err != nil {
|
|
return tunnelReq, newAnthropicClientError("invalid_request_error", err)
|
|
}
|
|
bridged, _, err := prepareAnthropicResponsesBridge(body, selected.ActualModel, profile, plan)
|
|
if err != nil {
|
|
return tunnelReq, newAnthropicClientError("invalid_request_error", err)
|
|
}
|
|
tunnelReq.Operation = string(config.OperationResponses)
|
|
tunnelReq.Path = "/v1/responses"
|
|
tunnelReq.Body = bridged
|
|
tunnelReq.BuildBody = nil
|
|
default:
|
|
return tunnelReq, newAnthropicClientError("not_supported_error", fmt.Errorf("unsupported protocol driver %q", profile.Driver))
|
|
}
|
|
return tunnelReq, nil
|
|
}
|
|
return poolReq, presetIngress, nil
|
|
}
|
|
|
|
func anthropicCandidatePredicate(operation config.ProtocolOperation, requirements providerRequestRequirements) edgeservice.ProviderPoolCandidatePredicate {
|
|
return func(candidate edgeservice.ProviderPoolCandidate) bool {
|
|
profile := candidate.ProtocolProfile
|
|
if profile == nil || candidate.ExecutionPath != string(edgeservice.ProviderPoolPathTunnel) {
|
|
return false
|
|
}
|
|
switch operation {
|
|
case config.OperationCountTokens:
|
|
return profile.Driver == config.ProtocolDriverAnthropicMessages &&
|
|
profile.HasCapability("count_tokens") && profileHasOperation(*profile, config.OperationCountTokens)
|
|
case config.OperationMessages:
|
|
_, err := selectProviderOperation(*profile, operation, requirements)
|
|
return err == nil
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
func profileHasOperation(profile config.ConcreteProtocolProfile, operation config.ProtocolOperation) bool {
|
|
_, ok := profile.Operations[string(operation)]
|
|
return ok
|
|
}
|
|
|
|
func (s *Server) anthropicUpstreamHeaders(r *http.Request, profile config.ConcreteProtocolProfile, native bool) (map[string]string, error) {
|
|
headers := map[string]string{"Content-Type": "application/json"}
|
|
if accept := strings.TrimSpace(r.Header.Get("Accept")); accept != "" {
|
|
headers["Accept"] = accept
|
|
}
|
|
if native {
|
|
headers[anthropicVersionHeader] = strings.TrimSpace(r.Header.Get(anthropicVersionHeader))
|
|
betas, err := anthropicBetaValues(r.Header.Values(anthropicBetaHeader))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(betas) > 0 {
|
|
headers[anthropicBetaHeader] = strings.Join(betas, ",")
|
|
}
|
|
}
|
|
credentialHeaders, err := s.providerTunnelAuthHeaders(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, value := range credentialHeaders {
|
|
credential := strings.TrimSpace(value)
|
|
if fields := strings.Fields(credential); len(fields) > 1 {
|
|
credential = strings.Join(fields[1:], " ")
|
|
}
|
|
if scheme := strings.TrimSpace(profile.Auth.Scheme); scheme != "" {
|
|
credential = scheme + " " + credential
|
|
}
|
|
if credential != "" {
|
|
headers[http.CanonicalHeaderKey(profile.Auth.Header)] = credential
|
|
}
|
|
break
|
|
}
|
|
return headers, nil
|
|
}
|
|
|
|
func (s *Server) writeAnthropicDispatchError(w http.ResponseWriter, err error) {
|
|
var clientErr *anthropicClientError
|
|
if errors.As(err, &clientErr) {
|
|
writeAnthropicError(w, http.StatusBadRequest, clientErr.errorType, clientErr.message)
|
|
return
|
|
}
|
|
if errors.Is(err, edgeservice.ErrProviderPoolCandidateRejected) {
|
|
writeAnthropicError(w, http.StatusBadRequest, "not_supported_error", "no provider profile supports the requested Messages operation")
|
|
return
|
|
}
|
|
if isProviderCredentialClientError(err) {
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", providerCredentialClientMessage(err))
|
|
return
|
|
}
|
|
writeAnthropicError(w, http.StatusBadGateway, "api_error", "provider dispatch failed")
|
|
}
|
|
|
|
func writeAnthropicIngressError(w http.ResponseWriter, err error) {
|
|
if errors.Is(err, errOpenAIIngressTooLarge) {
|
|
writeAnthropicError(w, http.StatusRequestEntityTooLarge, "invalid_request_error", "request body is too large")
|
|
return
|
|
}
|
|
writeAnthropicError(w, http.StatusBadRequest, "invalid_request_error", "request body could not be read")
|
|
}
|
|
|
|
func anthropicIngressErrorStatus(err error) int {
|
|
if errors.Is(err, errOpenAIIngressTooLarge) {
|
|
return http.StatusRequestEntityTooLarge
|
|
}
|
|
return http.StatusBadRequest
|
|
}
|
|
|
|
func countAnthropicInputTokens(req anthropicMessageRequest, counter config.TokenCounterConf) (int, error) {
|
|
payload := map[string]any{"messages": req.Messages}
|
|
if len(req.System) > 0 {
|
|
payload["system"] = json.RawMessage(req.System)
|
|
}
|
|
if len(req.Tools) > 0 {
|
|
payload["tools"] = req.Tools
|
|
}
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("encode token counter input: %w", err)
|
|
}
|
|
runes := utf8.RuneCount(encoded)
|
|
var count int
|
|
switch counter.Mode {
|
|
case config.TokenCounterDeterministic:
|
|
count = (runes + 3) / 4
|
|
case config.TokenCounterEstimate:
|
|
count = (runes*counter.Per1kInput + 999) / 1000
|
|
default:
|
|
return 0, fmt.Errorf("unsupported token counter mode %q", counter.Mode)
|
|
}
|
|
if count < 1 {
|
|
count = 1
|
|
}
|
|
return count, nil
|
|
}
|