toki
f9442edfef
장시간 무응답 attempt를 안전하게 fence하고 provider health와 분리 관측해야 중복 출력 없이 기존 recovery budget으로 재실행할 수 있다.
260 lines
8.6 KiB
Go
260 lines
8.6 KiB
Go
package node
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
"google.golang.org/protobuf/types/known/structpb"
|
|
|
|
runtime "iop/packages/go/execution"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
func TestRunRequestFromProtoPreservesWireFields(t *testing.T) {
|
|
policy, err := structpb.NewStruct(map[string]any{"allow": true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
input, err := structpb.NewStruct(map[string]any{"prompt": "hello"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wire := &iop.RunRequest{
|
|
RunId: "run-1",
|
|
Adapter: "ollama",
|
|
Target: "model",
|
|
SessionId: "correlation-1",
|
|
Background: true,
|
|
Policy: policy,
|
|
Input: input,
|
|
TimeoutSec: 30,
|
|
Metadata: map[string]string{"source": "test"},
|
|
}
|
|
|
|
got := runRequestFromProto(wire)
|
|
if got.RunID != wire.RunId || got.Adapter != wire.Adapter || got.Target != wire.Target ||
|
|
got.SessionID != wire.SessionId || got.Background != wire.Background ||
|
|
got.TimeoutSec != int(wire.TimeoutSec) {
|
|
t.Fatalf("runRequestFromProto() = %#v", got)
|
|
}
|
|
if !reflect.DeepEqual(got.Policy, policy.AsMap()) || !reflect.DeepEqual(got.Input, input.AsMap()) ||
|
|
!reflect.DeepEqual(got.Metadata, wire.Metadata) {
|
|
t.Fatalf("mapped maps = policy %#v input %#v metadata %#v", got.Policy, got.Input, got.Metadata)
|
|
}
|
|
}
|
|
|
|
func TestRunEventToProtoPreservesLegacyValues(t *testing.T) {
|
|
timestamp := time.Unix(0, 1234)
|
|
event := runtime.RuntimeEvent{
|
|
RunID: "run-1",
|
|
Type: runtime.EventTypeError,
|
|
Error: "legacy error",
|
|
Failure: &runtime.Failure{Code: runtime.FailureCodeProvider, Message: "typed error"},
|
|
Metadata: map[string]string{"key": "value"},
|
|
Usage: &runtime.UsageStats{
|
|
InputTokens: 1,
|
|
OutputTokens: 2,
|
|
ReasoningTokens: 3,
|
|
CachedInputTokens: 4,
|
|
},
|
|
Timestamp: timestamp,
|
|
}
|
|
|
|
got := runEventToProto(event, "node-1", "session-1", true)
|
|
if got.GetRunId() != "run-1" || got.GetType() != "error" || got.GetError() != "legacy error" ||
|
|
got.GetNodeId() != "node-1" || got.GetSessionId() != "session-1" || !got.GetBackground() ||
|
|
got.GetTimestamp() != timestamp.UnixNano() {
|
|
t.Fatalf("runEventToProto() = %#v", got)
|
|
}
|
|
if got.GetUsage().GetInputTokens() != 1 || got.GetUsage().GetOutputTokens() != 2 ||
|
|
got.GetUsage().GetReasoningTokens() != 3 || got.GetUsage().GetCachedInputTokens() != 4 {
|
|
t.Fatalf("usage = %#v", got.GetUsage())
|
|
}
|
|
if !reflect.DeepEqual(got.GetMetadata(), event.Metadata) {
|
|
t.Fatalf("metadata = %#v, want %#v", got.GetMetadata(), event.Metadata)
|
|
}
|
|
}
|
|
|
|
func TestRunEventToProtoUsesTypedFailureMessageAsFallback(t *testing.T) {
|
|
got := runEventToProto(runtime.RuntimeEvent{
|
|
RunID: "run-1",
|
|
Type: runtime.EventTypeError,
|
|
Failure: &runtime.Failure{Code: runtime.FailureCodeUnavailable, Message: "unavailable"},
|
|
Timestamp: time.Unix(0, 1),
|
|
}, "node-1", "default", false)
|
|
if got.GetError() != "unavailable" {
|
|
t.Fatalf("error = %q, want unavailable", got.GetError())
|
|
}
|
|
}
|
|
|
|
func TestRunRequestFromProtoLeavesRawStallTimeoutForHandlerValidation(t *testing.T) {
|
|
// The handler, not the protobuf mapper, validates and resolves the raw value.
|
|
req := &iop.RunRequest{
|
|
RunId: "r1",
|
|
Adapter: "ollama",
|
|
}
|
|
runtimeReq := runRequestFromProto(req)
|
|
if runtimeReq.ResponseStallTimeoutMS != 0 {
|
|
t.Errorf("zero wire must remain raw before handler validation, got %d", runtimeReq.ResponseStallTimeoutMS)
|
|
}
|
|
|
|
// Positive override → passes through.
|
|
req2 := &iop.RunRequest{
|
|
RunId: "r2",
|
|
Adapter: "ollama",
|
|
ResponseStallTimeoutMs: 60000,
|
|
}
|
|
runtimeReq2 := runRequestFromProto(req2)
|
|
if runtimeReq2.ResponseStallTimeoutMS != 0 {
|
|
t.Errorf("positive wire value must remain raw before handler validation, got %d", runtimeReq2.ResponseStallTimeoutMS)
|
|
}
|
|
}
|
|
|
|
func TestValidateStallTimeoutOnWire(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
raw int64
|
|
want int64
|
|
bad bool
|
|
}{
|
|
{name: "zero defaults", raw: 0, want: runtime.DefaultResponseStallTimeoutMS},
|
|
{name: "positive preserved", raw: 60000, want: 60000},
|
|
{name: "negative rejected", raw: -1, bad: true},
|
|
{name: "overflow rejected", raw: 99999999999999, bad: true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := ValidateStallTimeoutOnWire(tc.raw)
|
|
if (err != nil) != tc.bad {
|
|
t.Fatalf("ValidateStallTimeoutOnWire(%d) error = %v, want bad=%t", tc.raw, err, tc.bad)
|
|
}
|
|
if !tc.bad && got != tc.want {
|
|
t.Errorf("ValidateStallTimeoutOnWire(%d) = %d, want %d", tc.raw, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResponseStallTimeoutWireRoundTrip(t *testing.T) {
|
|
values := []int64{0, 60000, -1, (int64(1) << 62) / int64(time.Millisecond)}
|
|
for _, value := range values {
|
|
t.Run("run", func(t *testing.T) {
|
|
original := &iop.RunRequest{ResponseStallTimeoutMs: value}
|
|
encoded, err := proto.Marshal(original)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
decoded := &iop.RunRequest{}
|
|
if err := proto.Unmarshal(encoded, decoded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decoded.GetResponseStallTimeoutMs() != value {
|
|
t.Fatalf("RunRequest round trip = %d, want %d", decoded.GetResponseStallTimeoutMs(), value)
|
|
}
|
|
})
|
|
t.Run("tunnel", func(t *testing.T) {
|
|
original := &iop.ProviderTunnelRequest{ResponseStallTimeoutMs: value}
|
|
encoded, err := proto.Marshal(original)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
decoded := &iop.ProviderTunnelRequest{}
|
|
if err := proto.Unmarshal(encoded, decoded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decoded.GetResponseStallTimeoutMs() != value {
|
|
t.Fatalf("ProviderTunnelRequest round trip = %d, want %d", decoded.GetResponseStallTimeoutMs(), value)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRuntimeEventToProtoPreservesTypedFailure(t *testing.T) {
|
|
t.Run("stalled failure populated with allowlisted metadata", func(t *testing.T) {
|
|
inputMeta := map[string]string{
|
|
"failure_code": "response_stalled",
|
|
"provider_health": "available",
|
|
"liveness_classification": "request_stalled",
|
|
"idle_duration_ms": "5000",
|
|
"run_id": "run-1",
|
|
"attempt_id": "run-1",
|
|
"attempt_fence": "confirmed",
|
|
"adapter": "ollama",
|
|
"target": "llama3",
|
|
"health_observation_seq": "1",
|
|
"recovery_eligible": "true",
|
|
"secret_key": "sensitive",
|
|
}
|
|
event := runtime.RuntimeEvent{
|
|
RunID: eventTypeStalledRunID(),
|
|
Type: runtime.EventTypeError,
|
|
Error: "provider response stalled",
|
|
Failure: &runtime.Failure{
|
|
Code: runtime.FailureCodeResponseStalled,
|
|
Message: "provider response stalled",
|
|
Retryable: true,
|
|
Metadata: inputMeta,
|
|
},
|
|
}
|
|
|
|
wire := runEventToProto(event, "node-1", "session-1", false)
|
|
|
|
if wire.GetFailure() == nil {
|
|
t.Fatal("expected non-nil wire.Failure")
|
|
}
|
|
if wire.GetFailure().GetCode() != "response_stalled" {
|
|
t.Fatalf("code = %q, want response_stalled", wire.GetFailure().GetCode())
|
|
}
|
|
if wire.GetFailure().GetMessage() != "provider response stalled" {
|
|
t.Fatalf("message = %q", wire.GetFailure().GetMessage())
|
|
}
|
|
if !wire.GetFailure().GetRetryable() {
|
|
t.Fatal("expected retryable = true")
|
|
}
|
|
|
|
meta := wire.GetFailure().GetMetadata()
|
|
if meta["provider_health"] != "available" || meta["liveness_classification"] != "request_stalled" || meta["health_observation_seq"] != "1" {
|
|
t.Fatalf("allowlisted metadata missing or invalid = %#v", meta)
|
|
}
|
|
if meta["recovery_eligible"] != "" || meta["secret_key"] != "" {
|
|
t.Fatalf("non-allowlisted metadata present in wire failure: %#v", meta)
|
|
}
|
|
|
|
// Verify defensive cloning: mutating input map must not alter wire failure metadata
|
|
inputMeta["attempt_fence"] = "mutated"
|
|
if meta["attempt_fence"] != "confirmed" {
|
|
t.Fatal("wire failure metadata shared mutable alias with input metadata")
|
|
}
|
|
})
|
|
|
|
t.Run("non-stalled failure leaves wire failure nil", func(t *testing.T) {
|
|
event := runtime.RuntimeEvent{
|
|
RunID: "run-2",
|
|
Type: runtime.EventTypeError,
|
|
Error: "cancelled error",
|
|
Failure: &runtime.Failure{Code: runtime.FailureCodeCancelled, Message: "cancelled error"},
|
|
}
|
|
wire := runEventToProto(event, "node-1", "session-1", false)
|
|
if wire.GetFailure() != nil {
|
|
t.Fatalf("expected nil wire.Failure for non-stalled code, got %#v", wire.GetFailure())
|
|
}
|
|
if wire.GetError() != "cancelled error" {
|
|
t.Fatalf("error string = %q, want cancelled error", wire.GetError())
|
|
}
|
|
})
|
|
|
|
t.Run("nil failure leaves wire failure nil", func(t *testing.T) {
|
|
event := runtime.RuntimeEvent{
|
|
RunID: "run-3",
|
|
Type: runtime.EventTypeComplete,
|
|
}
|
|
wire := runEventToProto(event, "node-1", "session-1", false)
|
|
if wire.GetFailure() != nil {
|
|
t.Fatalf("expected nil wire.Failure for nil failure, got %#v", wire.GetFailure())
|
|
}
|
|
})
|
|
}
|
|
|
|
func eventTypeStalledRunID() string { return "run-1" }
|