toki
fef1f7a9dc
Node의 provider progress 기반 stall timeout, watchdog fencing과 bounded health probe evidence를 실행 경로에 반영한다. Edge-Node 계약과 구현 스펙, 테스트 및 Milestone 완료 evidence를 현재 상태와 맞춘다.
402 lines
15 KiB
Go
402 lines
15 KiB
Go
package execution_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"iop/packages/go/execution"
|
|
)
|
|
|
|
func TestResolveStallTimeoutMS(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
ms int64
|
|
want int64
|
|
wantErr bool
|
|
}{
|
|
{"zero maps to default", 0, execution.DefaultResponseStallTimeoutMS, false},
|
|
{"default passes through", execution.DefaultResponseStallTimeoutMS, execution.DefaultResponseStallTimeoutMS, false},
|
|
{"custom positive passes through", 60000, 60000, false},
|
|
{"small positive passes through", 1, 1, false},
|
|
{"exact safe boundary passes through", math.MaxInt64 / int64(time.Millisecond), math.MaxInt64 / int64(time.Millisecond), false},
|
|
{"first overflowing millisecond rejected", math.MaxInt64/int64(time.Millisecond) + 1, 0, true},
|
|
{"negative rejected", -1, 0, true},
|
|
{"overflow rejected", execution.DefaultResponseStallTimeoutMS * 100000000, 0, true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := execution.ResolveStallTimeoutMS(tc.ms)
|
|
if (err != nil) != tc.wantErr {
|
|
t.Fatalf("ResolveStallTimeoutMS(%d) error = %v, want error=%t", tc.ms, err, tc.wantErr)
|
|
}
|
|
if !tc.wantErr && got != tc.want {
|
|
t.Errorf("ResolveStallTimeoutMS(%d) = %d, want %d", tc.ms, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateStallTimeoutMS(t *testing.T) {
|
|
if err := execution.ValidateStallTimeoutMS(300000); err != nil {
|
|
t.Errorf("expected nil for 300000, got %v", err)
|
|
}
|
|
if err := execution.ValidateStallTimeoutMS(1); err != nil {
|
|
t.Errorf("expected nil for 1, got %v", err)
|
|
}
|
|
if err := execution.ValidateStallTimeoutMS(0); err != nil {
|
|
t.Errorf("expected nil for 0 (use default), got %v", err)
|
|
}
|
|
if err := execution.ValidateStallTimeoutMS(-1); err == nil {
|
|
t.Error("expected error for -1")
|
|
}
|
|
if err := execution.ValidateStallTimeoutMS(execution.DefaultResponseStallTimeoutMS * 100000000); err == nil {
|
|
t.Error("expected error for overflow value")
|
|
}
|
|
}
|
|
|
|
func TestStallTimeoutValidationError(t *testing.T) {
|
|
e := &execution.StallTimeoutValidationError{Value: -1, Msg: "must be positive"}
|
|
if e.Error() != "must be positive" {
|
|
t.Errorf("Error() = %q, want 'must be positive'", e.Error())
|
|
}
|
|
e2 := &execution.StallTimeoutValidationError{Value: 0}
|
|
if e2.Error() != "invalid response_stall_timeout_ms" {
|
|
t.Errorf("Error() = %q, want 'invalid response_stall_timeout_ms'", e2.Error())
|
|
}
|
|
}
|
|
|
|
func TestClassifyRuntimeEvent(t *testing.T) {
|
|
now := time.Now()
|
|
cases := []struct {
|
|
name string
|
|
ev execution.RuntimeEvent
|
|
want execution.ProviderActivityDisposition
|
|
}{
|
|
{
|
|
name: "start event",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeStart, RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionStart,
|
|
},
|
|
{
|
|
name: "delta with text",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeDelta, Delta: "hello", RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{
|
|
name: "delta with message",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeDelta, Message: "hi", RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{
|
|
name: "reasoning_delta with text",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeReasoningDelta, Delta: "thinking...", RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{name: "delta with zero usage", ev: execution.RuntimeEvent{Type: execution.EventTypeDelta, Usage: &execution.UsageStats{}, RunID: "r1", Timestamp: now}, want: execution.DispositionProgress},
|
|
{name: "delta with token usage", ev: execution.RuntimeEvent{Type: execution.EventTypeDelta, Usage: &execution.UsageStats{OutputTokens: 1}, RunID: "r1", Timestamp: now}, want: execution.DispositionProgress},
|
|
{name: "reasoning delta with token usage", ev: execution.RuntimeEvent{Type: execution.EventTypeReasoningDelta, Usage: &execution.UsageStats{ReasoningTokens: 1}, RunID: "r1", Timestamp: now}, want: execution.DispositionProgress},
|
|
{name: "delta empty no usage", ev: execution.RuntimeEvent{Type: execution.EventTypeDelta, RunID: "r1", Timestamp: now}, want: execution.DispositionNone},
|
|
{
|
|
name: "complete with usage",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeComplete, Usage: &execution.UsageStats{OutputTokens: 10}, RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionTerminal,
|
|
},
|
|
{
|
|
name: "complete without usage",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeComplete, RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionTerminal,
|
|
},
|
|
{name: "error with payload and usage", ev: execution.RuntimeEvent{Type: execution.EventTypeError, Delta: "last", Error: "boom", Usage: &execution.UsageStats{OutputTokens: 1}, RunID: "r1", Timestamp: now}, want: execution.DispositionTerminal},
|
|
{name: "cancelled with payload and usage", ev: execution.RuntimeEvent{Type: execution.EventTypeCancelled, Message: "last", Usage: &execution.UsageStats{InputTokens: 1}, RunID: "r1", Timestamp: now}, want: execution.DispositionTerminal},
|
|
{
|
|
name: "unknown type",
|
|
ev: execution.RuntimeEvent{Type: "unknown", RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionNone,
|
|
},
|
|
{
|
|
name: "delta with terminal usage takes terminal",
|
|
ev: execution.RuntimeEvent{Type: execution.EventTypeComplete, Delta: "last", Usage: &execution.UsageStats{OutputTokens: 5}, RunID: "r1", Timestamp: now},
|
|
want: execution.DispositionTerminal,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := execution.ClassifyRuntimeEvent(tc.ev)
|
|
if got != tc.want {
|
|
t.Errorf("ClassifyRuntimeEvent: got %q, want %q", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestClassifyProviderTunnelFrame(t *testing.T) {
|
|
now := time.Now()
|
|
cases := []struct {
|
|
name string
|
|
f execution.ProviderTunnelFrame
|
|
want execution.ProviderActivityDisposition
|
|
}{
|
|
{
|
|
name: "response_start",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindResponseStart, RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{
|
|
name: "response_start with headers",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindResponseStart, Headers: map[string]string{"content-type": "text/event-stream"}, RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{
|
|
name: "body with data",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindBody, Body: []byte("hello"), RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{
|
|
name: "body empty",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindBody, RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionNone,
|
|
},
|
|
{
|
|
name: "end",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindEnd, RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionTerminal,
|
|
},
|
|
{
|
|
name: "error",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindError, Error: "provider timeout", RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionTerminal,
|
|
},
|
|
{
|
|
name: "usage with tokens",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindUsage, Usage: &execution.UsageStats{OutputTokens: 10}, RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionProgress,
|
|
},
|
|
{
|
|
name: "unknown kind",
|
|
f: execution.ProviderTunnelFrame{Kind: "bogus", RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionNone,
|
|
},
|
|
{
|
|
name: "end with body takes terminal",
|
|
f: execution.ProviderTunnelFrame{Kind: execution.ProviderTunnelFrameKindEnd, Body: []byte("final"), RunID: "r1", TunnelID: "t1", Timestamp: now},
|
|
want: execution.DispositionTerminal,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := execution.ClassifyProviderTunnelFrame(tc.f)
|
|
if got != tc.want {
|
|
t.Errorf("ClassifyProviderTunnelFrame: got %q, want %q", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// assertProbeOutcome checks both the liveness classification and the normalized
|
|
// health for a probe outcome, keeping the table-driven probe tests compact.
|
|
func assertProbeOutcome(t *testing.T, outcome execution.ProbeOutcome, wantClass execution.LivenessClassification, wantHealth execution.ProviderHealth) {
|
|
t.Helper()
|
|
if gotClass := execution.ClassifyProbeOutcome(outcome); gotClass != wantClass {
|
|
t.Errorf("ClassifyProbeOutcome: got %q, want %q", gotClass, wantClass)
|
|
}
|
|
if gotHealth := execution.NormalizeProbeOutcome(outcome); gotHealth != wantHealth {
|
|
t.Errorf("NormalizeProbeOutcome: got %q, want %q", gotHealth, wantHealth)
|
|
}
|
|
}
|
|
|
|
func TestClassifyProbeOutcomeDefinitive(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
outcome execution.ProbeOutcome
|
|
wantClass execution.LivenessClassification
|
|
wantHealth execution.ProviderHealth
|
|
}{
|
|
{
|
|
name: "matching available",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Status: execution.ProviderStatusAvailable,
|
|
},
|
|
wantClass: execution.LivenessAvailable, wantHealth: execution.RequestStalled,
|
|
},
|
|
{
|
|
name: "matching unavailable",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "ollama", ExpectedAdapter: "ollama",
|
|
Target: "m-b", ExpectedTarget: "m-b",
|
|
Status: execution.ProviderStatusUnavailable,
|
|
},
|
|
wantClass: execution.LivenessUnavailable, wantHealth: execution.ProviderUnhealthy,
|
|
},
|
|
{
|
|
name: "matching available with pinned instance",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
InstanceKey: "vllm-gpu", ExpectedInstance: "vllm-gpu",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Status: execution.ProviderStatusAvailable,
|
|
},
|
|
wantClass: execution.LivenessAvailable, wantHealth: execution.RequestStalled,
|
|
},
|
|
{
|
|
name: "pinned instance mismatch stays inconclusive",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
InstanceKey: "vllm-gpu", ExpectedInstance: "vllm-other",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Status: execution.ProviderStatusAvailable,
|
|
},
|
|
wantClass: execution.LivenessIdentityMismatch, wantHealth: execution.HealthUnknown,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
assertProbeOutcome(t, tc.outcome, tc.wantClass, tc.wantHealth)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestClassifyProbeOutcomeInconclusive(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
outcome execution.ProbeOutcome
|
|
wantClass execution.LivenessClassification
|
|
wantHealth execution.ProviderHealth
|
|
}{
|
|
{
|
|
name: "transport error takes precedence over available status",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Status: execution.ProviderStatusAvailable, Err: errors.New("boom"),
|
|
},
|
|
wantClass: execution.LivenessError, wantHealth: execution.HealthUnknown,
|
|
},
|
|
{
|
|
name: "deadline exceeded is timeout",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Err: context.DeadlineExceeded,
|
|
},
|
|
wantClass: execution.LivenessTimeout, wantHealth: execution.HealthUnknown,
|
|
},
|
|
{
|
|
name: "cancellation is timeout",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Err: context.Canceled,
|
|
},
|
|
wantClass: execution.LivenessTimeout, wantHealth: execution.HealthUnknown,
|
|
},
|
|
{
|
|
name: "unsupported adapter",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "worker", ExpectedAdapter: "worker",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Err: execution.ErrProbeUnsupported,
|
|
},
|
|
wantClass: execution.LivenessUnsupported, wantHealth: execution.HealthUnknown,
|
|
},
|
|
{
|
|
name: "wrapped unsupported is still unsupported",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "worker", ExpectedAdapter: "worker",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Err: fmt.Errorf("resolve: %w", execution.ErrProbeUnsupported),
|
|
},
|
|
wantClass: execution.LivenessUnsupported, wantHealth: execution.HealthUnknown,
|
|
},
|
|
{
|
|
name: "unknown status",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a",
|
|
Status: execution.ProviderStatusUnknown,
|
|
},
|
|
wantClass: execution.LivenessUnknown, wantHealth: execution.HealthUnknown,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
assertProbeOutcome(t, tc.outcome, tc.wantClass, tc.wantHealth)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestClassifyProbeOutcomeIdentity(t *testing.T) {
|
|
avail := execution.ProviderStatusAvailable
|
|
mismatch := execution.LivenessIdentityMismatch
|
|
cases := []struct {
|
|
name string
|
|
outcome execution.ProbeOutcome
|
|
}{
|
|
{
|
|
name: "empty adapter identity",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a", Status: avail,
|
|
},
|
|
},
|
|
{
|
|
name: "empty expected adapter",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "",
|
|
Target: "m-a", ExpectedTarget: "m-a", Status: avail,
|
|
},
|
|
},
|
|
{
|
|
name: "empty target identity",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "", ExpectedTarget: "m-a", Status: avail,
|
|
},
|
|
},
|
|
{
|
|
name: "mismatched adapter",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "ollama", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-a", Status: avail,
|
|
},
|
|
},
|
|
{
|
|
name: "mismatched target",
|
|
outcome: execution.ProbeOutcome{
|
|
AdapterName: "vllm", ExpectedAdapter: "vllm",
|
|
Target: "m-a", ExpectedTarget: "m-b", Status: avail,
|
|
},
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
assertProbeOutcome(t, tc.outcome, mismatch, execution.HealthUnknown)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHealthFromClassification(t *testing.T) {
|
|
cases := []struct {
|
|
class execution.LivenessClassification
|
|
want execution.ProviderHealth
|
|
}{
|
|
{execution.LivenessAvailable, execution.RequestStalled},
|
|
{execution.LivenessUnavailable, execution.ProviderUnhealthy},
|
|
{execution.LivenessTimeout, execution.HealthUnknown},
|
|
{execution.LivenessError, execution.HealthUnknown},
|
|
{execution.LivenessUnsupported, execution.HealthUnknown},
|
|
{execution.LivenessUnknown, execution.HealthUnknown},
|
|
{execution.LivenessIdentityMismatch, execution.HealthUnknown},
|
|
{execution.LivenessClassification("bogus"), execution.HealthUnknown},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := execution.HealthFromClassification(tc.class); got != tc.want {
|
|
t.Errorf("HealthFromClassification(%q): got %q, want %q", tc.class, got, tc.want)
|
|
}
|
|
}
|
|
}
|