toki
01dc2ef78b
- agent-readable-repository-refactor 완료: 기존 테스트 파일 아카이브 이동 - 새 테스트 파일 추가 (edge, node, client, config, readability) - readability_audit 스크립트 및 baseline 추가 - roadmap/SDD 문서 갱신 - agent-client/pi/extensions/openai-sampling-parameters 추가
573 lines
16 KiB
Go
573 lines
16 KiB
Go
package node_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
"google.golang.org/protobuf/types/known/structpb"
|
|
"iop/apps/node/internal/adapters"
|
|
"iop/apps/node/internal/node"
|
|
"iop/apps/node/internal/router"
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/apps/node/internal/store"
|
|
"iop/apps/node/internal/transport"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
// --- registry lifecycle refresh tests ---
|
|
|
|
// TestNodeConfigRefreshDoesNotStopOldRegistryWithActiveRun verifies that
|
|
// an active run prevents old-registry stop during config refresh.
|
|
func TestNodeConfigRefreshDoesNotStopOldRegistryWithActiveRun(t *testing.T) {
|
|
reg1 := adapters.NewRegistry()
|
|
oldAdapter := &lifecycleTestAdapter{
|
|
name: "my-adapter",
|
|
started: make(chan struct{}),
|
|
blockChan: make(chan struct{}),
|
|
}
|
|
reg1.RegisterKeyed("my-adapter", "lifecycle", oldAdapter)
|
|
|
|
initialConfigSet := &adapters.ConfigSet{
|
|
Registry: reg1,
|
|
Items: map[string]adapters.ConfigItem{
|
|
"my-adapter": {
|
|
Key: "my-adapter",
|
|
Type: "lifecycle",
|
|
Fingerprint: "fingerprint-1",
|
|
},
|
|
},
|
|
}
|
|
|
|
rtr := router.New(reg1, zap.NewNop())
|
|
st, err := store.New(":memory:", zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("store: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
|
|
n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), initialConfigSet)
|
|
|
|
// Start active run on my-adapter
|
|
errChan := make(chan error, 1)
|
|
go func() {
|
|
errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-1",
|
|
Adapter: "my-adapter",
|
|
Background: false,
|
|
})
|
|
}()
|
|
|
|
// Wait for run to start
|
|
<-oldAdapter.started
|
|
|
|
// Refresh to new config: "my-adapter" is removed (or updated)
|
|
refreshPayload := &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{
|
|
{
|
|
Name: "new-mock",
|
|
Type: "mock",
|
|
Enabled: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
refreshResp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{
|
|
RequestId: "refresh-1",
|
|
Config: refreshPayload,
|
|
})
|
|
if refreshErr != nil {
|
|
t.Fatalf("OnConfigRefresh failed: %v", refreshErr)
|
|
}
|
|
if refreshResp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED {
|
|
t.Fatalf("expected refresh status APPLIED, got %v", refreshResp.Status)
|
|
}
|
|
|
|
// Verify that the old adapter's Stop was NOT called because of the active run
|
|
if calls := atomic.LoadInt32(&oldAdapter.stopCalls); calls > 0 {
|
|
t.Fatalf("expected old adapter Stop calls to be 0, got %d", calls)
|
|
}
|
|
|
|
// Unblock the run
|
|
close(oldAdapter.blockChan)
|
|
if err := <-errChan; err != nil {
|
|
t.Fatalf("active run failed: %v", err)
|
|
}
|
|
|
|
// Verify that new requests now resolve to the new registry (new-mock should resolve, my-adapter should fail)
|
|
_, _, err = rtr.ResolveAdapter(context.Background(), runtime.RunRequest{
|
|
RunID: "run-2",
|
|
Adapter: "new-mock",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected new-mock to resolve on the new registry, got: %v", err)
|
|
}
|
|
|
|
_, _, err = rtr.ResolveAdapter(context.Background(), runtime.RunRequest{
|
|
RunID: "run-3",
|
|
Adapter: "my-adapter",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected my-adapter lookup to fail on the new registry")
|
|
}
|
|
}
|
|
|
|
func TestNodeConfigRefreshDoesNotStopUnchangedActiveAdapterWhenSiblingChanges(t *testing.T) {
|
|
blockChan := make(chan struct{})
|
|
defer func() {
|
|
select {
|
|
case <-blockChan:
|
|
default:
|
|
close(blockChan)
|
|
}
|
|
}()
|
|
|
|
startedChan := make(chan struct{}, 1)
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if strings.Contains(r.URL.Path, "/v1/models") {
|
|
_, _ = w.Write([]byte(`{"data": [{"id": "gpt-4o"}]}`))
|
|
return
|
|
}
|
|
select {
|
|
case startedChan <- struct{}{}:
|
|
default:
|
|
}
|
|
<-blockChan
|
|
_, _ = w.Write([]byte("data: {\"choices\": [{\"delta\": {\"content\": \"hello\"}}]}\n\ndata: [DONE]\n\n"))
|
|
}))
|
|
defer server.Close()
|
|
|
|
initialPayload := &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{
|
|
{
|
|
Name: "openai",
|
|
Type: "openai_compat",
|
|
Enabled: true,
|
|
Config: &iop.AdapterConfig_OpenaiCompat{
|
|
OpenaiCompat: &iop.OpenAICompatAdapterConfig{
|
|
Provider: "openai",
|
|
Endpoint: server.URL,
|
|
Capacity: 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
set, err := adapters.BuildConfigSet(initialPayload, zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("BuildConfigSet: %v", err)
|
|
}
|
|
|
|
sibling := &lifecycleTestAdapter{
|
|
name: "sibling-adapter",
|
|
started: make(chan struct{}),
|
|
blockChan: make(chan struct{}),
|
|
}
|
|
set.Registry.RegisterKeyed("sibling-adapter", "lifecycle", sibling)
|
|
set.Items["sibling-adapter"] = adapters.ConfigItem{
|
|
Key: "sibling-adapter",
|
|
Type: "lifecycle",
|
|
Fingerprint: "fingerprint-sibling-1",
|
|
}
|
|
|
|
rtr := router.New(set.Registry, zap.NewNop())
|
|
st, err := store.New(":memory:", zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("store: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
|
|
n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), set)
|
|
|
|
inputMap := map[string]interface{}{
|
|
"prompt": "hello",
|
|
}
|
|
inputStruct, err := structpb.NewStruct(inputMap)
|
|
if err != nil {
|
|
t.Fatalf("failed to create input struct: %v", err)
|
|
}
|
|
|
|
errChan := make(chan error, 1)
|
|
go func() {
|
|
errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-openai-active",
|
|
Adapter: "openai",
|
|
Target: "gpt-4o",
|
|
Background: false,
|
|
Input: inputStruct,
|
|
})
|
|
}()
|
|
|
|
<-startedChan
|
|
|
|
refreshPayload := &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{
|
|
{
|
|
Name: "openai",
|
|
Type: "openai_compat",
|
|
Enabled: true,
|
|
Config: &iop.AdapterConfig_OpenaiCompat{
|
|
OpenaiCompat: &iop.OpenAICompatAdapterConfig{
|
|
Provider: "openai",
|
|
Endpoint: server.URL,
|
|
Capacity: 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
refreshResp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{
|
|
RequestId: "refresh-1",
|
|
Config: refreshPayload,
|
|
})
|
|
if refreshErr != nil {
|
|
t.Fatalf("OnConfigRefresh failed: %v", refreshErr)
|
|
}
|
|
if refreshResp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED {
|
|
t.Fatalf("expected refresh status APPLIED, got %v", refreshResp.Status)
|
|
}
|
|
|
|
if calls := atomic.LoadInt32(&sibling.stopCalls); calls > 0 {
|
|
t.Fatalf("expected sibling-adapter Stop calls to be 0, got %d", calls)
|
|
}
|
|
|
|
close(blockChan)
|
|
if err := <-errChan; err != nil {
|
|
t.Fatalf("active run failed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNodeConfigRefreshWaitsForResolvedRunRegistrationBeforeStoppingOldRegistry(t *testing.T) {
|
|
reg1 := adapters.NewRegistry()
|
|
blockingAdapter := &blockingCapsAdapter{
|
|
name: "blocking-adapter",
|
|
capsStarted: make(chan struct{}, 1),
|
|
capsBlock: make(chan struct{}),
|
|
}
|
|
reg1.RegisterKeyed("blocking-adapter", "lifecycle", blockingAdapter)
|
|
|
|
initialConfigSet := &adapters.ConfigSet{
|
|
Registry: reg1,
|
|
Items: map[string]adapters.ConfigItem{
|
|
"blocking-adapter": {
|
|
Key: "blocking-adapter",
|
|
Type: "lifecycle",
|
|
Fingerprint: "fingerprint-1",
|
|
},
|
|
},
|
|
}
|
|
|
|
rtr := router.New(reg1, zap.NewNop())
|
|
st, err := store.New(":memory:", zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("store: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
|
|
n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), initialConfigSet)
|
|
|
|
errChan := make(chan error, 1)
|
|
go func() {
|
|
errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-1",
|
|
Adapter: "blocking-adapter",
|
|
Background: false,
|
|
})
|
|
}()
|
|
|
|
<-blockingAdapter.capsStarted
|
|
|
|
refreshErrChan := make(chan error, 1)
|
|
refreshRespChan := make(chan *iop.NodeConfigRefreshResponse, 1)
|
|
go func() {
|
|
refreshPayload := &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{
|
|
{
|
|
Name: "new-mock",
|
|
Type: "mock",
|
|
Enabled: true,
|
|
},
|
|
},
|
|
}
|
|
resp, err := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{
|
|
RequestId: "refresh-1",
|
|
Config: refreshPayload,
|
|
})
|
|
refreshRespChan <- resp
|
|
refreshErrChan <- err
|
|
}()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
select {
|
|
case resp := <-refreshRespChan:
|
|
t.Fatalf("expected refresh to block, but it completed: %v", resp)
|
|
default:
|
|
// Refresh is successfully blocking!
|
|
}
|
|
|
|
close(blockingAdapter.capsBlock)
|
|
|
|
refreshErr := <-refreshErrChan
|
|
if refreshErr != nil {
|
|
t.Fatalf("refresh failed: %v", refreshErr)
|
|
}
|
|
resp := <-refreshRespChan
|
|
if resp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED {
|
|
t.Fatalf("expected refresh status APPLIED, got %v", resp.Status)
|
|
}
|
|
|
|
runErr := <-errChan
|
|
if runErr != nil {
|
|
t.Fatalf("run failed: %v", runErr)
|
|
}
|
|
}
|
|
|
|
func TestNodeConfigRefreshDeferredStopRunsAfterActiveRunCompletes(t *testing.T) {
|
|
reg1 := adapters.NewRegistry()
|
|
oldAdapter := &lifecycleTestAdapter{
|
|
name: "my-adapter",
|
|
started: make(chan struct{}),
|
|
blockChan: make(chan struct{}),
|
|
}
|
|
reg1.RegisterKeyed("my-adapter", "lifecycle", oldAdapter)
|
|
|
|
initialConfigSet := &adapters.ConfigSet{
|
|
Registry: reg1,
|
|
Items: map[string]adapters.ConfigItem{
|
|
"my-adapter": {Key: "my-adapter", Type: "lifecycle", Fingerprint: "fingerprint-1"},
|
|
},
|
|
}
|
|
|
|
rtr := router.New(reg1, zap.NewNop())
|
|
|
|
st, err := store.New(":memory:", zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("store: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
|
|
n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), initialConfigSet)
|
|
|
|
errChan := make(chan error, 1)
|
|
go func() {
|
|
errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-1",
|
|
Adapter: "my-adapter",
|
|
})
|
|
}()
|
|
<-oldAdapter.started
|
|
|
|
refreshResp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{
|
|
RequestId: "refresh-1",
|
|
Config: &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{{Name: "new-mock", Type: "mock", Enabled: true}},
|
|
},
|
|
})
|
|
if refreshErr != nil {
|
|
t.Fatalf("OnConfigRefresh failed: %v", refreshErr)
|
|
}
|
|
if refreshResp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED {
|
|
t.Fatalf("expected refresh status APPLIED, got %v", refreshResp.Status)
|
|
}
|
|
|
|
// Old registry stop must be deferred while the run is still active.
|
|
if calls := atomic.LoadInt32(&oldAdapter.stopCalls); calls != 0 {
|
|
t.Fatalf("expected old adapter Stop to be deferred (0 calls), got %d", calls)
|
|
}
|
|
|
|
// Drain the active run; the deferred stop should fire afterwards.
|
|
close(oldAdapter.blockChan)
|
|
if err := <-errChan; err != nil {
|
|
t.Fatalf("active run failed: %v", err)
|
|
}
|
|
|
|
deadline := time.After(2 * time.Second)
|
|
for atomic.LoadInt32(&oldAdapter.stopCalls) == 0 {
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("deferred old registry stop never ran after the active run completed")
|
|
default:
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- NCDRAIN-3: failed adapter start does not rollback ---
|
|
|
|
// blockingAdapterWithStart blocks runs via blockChan while allowing Start to succeed.
|
|
type blockingAdapterWithStart struct {
|
|
started chan struct{}
|
|
blockChan chan struct{}
|
|
}
|
|
|
|
func newBlockingAdapterWithStart() *blockingAdapterWithStart {
|
|
return &blockingAdapterWithStart{
|
|
started: make(chan struct{}, 3),
|
|
blockChan: make(chan struct{}, 3),
|
|
}
|
|
}
|
|
|
|
func (a *blockingAdapterWithStart) Name() string { return "blocking" }
|
|
|
|
func (a *blockingAdapterWithStart) Capabilities(_ context.Context) (runtime.Capabilities, error) {
|
|
return runtime.Capabilities{
|
|
AdapterName: "blocking",
|
|
MaxConcurrency: 0, // unlimited per-adapter
|
|
}, nil
|
|
}
|
|
|
|
func (a *blockingAdapterWithStart) Execute(ctx context.Context, _ runtime.ExecutionSpec, _ runtime.EventSink) error {
|
|
a.started <- struct{}{}
|
|
<-a.blockChan
|
|
return nil
|
|
}
|
|
|
|
func (a *blockingAdapterWithStart) Start(_ context.Context) error { return nil }
|
|
func (a *blockingAdapterWithStart) Stop(_ context.Context) error { return nil }
|
|
|
|
// TestConfigRefreshFailedAdapterStartDoesNotRollbackConfig verifies that when
|
|
// adapter registry Start fails during a config refresh, the existing adapter
|
|
// state and gates are preserved and the node returns FAILED status.
|
|
func TestConfigRefreshFailedAdapterStartDoesNotRollbackConfig(t *testing.T) {
|
|
initialPayload := &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{
|
|
{
|
|
Name: "blocking",
|
|
Type: "mock",
|
|
Enabled: true,
|
|
},
|
|
},
|
|
}
|
|
set, err := adapters.BuildConfigSet(initialPayload, zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("BuildConfigSet: %v", err)
|
|
}
|
|
|
|
rtr := router.New(set.Registry, zap.NewNop())
|
|
st, err := store.New(":memory:", zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("store: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
|
|
n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), set)
|
|
|
|
// Add a blocking adapter that occupies the adapter slot.
|
|
blockingAdapt := newBlockingAdapterWithStart()
|
|
set.Registry.RegisterKeyed("blocking", "mock", blockingAdapt)
|
|
|
|
// Start an active run on the blocking adapter.
|
|
errCh1 := make(chan error, 1)
|
|
go func() {
|
|
errCh1 <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-hold",
|
|
Adapter: "blocking",
|
|
Background: true,
|
|
})
|
|
}()
|
|
|
|
// Wait for the run to start.
|
|
select {
|
|
case <-blockingAdapt.started:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("blocking adapter never started")
|
|
}
|
|
|
|
// Second run on the same adapter must succeed: adapter gate is unlimited.
|
|
errCh2 := make(chan error, 1)
|
|
go func() {
|
|
errCh2 <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-concurrent",
|
|
Adapter: "blocking",
|
|
Background: true,
|
|
})
|
|
}()
|
|
|
|
// Wait until the blocking adapter's Execute has been called (both runs started).
|
|
select {
|
|
case <-blockingAdapt.started:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("blocking adapter second Execute never started")
|
|
}
|
|
|
|
// Now send a refresh that includes a failing CLI adapter.
|
|
refreshPayload := &iop.NodeConfigPayload{
|
|
Adapters: []*iop.AdapterConfig{
|
|
{
|
|
Name: "blocking",
|
|
Type: "mock",
|
|
Enabled: true,
|
|
},
|
|
{
|
|
Name: "cli",
|
|
Type: "cli",
|
|
Enabled: true,
|
|
Config: &iop.AdapterConfig_Cli{
|
|
Cli: &iop.CLIAdapterConfig{
|
|
Profiles: map[string]*iop.CLIProfileConfig{
|
|
"failprofile": {
|
|
Command: "/nonexistent/path/to/binary",
|
|
Args: []string{"--fail"},
|
|
Persistent: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
resp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{
|
|
RequestId: "refresh-fail",
|
|
Config: refreshPayload,
|
|
})
|
|
if refreshErr != nil {
|
|
t.Fatalf("OnConfigRefresh failed: %v", refreshErr)
|
|
}
|
|
if resp.GetStatus() != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_FAILED {
|
|
t.Fatalf("expected FAILED status, got %v", resp.GetStatus())
|
|
}
|
|
|
|
// Clean up: unblock the held runs.
|
|
blockingAdapt.blockChan <- struct{}{}
|
|
blockingAdapt.blockChan <- struct{}{}
|
|
if err := <-errCh1; err != nil && !errors.Is(err, runtime.ErrRunCancelled) {
|
|
t.Fatalf("run-hold failed: %v", err)
|
|
}
|
|
if err := <-errCh2; err != nil && !errors.Is(err, runtime.ErrRunCancelled) {
|
|
t.Fatalf("run-concurrent failed: %v", err)
|
|
}
|
|
|
|
// 추가 검증: failed refresh 뒤에도 기존 blocking adapter가 계속 사용 가능함을 확인한다.
|
|
errCh3 := make(chan error, 1)
|
|
go func() {
|
|
errCh3 <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{
|
|
RunId: "run-post-verify",
|
|
Adapter: "blocking",
|
|
Background: true,
|
|
})
|
|
}()
|
|
|
|
select {
|
|
case <-blockingAdapt.started:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("blocking adapter never started for post-verify run")
|
|
}
|
|
|
|
blockingAdapt.blockChan <- struct{}{}
|
|
if err := <-errCh3; err != nil && !errors.Is(err, runtime.ErrRunCancelled) {
|
|
t.Fatalf("post-verify run failed: %v", err)
|
|
}
|
|
}
|