package node_test import ( "context" "errors" "io" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "go.uber.org/zap" "google.golang.org/protobuf/types/known/structpb" "iop/apps/node/internal/adapters" "iop/apps/node/internal/node" "iop/apps/node/internal/router" "iop/apps/node/internal/runtime" "iop/apps/node/internal/store" "iop/apps/node/internal/transport" iop "iop/proto/gen/iop" ) // --- registry lifecycle refresh tests --- // TestNodeConfigRefreshDoesNotStopOldRegistryWithActiveRun verifies that // an active run prevents old-registry stop during config refresh. func TestNodeConfigRefreshDoesNotStopOldRegistryWithActiveRun(t *testing.T) { reg1 := adapters.NewRegistry() oldAdapter := &lifecycleTestAdapter{ name: "my-adapter", started: make(chan struct{}), blockChan: make(chan struct{}), } reg1.RegisterKeyed("my-adapter", "lifecycle", oldAdapter) initialConfigSet := &adapters.ConfigSet{ Registry: reg1, Items: map[string]adapters.ConfigItem{ "my-adapter": { Key: "my-adapter", Type: "lifecycle", Fingerprint: "fingerprint-1", }, }, } rtr := router.New(reg1, zap.NewNop()) st, err := store.New(":memory:", zap.NewNop()) if err != nil { t.Fatalf("store: %v", err) } t.Cleanup(func() { _ = st.Close() }) n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), initialConfigSet) // Start active run on my-adapter errChan := make(chan error, 1) go func() { errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-1", Adapter: "my-adapter", Background: false, }) }() // Wait for run to start <-oldAdapter.started // Refresh to new config: "my-adapter" is removed (or updated) refreshPayload := &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{ { Name: "new-mock", Type: "mock", Enabled: true, }, }, } refreshResp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{ RequestId: "refresh-1", Config: refreshPayload, }) if refreshErr != nil { t.Fatalf("OnConfigRefresh failed: %v", refreshErr) } if refreshResp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED { t.Fatalf("expected refresh status APPLIED, got %v", refreshResp.Status) } // Verify that the old adapter's Stop was NOT called because of the active run if calls := atomic.LoadInt32(&oldAdapter.stopCalls); calls > 0 { t.Fatalf("expected old adapter Stop calls to be 0, got %d", calls) } // Unblock the run close(oldAdapter.blockChan) if err := <-errChan; err != nil { t.Fatalf("active run failed: %v", err) } // Verify that new requests now resolve to the new registry (new-mock should resolve, my-adapter should fail) _, _, err = rtr.ResolveAdapter(context.Background(), runtime.RunRequest{ RunID: "run-2", Adapter: "new-mock", }) if err != nil { t.Fatalf("expected new-mock to resolve on the new registry, got: %v", err) } _, _, err = rtr.ResolveAdapter(context.Background(), runtime.RunRequest{ RunID: "run-3", Adapter: "my-adapter", }) if err == nil { t.Fatal("expected my-adapter lookup to fail on the new registry") } } func TestNodeConfigRefreshDoesNotStopUnchangedActiveAdapterWhenSiblingChanges(t *testing.T) { blockChan := make(chan struct{}) defer func() { select { case <-blockChan: default: close(blockChan) } }() startedChan := make(chan struct{}, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if strings.Contains(r.URL.Path, "/v1/models") { _, _ = w.Write([]byte(`{"data": [{"id": "gpt-4o"}]}`)) return } select { case startedChan <- struct{}{}: default: } <-blockChan _, _ = w.Write([]byte("data: {\"choices\": [{\"delta\": {\"content\": \"hello\"}}]}\n\ndata: [DONE]\n\n")) })) defer server.Close() initialPayload := &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{ { Name: "openai", Type: "openai_compat", Enabled: true, Config: &iop.AdapterConfig_OpenaiCompat{ OpenaiCompat: &iop.OpenAICompatAdapterConfig{ Provider: "openai", Endpoint: server.URL, Capacity: 2, }, }, }, }, } set, err := adapters.BuildConfigSet(initialPayload, zap.NewNop()) if err != nil { t.Fatalf("BuildConfigSet: %v", err) } sibling := &lifecycleTestAdapter{ name: "sibling-adapter", started: make(chan struct{}), blockChan: make(chan struct{}), } set.Registry.RegisterKeyed("sibling-adapter", "lifecycle", sibling) set.Items["sibling-adapter"] = adapters.ConfigItem{ Key: "sibling-adapter", Type: "lifecycle", Fingerprint: "fingerprint-sibling-1", } rtr := router.New(set.Registry, zap.NewNop()) st, err := store.New(":memory:", zap.NewNop()) if err != nil { t.Fatalf("store: %v", err) } t.Cleanup(func() { _ = st.Close() }) n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), set) inputMap := map[string]interface{}{ "prompt": "hello", } inputStruct, err := structpb.NewStruct(inputMap) if err != nil { t.Fatalf("failed to create input struct: %v", err) } errChan := make(chan error, 1) go func() { errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-openai-active", Adapter: "openai", Target: "gpt-4o", Background: false, Input: inputStruct, }) }() <-startedChan refreshPayload := &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{ { Name: "openai", Type: "openai_compat", Enabled: true, Config: &iop.AdapterConfig_OpenaiCompat{ OpenaiCompat: &iop.OpenAICompatAdapterConfig{ Provider: "openai", Endpoint: server.URL, Capacity: 2, }, }, }, }, } refreshResp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{ RequestId: "refresh-1", Config: refreshPayload, }) if refreshErr != nil { t.Fatalf("OnConfigRefresh failed: %v", refreshErr) } if refreshResp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED { t.Fatalf("expected refresh status APPLIED, got %v", refreshResp.Status) } if calls := atomic.LoadInt32(&sibling.stopCalls); calls > 0 { t.Fatalf("expected sibling-adapter Stop calls to be 0, got %d", calls) } close(blockChan) if err := <-errChan; err != nil { t.Fatalf("active run failed: %v", err) } } func TestNodeConfigRefreshWaitsForResolvedRunRegistrationBeforeStoppingOldRegistry(t *testing.T) { reg1 := adapters.NewRegistry() blockingAdapter := &blockingCapsAdapter{ name: "blocking-adapter", capsStarted: make(chan struct{}, 1), capsBlock: make(chan struct{}), } reg1.RegisterKeyed("blocking-adapter", "lifecycle", blockingAdapter) initialConfigSet := &adapters.ConfigSet{ Registry: reg1, Items: map[string]adapters.ConfigItem{ "blocking-adapter": { Key: "blocking-adapter", Type: "lifecycle", Fingerprint: "fingerprint-1", }, }, } rtr := router.New(reg1, zap.NewNop()) st, err := store.New(":memory:", zap.NewNop()) if err != nil { t.Fatalf("store: %v", err) } t.Cleanup(func() { _ = st.Close() }) n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), initialConfigSet) errChan := make(chan error, 1) go func() { errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-1", Adapter: "blocking-adapter", Background: false, }) }() <-blockingAdapter.capsStarted refreshErrChan := make(chan error, 1) refreshRespChan := make(chan *iop.NodeConfigRefreshResponse, 1) go func() { refreshPayload := &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{ { Name: "new-mock", Type: "mock", Enabled: true, }, }, } resp, err := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{ RequestId: "refresh-1", Config: refreshPayload, }) refreshRespChan <- resp refreshErrChan <- err }() time.Sleep(50 * time.Millisecond) select { case resp := <-refreshRespChan: t.Fatalf("expected refresh to block, but it completed: %v", resp) default: // Refresh is successfully blocking! } close(blockingAdapter.capsBlock) refreshErr := <-refreshErrChan if refreshErr != nil { t.Fatalf("refresh failed: %v", refreshErr) } resp := <-refreshRespChan if resp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED { t.Fatalf("expected refresh status APPLIED, got %v", resp.Status) } runErr := <-errChan if runErr != nil { t.Fatalf("run failed: %v", runErr) } } func TestNodeConfigRefreshDeferredStopRunsAfterActiveRunCompletes(t *testing.T) { reg1 := adapters.NewRegistry() oldAdapter := &lifecycleTestAdapter{ name: "my-adapter", started: make(chan struct{}), blockChan: make(chan struct{}), } reg1.RegisterKeyed("my-adapter", "lifecycle", oldAdapter) initialConfigSet := &adapters.ConfigSet{ Registry: reg1, Items: map[string]adapters.ConfigItem{ "my-adapter": {Key: "my-adapter", Type: "lifecycle", Fingerprint: "fingerprint-1"}, }, } rtr := router.New(reg1, zap.NewNop()) st, err := store.New(":memory:", zap.NewNop()) if err != nil { t.Fatalf("store: %v", err) } t.Cleanup(func() { _ = st.Close() }) n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), initialConfigSet) errChan := make(chan error, 1) go func() { errChan <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-1", Adapter: "my-adapter", }) }() <-oldAdapter.started refreshResp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{ RequestId: "refresh-1", Config: &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{{Name: "new-mock", Type: "mock", Enabled: true}}, }, }) if refreshErr != nil { t.Fatalf("OnConfigRefresh failed: %v", refreshErr) } if refreshResp.Status != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED { t.Fatalf("expected refresh status APPLIED, got %v", refreshResp.Status) } // Old registry stop must be deferred while the run is still active. if calls := atomic.LoadInt32(&oldAdapter.stopCalls); calls != 0 { t.Fatalf("expected old adapter Stop to be deferred (0 calls), got %d", calls) } // Drain the active run; the deferred stop should fire afterwards. close(oldAdapter.blockChan) if err := <-errChan; err != nil { t.Fatalf("active run failed: %v", err) } deadline := time.After(2 * time.Second) for atomic.LoadInt32(&oldAdapter.stopCalls) == 0 { select { case <-deadline: t.Fatal("deferred old registry stop never ran after the active run completed") default: time.Sleep(5 * time.Millisecond) } } } // --- NCDRAIN-3: failed adapter start does not rollback --- // blockingAdapterWithStart blocks runs via blockChan while allowing Start to succeed. type blockingAdapterWithStart struct { started chan struct{} blockChan chan struct{} } func newBlockingAdapterWithStart() *blockingAdapterWithStart { return &blockingAdapterWithStart{ started: make(chan struct{}, 3), blockChan: make(chan struct{}, 3), } } func (a *blockingAdapterWithStart) Name() string { return "blocking" } func (a *blockingAdapterWithStart) Capabilities(_ context.Context) (runtime.Capabilities, error) { return runtime.Capabilities{ AdapterName: "blocking", MaxConcurrency: 0, // unlimited per-adapter }, nil } func (a *blockingAdapterWithStart) Execute(ctx context.Context, _ runtime.ExecutionSpec, _ runtime.EventSink) error { a.started <- struct{}{} <-a.blockChan return nil } func (a *blockingAdapterWithStart) Start(_ context.Context) error { return nil } func (a *blockingAdapterWithStart) Stop(_ context.Context) error { return nil } // TestConfigRefreshFailedAdapterStartDoesNotRollbackConfig verifies that when // adapter registry Start fails during a config refresh, the existing adapter // state and gates are preserved and the node returns FAILED status. func TestConfigRefreshFailedAdapterStartDoesNotRollbackConfig(t *testing.T) { initialPayload := &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{ { Name: "blocking", Type: "mock", Enabled: true, }, }, } set, err := adapters.BuildConfigSet(initialPayload, zap.NewNop()) if err != nil { t.Fatalf("BuildConfigSet: %v", err) } rtr := router.New(set.Registry, zap.NewNop()) st, err := store.New(":memory:", zap.NewNop()) if err != nil { t.Fatalf("store: %v", err) } t.Cleanup(func() { _ = st.Close() }) n := node.New("test-node", rtr, st, 0, io.Discard, zap.NewNop(), set) // Add a blocking adapter that occupies the adapter slot. blockingAdapt := newBlockingAdapterWithStart() set.Registry.RegisterKeyed("blocking", "mock", blockingAdapt) // Start an active run on the blocking adapter. errCh1 := make(chan error, 1) go func() { errCh1 <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-hold", Adapter: "blocking", Background: true, }) }() // Wait for the run to start. select { case <-blockingAdapt.started: case <-time.After(2 * time.Second): t.Fatal("blocking adapter never started") } // Second run on the same adapter must succeed: adapter gate is unlimited. errCh2 := make(chan error, 1) go func() { errCh2 <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-concurrent", Adapter: "blocking", Background: true, }) }() // Wait until the blocking adapter's Execute has been called (both runs started). select { case <-blockingAdapt.started: case <-time.After(2 * time.Second): t.Fatal("blocking adapter second Execute never started") } // Now send a refresh that includes a failing CLI adapter. refreshPayload := &iop.NodeConfigPayload{ Adapters: []*iop.AdapterConfig{ { Name: "blocking", Type: "mock", Enabled: true, }, { Name: "cli", Type: "cli", Enabled: true, Config: &iop.AdapterConfig_Cli{ Cli: &iop.CLIAdapterConfig{ Profiles: map[string]*iop.CLIProfileConfig{ "failprofile": { Command: "/nonexistent/path/to/binary", Args: []string{"--fail"}, Persistent: true, }, }, }, }, }, }, } resp, refreshErr := n.OnConfigRefresh(context.Background(), nil, &iop.NodeConfigRefreshRequest{ RequestId: "refresh-fail", Config: refreshPayload, }) if refreshErr != nil { t.Fatalf("OnConfigRefresh failed: %v", refreshErr) } if resp.GetStatus() != iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_FAILED { t.Fatalf("expected FAILED status, got %v", resp.GetStatus()) } // Clean up: unblock the held runs. blockingAdapt.blockChan <- struct{}{} blockingAdapt.blockChan <- struct{}{} if err := <-errCh1; err != nil && !errors.Is(err, runtime.ErrRunCancelled) { t.Fatalf("run-hold failed: %v", err) } if err := <-errCh2; err != nil && !errors.Is(err, runtime.ErrRunCancelled) { t.Fatalf("run-concurrent failed: %v", err) } // 추가 검증: failed refresh 뒤에도 기존 blocking adapter가 계속 사용 가능함을 확인한다. errCh3 := make(chan error, 1) go func() { errCh3 <- n.OnRunRequest(context.Background(), &transport.Session{}, &iop.RunRequest{ RunId: "run-post-verify", Adapter: "blocking", Background: true, }) }() select { case <-blockingAdapt.started: case <-time.After(2 * time.Second): t.Fatal("blocking adapter never started for post-verify run") } blockingAdapt.blockChan <- struct{}{} if err := <-errCh3; err != nil && !errors.Is(err, runtime.ErrRunCancelled) { t.Fatalf("post-verify run failed: %v", err) } }